From 30bac6c86a279ed1cf18138c9bd9698cd67abecc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 8 Nov 2026 19:01:33 +0100 Subject: [PATCH 1/6] Rerun only the failed tests on CI retries The `nick-fields/retry` steps reran the whole suite on every attempt, so a real failure cost a second full run (up to ~26 min) before the job went red, and a test killed by `pytest-timeout` reran all tensor tests. Each pytest command inside a retry step now passes `--lf`. The first attempt has no recorded failures and runs everything; a retry runs only the tests that failed. An attempt killed by the step timeout records nothing, so the next one runs all tests again. Coverage keeps full reruns, since a partial rerun would drop coverage data. --- .github/workflows/check-onemath.yaml | 9 +++++---- .github/workflows/conda-build-test.yml | 7 ++++--- .github/workflows/conda-package.yml | 13 +++++++------ .github/workflows/cron-run-tests.yaml | 5 +++-- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index e9b53dba708..9c8a458aa8f 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -31,6 +31,7 @@ env: test-pkg-env: 'environments/test.yml' test-env-name: 'test_onemath' rerun-tests-on-failure: 'true' + # A retry reruns only the failed tests (--lf), or all of them after a timeout kill rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 @@ -183,7 +184,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs dpnp.tests + python -m pytest -ra --lf --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 SYCL_CACHE_PERSISTENT: 1 @@ -201,7 +202,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -307,7 +308,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs dpnp.tests + python -m pytest -ra --lf --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 SYCL_CACHE_PERSISTENT: 1 @@ -325,6 +326,6 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index bc6a187ef95..52f9b85805d 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -33,6 +33,7 @@ env: fetch-depth: 1 python-ver-test-all-dtypes: '3.14' test-env-name: 'test' + # A retry reruns only the failed tests (--lf), or all of them after a timeout kill rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -254,9 +255,9 @@ jobs: if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.module-name }}.tests + python -m pytest -ra --lf --pyargs ${{ env.module-name }}.tests else - python -m pytest -n auto -ra --pyargs ${{ env.module-name }}.tests + python -m pytest -n auto -ra --lf --pyargs ${{ env.module-name }}.tests fi - name: Run tensor tests @@ -272,4 +273,4 @@ jobs: . "$CONDA_UNIX/etc/profile.d/conda.sh" conda activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index e5ee385a2b5..94b6b33338d 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -27,6 +27,7 @@ env: python-ver-test-all-dtypes: '3.14' test-env-name: 'test' rerun-tests-on-failure: 'true' + # A retry reruns only the failed tests (--lf), or all of them after a timeout kill rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -281,9 +282,9 @@ jobs: if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests else - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests fi - name: Run tensor tests @@ -304,7 +305,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor test_examples_linux: name: Test examples @@ -561,9 +562,9 @@ jobs: $env:SKIP_TENSOR_TESTS=1 if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) { $env:DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests } else { - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests } - name: Run tensor tests @@ -582,7 +583,7 @@ jobs: retry_on: any shell: pwsh command: | - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor upload: name: Upload diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 4b6f7a56fad..93a3a710cd7 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -15,6 +15,7 @@ env: channels-list: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' test-env-name: test test-packages: 'pytest scipy' + # A retry reruns only the failed tests (--lf), or all of them after a timeout kill rerun-tests-max-attempts: 2 rerun-tests-timeout: 20 @@ -146,7 +147,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -159,6 +160,6 @@ jobs: max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 From 097a285ef684ebfdeaf066f4695ddeafda6fb821 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 8 Nov 2026 20:59:41 +0100 Subject: [PATCH 2/6] Continue past collection errors in serial runs a --lf retry follows A collection error interrupts a serial pytest run before any test runs and records only the broken module, so a `--lf` retry ran just that module and could pass without the rest of the suite. With `--continue-on-collection-errors` the first attempt runs the other tests, and the retry covers their failures too. The `-n auto` runs don't need it, since xdist already runs the remaining tests after a collection error. --- .github/workflows/check-onemath.yaml | 5 +++-- .github/workflows/conda-build-test.yml | 3 ++- .github/workflows/conda-package.yml | 5 +++-- .github/workflows/cron-run-tests.yaml | 7 ++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 9c8a458aa8f..3543f6b030e 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -32,6 +32,7 @@ env: test-env-name: 'test_onemath' rerun-tests-on-failure: 'true' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill + # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 @@ -184,7 +185,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --lf --pyargs dpnp.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 SYCL_CACHE_PERSISTENT: 1 @@ -308,7 +309,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --lf --pyargs dpnp.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 SYCL_CACHE_PERSISTENT: 1 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index 52f9b85805d..9abe5eadf89 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -34,6 +34,7 @@ env: python-ver-test-all-dtypes: '3.14' test-env-name: 'test' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill + # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -255,7 +256,7 @@ jobs: if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --lf --pyargs ${{ env.module-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.module-name }}.tests else python -m pytest -n auto -ra --lf --pyargs ${{ env.module-name }}.tests fi diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 94b6b33338d..10f7e007df4 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -28,6 +28,7 @@ env: test-env-name: 'test' rerun-tests-on-failure: 'true' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill + # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -282,7 +283,7 @@ jobs: if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests else python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests fi @@ -562,7 +563,7 @@ jobs: $env:SKIP_TENSOR_TESTS=1 if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) { $env:DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests } else { python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests } diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 93a3a710cd7..1dab71e3186 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -16,6 +16,7 @@ env: test-env-name: test test-packages: 'pytest scipy' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill + # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 20 @@ -130,7 +131,7 @@ jobs: id: run_tests continue-on-error: true run: | - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -147,7 +148,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -160,6 +161,6 @@ jobs: max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | - python -m pytest -ra --lf --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 From 2b9b5e0eeb6c8989332e12902cdf66eefc2d7d5f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:34:05 +0100 Subject: [PATCH 3/6] Name the tests a --lf retry reruns in a warning annotation Before each --lf pytest run, print the tests recorded in the last-failed cache as a ::warning:: annotation. A first attempt has no cache and prints nothing, so the warning appears only on retries, and a test that fails in the full run but passes on its own stays visible on the run summary, also for fork PRs. --- .github/workflows/check-onemath.yaml | 7 +++++++ .github/workflows/conda-build-test.yml | 5 +++++ .github/workflows/conda-package.yml | 7 +++++++ .github/workflows/cron-run-tests.yaml | 5 +++++ 4 files changed, 24 insertions(+) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 3543f6b030e..721c7f4f56a 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -33,6 +33,9 @@ env: rerun-tests-on-failure: 'true' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests + # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + report-last-failed: >- + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 @@ -185,6 +188,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 @@ -203,6 +207,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -309,6 +314,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 @@ -327,6 +333,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index 9abe5eadf89..4274890fc73 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -35,6 +35,9 @@ env: test-env-name: 'test' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests + # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + report-last-failed: >- + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -254,6 +257,7 @@ jobs: conda activate ${{ env.test-env-name }} export SKIP_TENSOR_TESTS=1 + python -c "${{ env.report-last-failed }}" if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.module-name }}.tests @@ -274,4 +278,5 @@ jobs: . "$CONDA_UNIX/etc/profile.d/conda.sh" conda activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 10f7e007df4..bfbdd94fcbf 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -29,6 +29,9 @@ env: rerun-tests-on-failure: 'true' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests + # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + report-last-failed: >- + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -281,6 +284,7 @@ jobs: mamba activate ${{ env.test-env-name }} export SKIP_TENSOR_TESTS=1 + python -c "${{ env.report-last-failed }}" if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests @@ -306,6 +310,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor test_examples_linux: @@ -561,6 +566,7 @@ jobs: shell: pwsh command: | $env:SKIP_TENSOR_TESTS=1 + python -c "${{ env.report-last-failed }}" if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) { $env:DPNP_TEST_ALL_INT_TYPES=1 python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests @@ -584,6 +590,7 @@ jobs: retry_on: any shell: pwsh command: | + python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor upload: diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 1dab71e3186..855ac0ae3c4 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -17,6 +17,9 @@ env: test-packages: 'pytest scipy' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests + # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + report-last-failed: >- + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) rerun-tests-max-attempts: 2 rerun-tests-timeout: 20 @@ -148,6 +151,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} + python -c "${{ env.report-last-failed }}" python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -161,6 +165,7 @@ jobs: max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | + python -c "${{ env.report-last-failed }}" python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 From bd17ed78de66b2d6b9644d7fd922fc432a971f49 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:40:41 +0100 Subject: [PATCH 4/6] Post the tests a --lf retry reran as a PR comment The conda-package test jobs record the tests each --lf retry reruns and upload them as an artifact. A report job, the only one with pull-requests: write, collects them and posts or updates one PR comment listing them per job and suite, or rewrites an earlier comment when nothing was retried. Like the Array API comment it is skipped on fork PRs, where the warning annotations still name the tests. --- .github/workflows/check-onemath.yaml | 3 +- .github/workflows/conda-build-test.yml | 3 +- .github/workflows/conda-package.yml | 96 +++++++++++++++++++++++++- .github/workflows/cron-run-tests.yaml | 3 +- 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 721c7f4f56a..e6df515d1f6 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -34,8 +34,9 @@ env: # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + # With TEST_RETRIES_FILE set, also appends them there as a JSON line report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index 4274890fc73..5b4f11dceb3 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -36,8 +36,9 @@ env: # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + # With TEST_RETRIES_FILE set, also appends them there as a JSON line report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index bfbdd94fcbf..f5df3222e5d 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -30,8 +30,9 @@ env: # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + # With TEST_RETRIES_FILE set, also appends them there as a JSON line report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -190,6 +191,7 @@ jobs: channel-path: '${{ github.workspace }}/channel/' pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' ver-json-path: '${{ github.workspace }}/version.json' + TEST_RETRIES_FILE: '${{ github.workspace }}/test-retries.jsonl' python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" @@ -313,6 +315,15 @@ jobs: python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + - name: Upload the list of retried tests + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-retries ${{ runner.os }} Python ${{ env.python-label }} + path: ${{ env.TEST_RETRIES_FILE }} + if-no-files-found: ignore + retention-days: 1 + test_examples_linux: name: Test examples @@ -447,6 +458,7 @@ jobs: channel-path: '${{ github.workspace }}\channel\' pkg-path-in-channel: '${{ github.workspace }}\channel\win-64\' ver-json-path: '${{ github.workspace }}\version.json' + TEST_RETRIES_FILE: '${{ github.workspace }}\test-retries.jsonl' python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" @@ -593,6 +605,88 @@ jobs: python -c "${{ env.report-last-failed }}" python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + - name: Upload the list of retried tests + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-retries ${{ runner.os }} Python ${{ env.python-label }} + path: ${{ env.TEST_RETRIES_FILE }} + if-no-files-found: ignore + retention-days: 1 + + report_test_retries: + name: Report retried tests + + needs: [test_linux, test_windows] + + # Skipped on fork PRs, whose token can't comment; the warning annotations still name the tests + if: ${{ !cancelled() && needs.test_linux.result != 'skipped' && github.event.pull_request && !github.event.pull_request.head.repo.fork }} + + permissions: + # Needed to add a comment to a pull request's issue + pull-requests: write + + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Download the lists of retried tests + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: test-retries * + path: test-retries + + - name: Compose the report + id: compose + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python3 - <<'EOF' + import json, os, pathlib + + max_tests = 100 + sections = [] + for d in sorted(pathlib.Path('test-retries').glob('*')): + job = d.name.removeprefix('test-retries ') + for line in (d / 'test-retries.jsonl').read_text().splitlines(): + rec = json.loads(line) + # node IDs are relative to the --pyargs package + suite, prefix = ('tensor', 'dpnp/tests/tensor/') if rec['tensor'] else ('dpnp', 'dpnp/tests/') + tests = [prefix + t for t in rec['tests']] + shown = tests[:max_tests] + ['...'] * (len(tests) > max_tests) + body = '\n'.join(shown) + sections.append( + f'
{job}, {suite} tests: {len(tests)} rerun\n\n' + f'```\n{body}\n```\n\n
' + ) + + if sections: + head = ( + f'Tests retried on CI at {os.environ["HEAD_SHA"]}: they failed in the full run and were ' + 'rerun on their own with `--lf`. A test that passed on the rerun may depend on the tests ' + 'that ran before it.' + ) + pathlib.Path('test-retries.md').write_text('\n\n'.join([head] + sections) + '\n') + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write(f'retried={"true" if sections else "false"}\n') + EOF + + - name: Post the retried tests to the PR + if: steps.compose.outputs.retried == 'true' + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 + with: + message-id: test_retries + message-path: test-retries.md + + - name: Update an earlier report when nothing was retried + if: steps.compose.outputs.retried != 'true' + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 + with: + message-id: test_retries + update-only: true + message: | + No tests were retried on CI at ${{ github.event.pull_request.head.sha }}. + upload: name: Upload diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 855ac0ae3c4..a80db05f8d1 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -18,8 +18,9 @@ env: # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible + # With TEST_RETRIES_FILE set, also appends them there as a JSON line report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))) + import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 20 From 49a584c9640acb917f0b99c758a93ec2ab610309 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:53:40 +0100 Subject: [PATCH 5/6] Move the retry reporting into a script The inline one-liner copied into four workflows and the report job's heredoc are replaced by .github/scripts/test_retries.py, with a record command run before each --lf attempt and a report command that composes the PR comment. Cron's sparse checkout and the report job now also check out .github/scripts. --- .github/scripts/test_retries.py | 131 +++++++++++++++++++++++++ .github/workflows/check-onemath.yaml | 12 +-- .github/workflows/conda-build-test.yml | 8 +- .github/workflows/conda-package.yml | 48 +++------ .github/workflows/cron-run-tests.yaml | 14 ++- 5 files changed, 154 insertions(+), 59 deletions(-) create mode 100644 .github/scripts/test_retries.py diff --git a/.github/scripts/test_retries.py b/.github/scripts/test_retries.py new file mode 100644 index 00000000000..01eedb4d5b0 --- /dev/null +++ b/.github/scripts/test_retries.py @@ -0,0 +1,131 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Report the tests a CI retry reruns with ``pytest --lf``. + +``record`` runs before each ``--lf`` attempt: it prints the tests left in +pytest's last-failed cache as a GitHub warning annotation and, with +``TEST_RETRIES_FILE`` set, appends them there as a JSON line. A first attempt +has no cache, so it prints nothing. + +``report`` collects the recorded files, downloaded as one directory per job +artifact, into the markdown of a PR comment and sets the ``retried`` step +output. +""" + +import argparse +import json +import os +import pathlib + +LASTFAILED = pathlib.Path(".pytest_cache", "v", "cache", "lastfailed") +MAX_ANNOTATED = 50 +MAX_REPORTED = 100 + + +def _capped(tests, limit): + return tests[:limit] + ["..."] * (len(tests) > limit) + + +def record(): + # the cache is in the rootdir, which is the cwd with --pyargs + if not LASTFAILED.exists(): + return + tests = sorted(json.loads(LASTFAILED.read_text())) + if not tests: + return + + # '%' and newlines must be escaped in a workflow command + msg = "%0A".join( + t.replace("%", "%25") for t in _capped(tests, MAX_ANNOTATED) + ) + print(f"::warning title=Rerunning {len(tests)} failed tests::{msg}") + + path = os.environ.get("TEST_RETRIES_FILE") + if path: + # SKIP_TENSOR_TESTS is set in exactly the dpnp suite runs + tensor = not os.environ.get("SKIP_TENSOR_TESTS") + with open(path, "a", encoding="utf-8") as f: + f.write(json.dumps({"tensor": tensor, "tests": tests}) + "\n") + + +def report(artifacts_dir, out_file, prefix): + sections = [] + for d in sorted(pathlib.Path(artifacts_dir).glob("*")): + job = d.name.removeprefix(prefix) + for f in sorted(d.glob("*.jsonl")): + for line in f.read_text(encoding="utf-8").splitlines(): + rec = json.loads(line) + # node IDs are relative to the --pyargs package + if rec["tensor"]: + suite, pkg = "tensor", "dpnp/tests/tensor/" + else: + suite, pkg = "dpnp", "dpnp/tests/" + tests = [pkg + t for t in rec["tests"]] + body = "\n".join(_capped(tests, MAX_REPORTED)) + sections.append( + f"
{job}, {suite} tests: " + f"{len(tests)} rerun\n\n" + f"```\n{body}\n```\n\n
" + ) + + if sections: + head = ( + f"Tests retried on CI at {os.environ['HEAD_SHA']}: they failed " + "in the full run and were rerun on their own with `--lf`. A test " + "that passed on the rerun may depend on the tests that ran " + "before it." + ) + pathlib.Path(out_file).write_text( + "\n\n".join([head] + sections) + "\n", encoding="utf-8" + ) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"retried={'true' if sections else 'false'}\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("record", help="annotate and record the tests to rerun") + rep = sub.add_parser("report", help="compose the PR comment") + rep.add_argument("artifacts_dir", help="directory of downloaded artifacts") + rep.add_argument("out_file", help="markdown file to write") + rep.add_argument( + "--prefix", default="", help="artifact name prefix to drop" + ) + args = parser.parse_args() + + if args.cmd == "record": + record() + else: + report(args.artifacts_dir, args.out_file, args.prefix) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index e6df515d1f6..6ef11769dfb 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -33,10 +33,6 @@ env: rerun-tests-on-failure: 'true' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests - # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible - # With TEST_RETRIES_FILE set, also appends them there as a JSON line - report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 @@ -189,7 +185,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python ".github/scripts/test_retries.py" record python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 @@ -208,7 +204,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python ".github/scripts/test_retries.py" record python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -315,7 +311,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python ".github/scripts/test_retries.py" record python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 @@ -334,7 +330,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python ".github/scripts/test_retries.py" record python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index 5b4f11dceb3..a1f0a0f0cfe 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -35,10 +35,6 @@ env: test-env-name: 'test' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests - # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible - # With TEST_RETRIES_FILE set, also appends them there as a JSON line - report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -258,7 +254,7 @@ jobs: conda activate ${{ env.test-env-name }} export SKIP_TENSOR_TESTS=1 - python -c "${{ env.report-last-failed }}" + python "${{ env.dpnp-repo-path }}/.github/scripts/test_retries.py" record if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.module-name }}.tests @@ -279,5 +275,5 @@ jobs: . "$CONDA_UNIX/etc/profile.d/conda.sh" conda activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python "${{ env.dpnp-repo-path }}/.github/scripts/test_retries.py" record python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index f5df3222e5d..160495ed17d 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -29,10 +29,6 @@ env: rerun-tests-on-failure: 'true' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests - # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible - # With TEST_RETRIES_FILE set, also appends them there as a JSON line - report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 @@ -286,7 +282,7 @@ jobs: mamba activate ${{ env.test-env-name }} export SKIP_TENSOR_TESTS=1 - python -c "${{ env.report-last-failed }}" + python "${{ env.dpnp-repo-path }}.github/scripts/test_retries.py" record if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests @@ -312,7 +308,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python "${{ env.dpnp-repo-path }}.github/scripts/test_retries.py" record python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor - name: Upload the list of retried tests @@ -578,7 +574,7 @@ jobs: shell: pwsh command: | $env:SKIP_TENSOR_TESTS=1 - python -c "${{ env.report-last-failed }}" + python "${{ env.dpnp-repo-path }}\.github\scripts\test_retries.py" record if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) { $env:DPNP_TEST_ALL_INT_TYPES=1 python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests @@ -602,7 +598,7 @@ jobs: retry_on: any shell: pwsh command: | - python -c "${{ env.report-last-failed }}" + python "${{ env.dpnp-repo-path }}\.github\scripts\test_retries.py" record python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor - name: Upload the list of retried tests @@ -630,6 +626,12 @@ jobs: timeout-minutes: 10 steps: + - name: Checkout DPNP repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: .github/scripts + - name: Download the lists of retried tests uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -641,35 +643,7 @@ jobs: env: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - python3 - <<'EOF' - import json, os, pathlib - - max_tests = 100 - sections = [] - for d in sorted(pathlib.Path('test-retries').glob('*')): - job = d.name.removeprefix('test-retries ') - for line in (d / 'test-retries.jsonl').read_text().splitlines(): - rec = json.loads(line) - # node IDs are relative to the --pyargs package - suite, prefix = ('tensor', 'dpnp/tests/tensor/') if rec['tensor'] else ('dpnp', 'dpnp/tests/') - tests = [prefix + t for t in rec['tests']] - shown = tests[:max_tests] + ['...'] * (len(tests) > max_tests) - body = '\n'.join(shown) - sections.append( - f'
{job}, {suite} tests: {len(tests)} rerun\n\n' - f'```\n{body}\n```\n\n
' - ) - - if sections: - head = ( - f'Tests retried on CI at {os.environ["HEAD_SHA"]}: they failed in the full run and were ' - 'rerun on their own with `--lf`. A test that passed on the rerun may depend on the tests ' - 'that ran before it.' - ) - pathlib.Path('test-retries.md').write_text('\n\n'.join([head] + sections) + '\n') - with open(os.environ['GITHUB_OUTPUT'], 'a') as out: - out.write(f'retried={"true" if sections else "false"}\n') - EOF + python3 .github/scripts/test_retries.py report test-retries test-retries.md --prefix "test-retries " - name: Post the retried tests to the PR if: steps.compose.outputs.retried == 'true' diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index a80db05f8d1..504f7311048 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -17,10 +17,6 @@ env: test-packages: 'pytest scipy' # A retry reruns only the failed tests (--lf), or all of them after a timeout kill # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests - # Names the tests a --lf retry reruns, so a test that fails only in the full run stays visible - # With TEST_RETRIES_FILE set, also appends them there as a JSON line - report-last-failed: >- - import json, os; p = os.path.join('.pytest_cache', 'v', 'cache', 'lastfailed'); f = sorted(json.load(open(p))) if os.path.exists(p) else []; f and print(f'::warning title=Rerunning {len(f)} failed tests::' + '%0A'.join([t.replace('%', '%25') for t in f[:50]] + ['...'] * (len(f) > 50))); r = os.environ.get('TEST_RETRIES_FILE'); f and r and open(r, 'a').write(json.dumps({'tensor': not os.environ.get('SKIP_TENSOR_TESTS'), 'tests': f}) + chr(10)) rerun-tests-max-attempts: 2 rerun-tests-timeout: 20 @@ -80,12 +76,14 @@ jobs: run: | echo "Latest tag is ${{ steps.find_latest_tag.outputs.tag }}" - # Only for the local setup-miniconda action; the tests run from the installed package + # Only for the local setup-miniconda action and the CI scripts; the tests run from the installed package - name: Checkout DPNP repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - sparse-checkout: .github/actions + sparse-checkout: | + .github/actions + .github/scripts - name: Setup miniconda uses: ./.github/actions/setup-miniconda @@ -152,7 +150,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -c "${{ env.report-last-failed }}" + python ".github/scripts/test_retries.py" record python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -166,7 +164,7 @@ jobs: max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | - python -c "${{ env.report-last-failed }}" + python ".github/scripts/test_retries.py" record python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 From 14eea92ecec86d701459dfba1d4c1af2186bd5a5 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:58:02 +0100 Subject: [PATCH 6/6] Drop the env-block comments about --lf retries They sat above the retry settings but described pytest flags far below them; the reasons are in the commit history and the PR description. --- .github/workflows/check-onemath.yaml | 2 -- .github/workflows/conda-build-test.yml | 2 -- .github/workflows/conda-package.yml | 2 -- .github/workflows/cron-run-tests.yaml | 2 -- 4 files changed, 8 deletions(-) diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index 6ef11769dfb..38d75b951ff 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -31,8 +31,6 @@ env: test-pkg-env: 'environments/test.yml' test-env-name: 'test_onemath' rerun-tests-on-failure: 'true' - # A retry reruns only the failed tests (--lf), or all of them after a timeout kill - # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 30 rerun-tensor-tests-timeout: 10 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index a1f0a0f0cfe..d2f346651e5 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -33,8 +33,6 @@ env: fetch-depth: 1 python-ver-test-all-dtypes: '3.14' test-env-name: 'test' - # A retry reruns only the failed tests (--lf), or all of them after a timeout kill - # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 160495ed17d..f220e76f01d 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -27,8 +27,6 @@ env: python-ver-test-all-dtypes: '3.14' test-env-name: 'test' rerun-tests-on-failure: 'true' - # A retry reruns only the failed tests (--lf), or all of them after a timeout kill - # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 45 rerun-tensor-tests-timeout: 15 diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 504f7311048..6f486b18e21 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -15,8 +15,6 @@ env: channels-list: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels' test-env-name: test test-packages: 'pytest scipy' - # A retry reruns only the failed tests (--lf), or all of them after a timeout kill - # Serial runs also need --continue-on-collection-errors, or a collection error skips all other tests rerun-tests-max-attempts: 2 rerun-tests-timeout: 20