diff --git a/.github/scripts/test_retries.py b/.github/scripts/test_retries.py new file mode 100644 index 00000000000..01eedb4d5b0 --- /dev/null +++ b/.github/scripts/test_retries.py @@ -0,0 +1,131 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Report the tests a CI retry reruns with ``pytest --lf``. + +``record`` runs before each ``--lf`` attempt: it prints the tests left in +pytest's last-failed cache as a GitHub warning annotation and, with +``TEST_RETRIES_FILE`` set, appends them there as a JSON line. A first attempt +has no cache, so it prints nothing. + +``report`` collects the recorded files, downloaded as one directory per job +artifact, into the markdown of a PR comment and sets the ``retried`` step +output. +""" + +import argparse +import json +import os +import pathlib + +LASTFAILED = pathlib.Path(".pytest_cache", "v", "cache", "lastfailed") +MAX_ANNOTATED = 50 +MAX_REPORTED = 100 + + +def _capped(tests, limit): + return tests[:limit] + ["..."] * (len(tests) > limit) + + +def record(): + # the cache is in the rootdir, which is the cwd with --pyargs + if not LASTFAILED.exists(): + return + tests = sorted(json.loads(LASTFAILED.read_text())) + if not tests: + return + + # '%' and newlines must be escaped in a workflow command + msg = "%0A".join( + t.replace("%", "%25") for t in _capped(tests, MAX_ANNOTATED) + ) + print(f"::warning title=Rerunning {len(tests)} failed tests::{msg}") + + path = os.environ.get("TEST_RETRIES_FILE") + if path: + # SKIP_TENSOR_TESTS is set in exactly the dpnp suite runs + tensor = not os.environ.get("SKIP_TENSOR_TESTS") + with open(path, "a", encoding="utf-8") as f: + f.write(json.dumps({"tensor": tensor, "tests": tests}) + "\n") + + +def report(artifacts_dir, out_file, prefix): + sections = [] + for d in sorted(pathlib.Path(artifacts_dir).glob("*")): + job = d.name.removeprefix(prefix) + for f in sorted(d.glob("*.jsonl")): + for line in f.read_text(encoding="utf-8").splitlines(): + rec = json.loads(line) + # node IDs are relative to the --pyargs package + if rec["tensor"]: + suite, pkg = "tensor", "dpnp/tests/tensor/" + else: + suite, pkg = "dpnp", "dpnp/tests/" + tests = [pkg + t for t in rec["tests"]] + body = "\n".join(_capped(tests, MAX_REPORTED)) + sections.append( + f"
{job}, {suite} tests: " + f"{len(tests)} rerun\n\n" + f"```\n{body}\n```\n\n
" + ) + + if sections: + head = ( + f"Tests retried on CI at {os.environ['HEAD_SHA']}: they failed " + "in the full run and were rerun on their own with `--lf`. A test " + "that passed on the rerun may depend on the tests that ran " + "before it." + ) + pathlib.Path(out_file).write_text( + "\n\n".join([head] + sections) + "\n", encoding="utf-8" + ) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"retried={'true' if sections else 'false'}\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("record", help="annotate and record the tests to rerun") + rep = sub.add_parser("report", help="compose the PR comment") + rep.add_argument("artifacts_dir", help="directory of downloaded artifacts") + rep.add_argument("out_file", help="markdown file to write") + rep.add_argument( + "--prefix", default="", help="artifact name prefix to drop" + ) + args = parser.parse_args() + + if args.cmd == "record": + record() + else: + report(args.artifacts_dir, args.out_file, args.prefix) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/check-onemath.yaml b/.github/workflows/check-onemath.yaml index e9b53dba708..38d75b951ff 100644 --- a/.github/workflows/check-onemath.yaml +++ b/.github/workflows/check-onemath.yaml @@ -183,7 +183,8 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs dpnp.tests + python ".github/scripts/test_retries.py" record + python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 SYCL_CACHE_PERSISTENT: 1 @@ -201,7 +202,8 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python ".github/scripts/test_retries.py" record + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 @@ -307,7 +309,8 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs dpnp.tests + python ".github/scripts/test_retries.py" record + python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests env: SKIP_TENSOR_TESTS: 1 SYCL_CACHE_PERSISTENT: 1 @@ -325,6 +328,7 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python ".github/scripts/test_retries.py" record + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor env: SYCL_CACHE_PERSISTENT: 1 diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml index bc6a187ef95..d2f346651e5 100644 --- a/.github/workflows/conda-build-test.yml +++ b/.github/workflows/conda-build-test.yml @@ -252,11 +252,12 @@ jobs: conda activate ${{ env.test-env-name }} export SKIP_TENSOR_TESTS=1 + python "${{ env.dpnp-repo-path }}/.github/scripts/test_retries.py" record if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.module-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.module-name }}.tests else - python -m pytest -n auto -ra --pyargs ${{ env.module-name }}.tests + python -m pytest -n auto -ra --lf --pyargs ${{ env.module-name }}.tests fi - name: Run tensor tests @@ -272,4 +273,5 @@ jobs: . "$CONDA_UNIX/etc/profile.d/conda.sh" conda activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python "${{ env.dpnp-repo-path }}/.github/scripts/test_retries.py" record + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index e5ee385a2b5..f220e76f01d 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -185,6 +185,7 @@ jobs: channel-path: '${{ github.workspace }}/channel/' pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/' ver-json-path: '${{ github.workspace }}/version.json' + TEST_RETRIES_FILE: '${{ github.workspace }}/test-retries.jsonl' python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" @@ -279,11 +280,12 @@ jobs: mamba activate ${{ env.test-env-name }} export SKIP_TENSOR_TESTS=1 + python "${{ env.dpnp-repo-path }}.github/scripts/test_retries.py" record if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then export DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests else - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests fi - name: Run tensor tests @@ -304,7 +306,17 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python "${{ env.dpnp-repo-path }}.github/scripts/test_retries.py" record + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + + - name: Upload the list of retried tests + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-retries ${{ runner.os }} Python ${{ env.python-label }} + path: ${{ env.TEST_RETRIES_FILE }} + if-no-files-found: ignore + retention-days: 1 test_examples_linux: name: Test examples @@ -440,6 +452,7 @@ jobs: channel-path: '${{ github.workspace }}\channel\' pkg-path-in-channel: '${{ github.workspace }}\channel\win-64\' ver-json-path: '${{ github.workspace }}\version.json' + TEST_RETRIES_FILE: '${{ github.workspace }}\test-retries.jsonl' python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }} python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}" @@ -559,11 +572,12 @@ jobs: shell: pwsh command: | $env:SKIP_TENSOR_TESTS=1 + python "${{ env.dpnp-repo-path }}\.github\scripts\test_retries.py" record if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) { $env:DPNP_TEST_ALL_INT_TYPES=1 - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests } else { - python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests } - name: Run tensor tests @@ -582,7 +596,68 @@ jobs: retry_on: any shell: pwsh command: | - python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + python "${{ env.dpnp-repo-path }}\.github\scripts\test_retries.py" record + python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor + + - name: Upload the list of retried tests + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-retries ${{ runner.os }} Python ${{ env.python-label }} + path: ${{ env.TEST_RETRIES_FILE }} + if-no-files-found: ignore + retention-days: 1 + + report_test_retries: + name: Report retried tests + + needs: [test_linux, test_windows] + + # Skipped on fork PRs, whose token can't comment; the warning annotations still name the tests + if: ${{ !cancelled() && needs.test_linux.result != 'skipped' && github.event.pull_request && !github.event.pull_request.head.repo.fork }} + + permissions: + # Needed to add a comment to a pull request's issue + pull-requests: write + + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout DPNP repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - name: Download the lists of retried tests + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: test-retries * + path: test-retries + + - name: Compose the report + id: compose + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python3 .github/scripts/test_retries.py report test-retries test-retries.md --prefix "test-retries " + + - name: Post the retried tests to the PR + if: steps.compose.outputs.retried == 'true' + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 + with: + message-id: test_retries + message-path: test-retries.md + + - name: Update an earlier report when nothing was retried + if: steps.compose.outputs.retried != 'true' + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 + with: + message-id: test_retries + update-only: true + message: | + No tests were retried on CI at ${{ github.event.pull_request.head.sha }}. upload: name: Upload diff --git a/.github/workflows/cron-run-tests.yaml b/.github/workflows/cron-run-tests.yaml index 4b6f7a56fad..6f486b18e21 100644 --- a/.github/workflows/cron-run-tests.yaml +++ b/.github/workflows/cron-run-tests.yaml @@ -74,12 +74,14 @@ jobs: run: | echo "Latest tag is ${{ steps.find_latest_tag.outputs.tag }}" - # Only for the local setup-miniconda action; the tests run from the installed package + # Only for the local setup-miniconda action and the CI scripts; the tests run from the installed package - name: Checkout DPNP repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - sparse-checkout: .github/actions + sparse-checkout: | + .github/actions + .github/scripts - name: Setup miniconda uses: ./.github/actions/setup-miniconda @@ -129,7 +131,7 @@ jobs: id: run_tests continue-on-error: true run: | - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python -m pytest -ra --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -146,7 +148,8 @@ jobs: . $CONDA/etc/profile.d/mamba.sh mamba activate ${{ env.test-env-name }} - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python ".github/scripts/test_retries.py" record + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1 @@ -159,6 +162,7 @@ jobs: max_attempts: ${{ env.rerun-tests-max-attempts }} retry_on: any command: | - python -m pytest -ra --pyargs ${{ env.package-name }}.tests + python ".github/scripts/test_retries.py" record + python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests env: SYCL_CACHE_PERSISTENT: 1