Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions .github/scripts/test_retries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# *****************************************************************************
# Copyright (c) 2026, Intel Corporation
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# - Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# - Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# *****************************************************************************

"""Report the tests a CI retry reruns with ``pytest --lf``.

``record`` runs before each ``--lf`` attempt: it prints the tests left in
pytest's last-failed cache as a GitHub warning annotation and, with
``TEST_RETRIES_FILE`` set, appends them there as a JSON line. A first attempt
has no cache, so it prints nothing.

``report`` collects the recorded files, downloaded as one directory per job
artifact, into the markdown of a PR comment and sets the ``retried`` step
output.
"""

import argparse
import json
import os
import pathlib

LASTFAILED = pathlib.Path(".pytest_cache", "v", "cache", "lastfailed")
MAX_ANNOTATED = 50
MAX_REPORTED = 100


def _capped(tests, limit):
return tests[:limit] + ["..."] * (len(tests) > limit)


def record():
# the cache is in the rootdir, which is the cwd with --pyargs
if not LASTFAILED.exists():
return
tests = sorted(json.loads(LASTFAILED.read_text()))
if not tests:
return

# '%' and newlines must be escaped in a workflow command
msg = "%0A".join(
t.replace("%", "%25") for t in _capped(tests, MAX_ANNOTATED)
)
print(f"::warning title=Rerunning {len(tests)} failed tests::{msg}")

path = os.environ.get("TEST_RETRIES_FILE")
if path:
# SKIP_TENSOR_TESTS is set in exactly the dpnp suite runs
tensor = not os.environ.get("SKIP_TENSOR_TESTS")
with open(path, "a", encoding="utf-8") as f:
f.write(json.dumps({"tensor": tensor, "tests": tests}) + "\n")


def report(artifacts_dir, out_file, prefix):
sections = []
for d in sorted(pathlib.Path(artifacts_dir).glob("*")):
job = d.name.removeprefix(prefix)
for f in sorted(d.glob("*.jsonl")):
for line in f.read_text(encoding="utf-8").splitlines():
rec = json.loads(line)
# node IDs are relative to the --pyargs package
if rec["tensor"]:
suite, pkg = "tensor", "dpnp/tests/tensor/"
else:
suite, pkg = "dpnp", "dpnp/tests/"
tests = [pkg + t for t in rec["tests"]]
body = "\n".join(_capped(tests, MAX_REPORTED))
sections.append(
f"<details><summary>{job}, {suite} tests: "
f"{len(tests)} rerun</summary>\n\n"
f"```\n{body}\n```\n\n</details>"
)

if sections:
head = (
f"Tests retried on CI at {os.environ['HEAD_SHA']}: they failed "
"in the full run and were rerun on their own with `--lf`. A test "
"that passed on the rerun may depend on the tests that ran "
"before it."
)
pathlib.Path(out_file).write_text(
"\n\n".join([head] + sections) + "\n", encoding="utf-8"
)

with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
f.write(f"retried={'true' if sections else 'false'}\n")


def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("record", help="annotate and record the tests to rerun")
rep = sub.add_parser("report", help="compose the PR comment")
rep.add_argument("artifacts_dir", help="directory of downloaded artifacts")
rep.add_argument("out_file", help="markdown file to write")
rep.add_argument(
"--prefix", default="", help="artifact name prefix to drop"
)
args = parser.parse_args()

if args.cmd == "record":
record()
else:
report(args.artifacts_dir, args.out_file, args.prefix)


if __name__ == "__main__":
main()
12 changes: 8 additions & 4 deletions .github/workflows/check-onemath.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,8 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs dpnp.tests
python ".github/scripts/test_retries.py" record
python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests
env:
SKIP_TENSOR_TESTS: 1
SYCL_CACHE_PERSISTENT: 1
Expand All @@ -201,7 +202,8 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python ".github/scripts/test_retries.py" record
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
env:
SYCL_CACHE_PERSISTENT: 1

Expand Down Expand Up @@ -307,7 +309,8 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs dpnp.tests
python ".github/scripts/test_retries.py" record
python -m pytest -ra --lf --continue-on-collection-errors --pyargs dpnp.tests
env:
SKIP_TENSOR_TESTS: 1
SYCL_CACHE_PERSISTENT: 1
Expand All @@ -325,6 +328,7 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python ".github/scripts/test_retries.py" record
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
env:
SYCL_CACHE_PERSISTENT: 1
8 changes: 5 additions & 3 deletions .github/workflows/conda-build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -252,11 +252,12 @@ jobs:
conda activate ${{ env.test-env-name }}
export SKIP_TENSOR_TESTS=1

python "${{ env.dpnp-repo-path }}/.github/scripts/test_retries.py" record
if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then
export DPNP_TEST_ALL_INT_TYPES=1
python -m pytest -ra --pyargs ${{ env.module-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.module-name }}.tests
else
python -m pytest -n auto -ra --pyargs ${{ env.module-name }}.tests
python -m pytest -n auto -ra --lf --pyargs ${{ env.module-name }}.tests
fi

- name: Run tensor tests
Expand All @@ -272,4 +273,5 @@ jobs:
. "$CONDA_UNIX/etc/profile.d/conda.sh"
conda activate ${{ env.test-env-name }}

python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python "${{ env.dpnp-repo-path }}/.github/scripts/test_retries.py" record
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
87 changes: 81 additions & 6 deletions .github/workflows/conda-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ jobs:
channel-path: '${{ github.workspace }}/channel/'
pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/'
ver-json-path: '${{ github.workspace }}/version.json'
TEST_RETRIES_FILE: '${{ github.workspace }}/test-retries.jsonl'
python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }}
python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}"

Expand Down Expand Up @@ -279,11 +280,12 @@ jobs:
mamba activate ${{ env.test-env-name }}
export SKIP_TENSOR_TESTS=1

python "${{ env.dpnp-repo-path }}.github/scripts/test_retries.py" record
if [[ "${{ env.python-label }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then
export DPNP_TEST_ALL_INT_TYPES=1
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
else
python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests
fi

- name: Run tensor tests
Expand All @@ -304,7 +306,17 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python "${{ env.dpnp-repo-path }}.github/scripts/test_retries.py" record
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor

- name: Upload the list of retried tests
if: ${{ !cancelled() }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: test-retries ${{ runner.os }} Python ${{ env.python-label }}
path: ${{ env.TEST_RETRIES_FILE }}
if-no-files-found: ignore
retention-days: 1

test_examples_linux:
name: Test examples
Expand Down Expand Up @@ -440,6 +452,7 @@ jobs:
channel-path: '${{ github.workspace }}\channel\'
pkg-path-in-channel: '${{ github.workspace }}\channel\win-64\'
ver-json-path: '${{ github.workspace }}\version.json'
TEST_RETRIES_FILE: '${{ github.workspace }}\test-retries.jsonl'
python-label: ${{ endsWith(matrix.python_spec, 't') && format('{0}t', matrix.python) || matrix.python }}
python-install-spec: "python ${{ matrix.python_spec || format('{0}.*', matrix.python) }}"

Expand Down Expand Up @@ -559,11 +572,12 @@ jobs:
shell: pwsh
command: |
$env:SKIP_TENSOR_TESTS=1
python "${{ env.dpnp-repo-path }}\.github\scripts\test_retries.py" record
if ( "${{ env.python-label }}" -eq "${{ env.python-ver-test-all-dtypes }}" ) {
$env:DPNP_TEST_ALL_INT_TYPES=1
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
} else {
python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -n auto -ra --lf --pyargs ${{ env.package-name }}.tests
}

- name: Run tensor tests
Expand All @@ -582,7 +596,68 @@ jobs:
retry_on: any
shell: pwsh
command: |
python -m pytest -n auto -ra --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor
python "${{ env.dpnp-repo-path }}\.github\scripts\test_retries.py" record
python -m pytest -n auto -ra --lf --timeout=${{ env.tensor-test-timeout }} --timeout-method=thread --pyargs dpnp.tests.tensor

- name: Upload the list of retried tests
if: ${{ !cancelled() }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: test-retries ${{ runner.os }} Python ${{ env.python-label }}
path: ${{ env.TEST_RETRIES_FILE }}
if-no-files-found: ignore
retention-days: 1

report_test_retries:
name: Report retried tests

needs: [test_linux, test_windows]

# Skipped on fork PRs, whose token can't comment; the warning annotations still name the tests
if: ${{ !cancelled() && needs.test_linux.result != 'skipped' && github.event.pull_request && !github.event.pull_request.head.repo.fork }}

permissions:
# Needed to add a comment to a pull request's issue
pull-requests: write

runs-on: ubuntu-latest
timeout-minutes: 10

steps:
- name: Checkout DPNP repo
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
sparse-checkout: .github/scripts

- name: Download the lists of retried tests
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: test-retries *
path: test-retries

- name: Compose the report
id: compose
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
python3 .github/scripts/test_retries.py report test-retries test-retries.md --prefix "test-retries "

- name: Post the retried tests to the PR
if: steps.compose.outputs.retried == 'true'
uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0
with:
message-id: test_retries
message-path: test-retries.md

- name: Update an earlier report when nothing was retried
if: steps.compose.outputs.retried != 'true'
uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0
with:
message-id: test_retries
update-only: true
message: |
No tests were retried on CI at ${{ github.event.pull_request.head.sha }}.

upload:
name: Upload
Expand Down
14 changes: 9 additions & 5 deletions .github/workflows/cron-run-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,14 @@ jobs:
run: |
echo "Latest tag is ${{ steps.find_latest_tag.outputs.tag }}"

# Only for the local setup-miniconda action; the tests run from the installed package
# Only for the local setup-miniconda action and the CI scripts; the tests run from the installed package
- name: Checkout DPNP repo
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
sparse-checkout: .github/actions
sparse-checkout: |
.github/actions
.github/scripts

- name: Setup miniconda
uses: ./.github/actions/setup-miniconda
Expand Down Expand Up @@ -129,7 +131,7 @@ jobs:
id: run_tests
continue-on-error: true
run: |
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python -m pytest -ra --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
env:
SYCL_CACHE_PERSISTENT: 1

Expand All @@ -146,7 +148,8 @@ jobs:
. $CONDA/etc/profile.d/mamba.sh
mamba activate ${{ env.test-env-name }}

python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python ".github/scripts/test_retries.py" record
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
env:
SYCL_CACHE_PERSISTENT: 1

Expand All @@ -159,6 +162,7 @@ jobs:
max_attempts: ${{ env.rerun-tests-max-attempts }}
retry_on: any
command: |
python -m pytest -ra --pyargs ${{ env.package-name }}.tests
python ".github/scripts/test_retries.py" record
python -m pytest -ra --lf --continue-on-collection-errors --pyargs ${{ env.package-name }}.tests
env:
SYCL_CACHE_PERSISTENT: 1
Loading