From 3e8208e69793ff06a8ec675e3cf082eaf9db27a1 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Tue, 10 Nov 2026 15:02:55 -0800 Subject: [PATCH 01/13] [doc-only] docs(cuda.core): fix the floor-move wording; advise against pinning cuda-bindings (#3441) * docs(cuda.core): drop the Breaking Changes section name from the floor policy The release notes no longer have a Breaking Changes section for a floor move; 1.3.0 lists it under Highlights. Say only that the release notes list every move. Co-Authored-By: Claude Opus 5.5 * docs(cuda.core): advise against pinning cuda-bindings next to cuda-core Users who pin cuda-bindings themselves can conflict with the floor of a later cuda-core release. Tell them to depend on cuda-core with the cu12 or cu13 extra and let cuda-core choose the cuda-bindings version. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 83a34225095969e31b38c619c4b7b7b1ceae16f3) --- .github/RELEASE-core.md | 3 +-- cuda_core/AGENTS.md | 6 +++--- cuda_core/docs/source/install.rst | 5 +++++ cuda_core/docs/source/support.rst | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/RELEASE-core.md b/.github/RELEASE-core.md index 45c49461416..05b058a9813 100644 --- a/.github/RELEASE-core.md +++ b/.github/RELEASE-core.md @@ -101,8 +101,7 @@ cuda-bindings floor" in `cuda_core/AGENTS.md`. Check the floors instead: `cuda-bindings==`, and no "predates ... headers" line. Standard CI does not install the CUDA 12 floor: its rows use the 12.9.x backport build. -- The release notes list every floor move since the last release under - "Breaking Changes". +- The release notes list every floor move since the last release. Update the `cuda-core` pin in `cuda_python/setup.py` to the new minor series (`cuda-core~=X.Y.0`), and plan a `cuda-python` release right after diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 20295a21dd2..a97f37edcfa 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -83,9 +83,9 @@ bump: pre-commit hook `check-cuda-core-bindings-floor` (`toolshed/check_cuda_core_bindings_floor.py`) checks the pins. It also checks that no documentation page spells a floor out by hand. -3. Add to the release notes a "Breaking Changes" entry that names the new - floors. The support-policy table in `docs/source/support.rst` reads the - floors and the release version at docs-build time. It needs no edit. +3. Add to the release notes an entry that names the new floors. The + support-policy table in `docs/source/support.rst` reads the floors and the + release version at docs-build time. It needs no edit. 4. If the pins moved past what the pixi lock files resolve, refresh the lock files. 5. Pin `cuda-bindings` to match in the conda-forge `cuda-core` feedstock. The diff --git a/cuda_core/docs/source/install.rst b/cuda_core/docs/source/install.rst index e2d74063140..7e67fadad8d 100644 --- a/cuda_core/docs/source/install.rst +++ b/cuda_core/docs/source/install.rst @@ -59,6 +59,10 @@ If you upgrade ``cuda-core`` alone, the installed ``cuda-bindings`` can be older release requires. ``import cuda.core`` then reports the required version and the ``pip`` command that installs it. +Do not pin ``cuda-bindings`` in your own requirements. Depend on ``cuda-core`` with the ``cu12`` +or ``cu13`` extra, and let ``cuda-core`` choose the ``cuda-bindings`` version. A pin of your own +can conflict with the floor of a later ``cuda-core`` release. + Installing from Conda (conda-forge) ----------------------------------- @@ -73,6 +77,7 @@ and likewise use ``cuda-version=13`` for CUDA 13. The conda-forge package depends on ``cuda-bindings`` of the same CUDA major. The ``cuda-bindings`` floor of the release also applies to it. See :ref:`cuda-core-bindings-floor`. +As with ``pip``, do not pin ``cuda-bindings`` yourself. Let ``cuda-core`` choose it. Development environment diff --git a/cuda_core/docs/source/support.rst b/cuda_core/docs/source/support.rst index 4a83d73f766..08670da95a1 100644 --- a/cuda_core/docs/source/support.rst +++ b/cuda_core/docs/source/support.rst @@ -107,7 +107,7 @@ all read them from there. A floor moves only for one of two reasons. A ``cuda-core`` release needs a newer ``cuda-bindings`` API or fix, or its wheels are built against a newer CUDA minor version. A new ``cuda-bindings`` release alone does not move a floor, and a ``cuda-core`` release alone does -not. The :doc:`release notes ` list every move under "Breaking Changes". +not. The :doc:`release notes ` list every move. Python Version Support ---------------------- From dd257d5a2e0f3cad944551ec1f8931c53f2e12cb Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 12 Nov 2026 08:56:26 -0800 Subject: [PATCH 02/13] Mark cuda-gdb source-line test thread_unsafe (#3433) Signed-off-by: Ralf Juengling (cherry picked from commit 1a22eec83310d7233c8fd2e4179cf59d50e53314) --- cuda_core/tests/test_program.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_core/tests/test_program.py b/cuda_core/tests/test_program.py index c045aba0786..6dc422c9c46 100644 --- a/cuda_core/tests/test_program.py +++ b/cuda_core/tests/test_program.py @@ -1263,6 +1263,7 @@ def test_nvrtc_debug_keeps_file_the_caller_named(init_cuda, tmp_path, debug): @pytest.mark.agent_authored(model="cursor-grok-4.6") +@pytest.mark.thread_unsafe(reason="concurrent cuda-gdb sessions on one GPU hang with CUDA_ERROR_INVALID_CONTEXT") def test_cuda_gdb_shows_nvrtc_debug_source_lines(init_cuda): import pathlib From 40782e679efe16dac5507725f322dae03165ad77 Mon Sep 17 00:00:00 2001 From: jpascucci-nv Date: Thu, 12 Nov 2026 20:49:43 -0500 Subject: [PATCH 03/13] fix(cuda.core): serialize pinned-memory kernel-access test on Windows (#3436) * fix(cuda.core): serialize pinned-memory kernel-access test on Windows closes #3426 Add @thread_unsafe_on_windows to the pinned-memory kernel-access test. The test accesses host memory while parallel workers may execute GPU work; synchronizing its own stream does not coordinate those workers. This follows the guard used by similar tests on Windows and WSL. Windows CI is needed to confirm it resolves the reported crashes. The separate OOM failures remain under investigation. Validation: Ruff and git diff --check pass. * fix(cuda.core): report CUDA synchronization errors during test teardown Check the cuCtxSynchronize result in init_cuda so asynchronous errors surface during the triggering test's teardown instead of going unnoticed until a later test. Always attempt context cleanup via finally. Related to #3426. * fix(cuda.core): synchronize the bound context during test teardown Use device.sync() because the test wrapper may pop the current context before init_cuda teardown. This avoids spurious INVALID_CONTEXT errors while preserving synchronization error reporting and context cleanup. Add regression coverage for a missing current context and cleanup after synchronization failures. Related to #3426. * fix(cuda.core): serialize memory-pool watermark reset tests Mark test_mempool_watermark_reset thread_unsafe, matching test_mempool_trim. Both exercise the pool teardown and synchronization sequence associated with the suspected driver race in #3350. * fix(cuda.core): bypass access revocation when destroying host pools Skip peer-access revocation for locally created HOST and HOST_NUMA pools to investigate pinned-pool teardown OOMs and crashes. Continue destroying the pools and preserve cleanup for device, managed, and imported pools. This is a diagnostic workaround pending Windows CI validation. Related to #3426. * fix(cuda.core): preserve the first exception during fixture cleanup Keep setup or synchronization failures primary and attach subsequent cleanup failures as exception notes. Always attempt to pop the context. Add regression coverage for combined failures, including interrupts. * fix(cuda.core): limit host-pool cleanup workaround to pinned allocations HOST and HOST_NUMA also describe managed-pool preferences. Check the allocation type so the pinned-memory workaround preserves managed-pool peer-access revocation. Signed-off-by: Ralf W. Grosse-Kunstleve * test(cuda.core): expose pinned allocation OOM and retain its traceback Signed-off-by: Ralf W. Grosse-Kunstleve * ci: diagnose PR3436 pinned pool teardown on Windows Signed-off-by: Ralf W. Grosse-Kunstleve * ci: move watermark investigation out of PR3436 Signed-off-by: Ralf W. Grosse-Kunstleve * test(cuda.core): restore expected pinned allocation OOM handling Signed-off-by: Ralf W. Grosse-Kunstleve * revert(cuda.core): defer pinned host pool cleanup workaround Signed-off-by: Ralf W. Grosse-Kunstleve --------- Signed-off-by: Ralf W. Grosse-Kunstleve Co-authored-by: Ralf W. Grosse-Kunstleve (cherry picked from commit b71507831f58d331c57eedc3e57023b4b35e9b05) --- cuda_core/tests/conftest.py | 23 +++- .../memory/test_init_cuda_collect_gate.py | 112 +++++++++++++++++- cuda_core/tests/test_memory.py | 2 + 3 files changed, 134 insertions(+), 3 deletions(-) diff --git a/cuda_core/tests/conftest.py b/cuda_core/tests/conftest.py index eeee15dff2a..b667f862d48 100644 --- a/cuda_core/tests/conftest.py +++ b/cuda_core/tests/conftest.py @@ -244,6 +244,7 @@ def init_cuda(request): # live_at_setup stays None until the setup snapshot is recorded. device.set_current() live_at_setup = None + primary_error = None try: # Set option to avoid spin-waiting on synchronization. if int(os.environ.get("CUDA_CORE_TEST_BLOCKING_SYNC", 0)) != 0: @@ -275,6 +276,9 @@ def init_cuda(request): # not counted, so no cached-default bookkeeping is needed. live_at_setup = _live_resource_counts() yield device + except BaseException as exc: + primary_error = exc + raise finally: # pytest holds fixture values in item.funcargs (a pytest-internal dict) # until all teardown has run; we deliberately drop them so already @@ -298,8 +302,23 @@ def init_cuda(request): # With pytest-run-parallel this runs after worker join. This also runs # when setup raises, so an escalated leftover warning cannot leave the # context current. - driver.cuCtxSynchronize() - _ = _device_unset_current() + # The test wrapper may already have popped the calling thread's + # context; synchronize the fixture device's bound context explicitly. + # Keep the first failure primary; cleanup still runs for interrupts. + try: + device.sync() + except BaseException as exc: + if primary_error is None: + primary_error = exc + raise + primary_error.add_note(f"CUDA fixture synchronization failed: {type(exc).__name__}: {exc}") + finally: + try: + _ = _device_unset_current() + except BaseException as exc: + if primary_error is None: + raise + primary_error.add_note(f"CUDA fixture context pop failed: {type(exc).__name__}: {exc}") # After the cleanup, so an escalated warning cannot skip it. if live_after is not None and any(now < snap for now, snap in zip(live_after, live_at_setup)): released = tuple(s - n for s, n in zip(live_at_setup, live_after)) diff --git a/cuda_core/tests/memory/test_init_cuda_collect_gate.py b/cuda_core/tests/memory/test_init_cuda_collect_gate.py index 3455ee8e21f..1f7122694e4 100644 --- a/cuda_core/tests/memory/test_init_cuda_collect_gate.py +++ b/cuda_core/tests/memory/test_init_cuda_collect_gate.py @@ -33,7 +33,8 @@ UnreleasedMemoryResourceWarning, ) -from cuda.core._utils.cuda_utils import driver, handle_return +from cuda.core import Device +from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return class _ScriptedCounts: @@ -250,3 +251,112 @@ def test_teardown_warns_on_baseline_release(request, monkeypatch): pytest.raises(StopIteration), ): next(gen) # teardown: live_after == (0, 0) -> baseline-release warning + + +@_THREAD_UNSAFE_MARKER +@pytest.mark.agent_authored(model="gpt-6") +@pytest.mark.parametrize("clear_current", [False, True]) +def test_teardown_synchronizes_fixture_context(request, monkeypatch, clear_current): + """Teardown works even after the test wrapper has popped the current context.""" + func = _init_cuda_func(request) + _patch_conftest_global(monkeypatch, func, "_mr_expected_live", None) + _patch_conftest_global(monkeypatch, func, "_live_resource_counts", _ScriptedCounts((0, 0))) + + saved = _pop_context_stack() + gen = func(_FakeRequest()) + try: + next(gen) + if clear_current: + handle_return(driver.cuCtxPopCurrent()) + assert current_context_handle() == 0 + _finish_teardown(gen) + assert current_context_handle() == 0 + finally: + gen.close() + _push_context_stack(saved) + + +@_THREAD_UNSAFE_MARKER +@pytest.mark.agent_authored(model="gpt-6") +def test_teardown_sync_failure_still_pops_context(request, monkeypatch): + """A synchronization error propagates without leaving the context current.""" + func = _init_cuda_func(request) + _patch_conftest_global(monkeypatch, func, "_mr_expected_live", None) + _patch_conftest_global(monkeypatch, func, "_live_resource_counts", _ScriptedCounts((0, 0))) + failure = CUDAError("injected synchronization failure") + + class FailingSyncDevice: + device_id = 0 + + def set_current(self): + Device(0).set_current() + + def sync(self): + raise failure + + _patch_conftest_global(monkeypatch, func, "Device", lambda _: FailingSyncDevice()) + saved = _pop_context_stack() + gen = func(_FakeRequest()) + try: + next(gen) + with pytest.raises(CUDAError) as excinfo: + next(gen) + assert excinfo.value is failure + assert current_context_handle() == 0 + finally: + gen.close() + _push_context_stack(saved) + + +@_THREAD_UNSAFE_MARKER +@pytest.mark.agent_authored(model="gpt-6") +@pytest.mark.parametrize( + "fail_setup, fail_sync, fail_pop", + [(True, True, True), (True, False, True), (True, True, False), (False, True, True), (False, False, True)], +) +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_cleanup_preserves_first_failure(request, monkeypatch, fail_setup, fail_sync, fail_pop, error_type): + """Later cleanup errors annotate the first failure, and pop is always attempted.""" + func = _init_cuda_func(request) + _patch_conftest_global(monkeypatch, func, "_mr_expected_live", None) + monkeypatch.setenv("CUDA_CORE_TEST_BLOCKING_SYNC", "0") + setup_error = error_type("injected setup failure") + sync_error = error_type("injected synchronization failure") + pop_error = error_type("injected context pop failure") + state = {"current": False} + + class FakeDevice: + def set_current(self): + state["current"] = True + + def sync(self): + if fail_sync: + raise sync_error + + def counts(): + if fail_setup: + raise setup_error + return (0, 0) + + def pop(): + state["current"] = False + if fail_pop: + raise pop_error + + _patch_conftest_global(monkeypatch, func, "Device", lambda _: FakeDevice()) + _patch_conftest_global(monkeypatch, func, "_live_resource_counts", counts) + _patch_conftest_global(monkeypatch, func, "_device_unset_current", pop) + gen = func(_FakeRequest()) + if not fail_setup: + next(gen) + with pytest.raises(error_type) as excinfo: + next(gen) + expected = setup_error if fail_setup else sync_error if fail_sync else pop_error + assert excinfo.value is expected + assert not state["current"] + notes = getattr(expected, "__notes__", []) + assert len(notes) == int(fail_setup and fail_sync) + int(fail_pop and (fail_setup or fail_sync)) + if fail_setup and fail_sync: + assert any(str(sync_error) in note for note in notes) + if fail_pop and (fail_setup or fail_sync): + assert any(str(pop_error) in note for note in notes) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index fbafb1b2207..5d4aab75e42 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -1246,6 +1246,7 @@ def test_pinned_memory_resource_initialization(init_cuda): buffer.close() +@thread_unsafe_on_windows @pytest.mark.human_reviewed @pytest.mark.parametrize("pool", ["current", "new", "new_numa"]) def test_pinned_memory_resource_kernel_access(init_cuda, pool): @@ -2972,6 +2973,7 @@ def test_mempool_release_threshold_accepts_index_types(memory_resource_factory, mr.close() +@pytest.mark.thread_unsafe(reason="CUDA legacy-default-stream mempool teardown/sync race (gh-3350)") @pytest.mark.agent_authored(model="claude-opus-5-5") @pytest.mark.parametrize("kind", ["reserved", "used"]) def test_mempool_watermark_reset(memory_resource_factory, kind): From 071268a20695ef285fbe39d2fb96149cb681f984 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 16 Nov 2026 10:31:41 -0800 Subject: [PATCH 04/13] [no-ci] ci: exclude unavailable historical GitHub references (#3471) * ci: exclude unavailable historical GitHub references Signed-off-by: Ralf W. Grosse-Kunstleve * ci: require manual copy-pr-bot approval Signed-off-by: Ralf W. Grosse-Kunstleve --------- Signed-off-by: Ralf W. Grosse-Kunstleve (cherry picked from commit 3991a6973dde813ef761983d9752713b592b1c83) --- lychee.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lychee.toml b/lychee.toml index 4ad1995e377..dc179508069 100644 --- a/lychee.toml +++ b/lychee.toml @@ -12,4 +12,8 @@ exclude = [ '^file://.*/cuda-bindings/latest/module/(driver|runtime)\.html#id[0-9]+$', # Preferred Networks rejects hosted-runner GETs, but the URL is browser reachable. '^https://www\.preferred\.jp/en/?$', + # Previously valid GitHub references became unavailable in November 2026. + '^https://github\.com/NVIDIA/cuda-python/pull/2236$', + '^https://github\.com/NVIDIA/cuda-python/issues/2344$', + '^https://github\.com/NVIDIA/cuda-python/issues/2345$', ] From 624896f4d56841fb809bd3a74813317842cc8938 Mon Sep 17 00:00:00 2001 From: B Sainath Reddy Date: Tue, 17 Nov 2026 04:57:03 +0530 Subject: [PATCH 05/13] cuda.core: map ml_dtypes bfloat16 to kDLBfloat in DLPack export (#3359) _smv_dtype_numpy_to_dlpack dispatched on numpy dtype kind, but ml_dtypes.bfloat16 has kind "V", so bf16 views fell through to the unsupported-dtype BufferError and could not be re-exported, even though the DLPack import path already accepts kDLBfloat and _tensor_map.pyx maps bfloat16. Detect bfloat16 up front and export it as kDLBfloat (16 bits). The new round-trip test goes through StridedMemoryView.from_dlpack so it cannot skip on stacks where NumPy's own DLPack support for extended dtypes is unavailable, unlike the existing parametrized round-trip test's NumPy-consumer probe. Signed-off-by: Sainath-Reddy7 <180950590+Sainath-Reddy7@users.noreply.github.com> Co-authored-by: Sainath-Reddy7 <180950590+Sainath-Reddy7@users.noreply.github.com> Co-authored-by: Sebastian Berg (cherry picked from commit d1d14572145c14a74e69594e6bc17bdd8cc08d23) --- cuda_core/cuda/core/_memoryview.pyx | 4 ++++ cuda_core/tests/test_utils_dlpack.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/cuda_core/cuda/core/_memoryview.pyx b/cuda_core/cuda/core/_memoryview.pyx index fd73cbb77ef..8e0d7ee13f3 100644 --- a/cuda_core/cuda/core/_memoryview.pyx +++ b/cuda_core/cuda/core/_memoryview.pyx @@ -695,6 +695,10 @@ cdef inline int _smv_dtype_numpy_to_dlpack(object dtype_obj, DLDataType* out_dty if bits not in (64, 128): raise BufferError(f"Unsupported complex dtype: {np_dtype}") code = kDLComplex + elif np_dtype.type is bfloat16: + # ml_dtypes bfloat16 has kind "V", so the kind chain cannot reach it; + # the byte-order check above has already run for it. + code = kDLBfloat else: raise BufferError(f"Unsupported dtype for DLPack export: {np_dtype}") diff --git a/cuda_core/tests/test_utils_dlpack.py b/cuda_core/tests/test_utils_dlpack.py index 9489c977d3f..5efb6b42df4 100644 --- a/cuda_core/tests/test_utils_dlpack.py +++ b/cuda_core/tests/test_utils_dlpack.py @@ -73,6 +73,30 @@ def test_dlpack_export_roundtrip_dtypes(dtype): _assert_dlpack_export_roundtrip(np.zeros((2, 3), dtype=dtype)) +@pytest.mark.agent_authored(model="glm-5.3") +def test_dlpack_export_bfloat16_roundtrip(init_cuda): + """bfloat16 views export as kDLBfloat and re-import as bfloat16 (#3337). + + This test becomes unnecessary when `_assert_dlpack_export_roundtrip` + starts testing bfloat16, which should be `ml_dtypes>0.6.0`. + """ + if ml_dtypes is None: + pytest.skip("ml_dtypes is not installed") + src = np.zeros((2, 3), dtype=np.uint16) + view = StridedMemoryView.from_any_interface(src, stream_ptr=-1).view(dtype=np.dtype(ml_dtypes.bfloat16)) + imported = StridedMemoryView.from_dlpack(view, stream_ptr=-1) + assert imported.dtype == np.dtype(ml_dtypes.bfloat16) + assert imported.shape == (2, 3) + assert imported.ptr == view.ptr + # Pin the exported type code and bit width directly: the importer maps + # kDLBfloat to bfloat16 without checking bits (kDLBfloat = 4 in dlpack.h). + capsule = view.__dlpack__() + dlm = ctypes.cast(_PyCapsule_GetPointer(capsule, b"dltensor"), ctypes.POINTER(_DLManagedTensor)) + exported_dtype = dlm.contents.dl_tensor.dtype + assert exported_dtype.code == 4 + assert exported_dtype.bits == 16 + + @pytest.mark.parametrize( "shape", [pytest.param((), id="scalar"), pytest.param((0, 3), id="empty")], From 909a4b493ddfcfadbc02bf4ea4b44d5c18f6d5b8 Mon Sep 17 00:00:00 2001 From: Rob Parolin Date: Mon, 16 Nov 2026 14:01:56 -0800 Subject: [PATCH 06/13] fix(cuda.core): make IPC imports wait for a pending release of the same descriptor (#3450) * fix(cuda.core): make IPC imports wait for a pending release of the same descriptor deviceptr_import_ipc() shares one handle per descriptor because the driver's first free of an imported pointer unmaps every import of it (nvbug 5570902). The handle's deleter took ipc_import_mutex only after the last reference was gone. In that gap an import found the cache entry expired, re-imported the still-mapped pointer and registered a new entry. The deleter then erased that new entry and freed the pointer, which unmapped it under the new Buffer. The cache entry now outlives the last reference and only its own deleter erases it, under the mutex, together with the free. An import that finds an expired entry waits on a condition variable until that release is done, then imports fresh. Co-Authored-By: Claude Opus 5.5 * Use CTAD for the ipc_import_mutex locks Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 2f1185ea1625b44704a8ed4a4470118b54959af8) --- cuda_core/cuda/core/_cpp/rt/api.hpp | 4 +- cuda_core/cuda/core/_cpp/rt/memory.cpp | 43 +++++-- .../test_ipc_import_release_race.py | 113 ++++++++++++++++++ 3 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 cuda_core/tests/memory_ipc/test_ipc_import_release_race.py diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index 4abf85bd5dc..40642ab2f26 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -229,7 +229,9 @@ DevicePtrHandle deviceptr_create_mapped_graphics( // Import a device pointer from IPC via cuMemPoolImportPointer. // When the last reference is released, cuMemFreeAsync is called on the stored stream. -// Note: Does not yet implement reference counting for nvbug 5570902. +// Repeat imports of one descriptor share one handle, because the driver's first +// free of an imported pointer unmaps every import of it (nvbug 5570902). An +// import that races the release of the last handle waits until the free is done. // On error, returns empty handle and sets thread-local error (use get_last_error()). DevicePtrHandle deviceptr_import_ipc( const MemoryPoolHandle &h_pool, const void *export_data, const StreamHandle &h_stream diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index fb95140734a..3cf92649dc4 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -9,11 +9,13 @@ #include "error.hpp" #include "internal.hpp" #include +#include #include #include #include #include #include +#include #include #ifndef _WIN32 #include @@ -519,10 +521,21 @@ struct ExportDataKeyHash { } }; +// The handle imported for one descriptor, and the box it aliases. +struct IpcCacheEntry { + std::weak_ptr handle; + const DevicePtrBox *box; +}; + } // namespace -static HandleRegistry ipc_ptr_cache; +// Guarded by ipc_import_mutex. An entry outlives its handle's last reference: +// only that handle's deleter erases it, under the mutex, together with the +// free. An importer that finds an expired entry therefore knows a release of +// the same mapping is in flight and waits on ipc_release_cv until it is done. +static std::unordered_map ipc_ptr_cache; static std::mutex ipc_import_mutex; +static std::condition_variable ipc_release_cv; DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, const StreamHandle &h_stream) @@ -552,10 +565,17 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co CUresult discard_status = CUDA_SUCCESS; CUdeviceptr discarded = 0; { - std::scoped_lock lock(ipc_import_mutex); + std::unique_lock lock(ipc_import_mutex); - if (auto h = ipc_ptr_cache.lookup(key)) { - return h; + for (auto it = ipc_ptr_cache.find(key); it != ipc_ptr_cache.end(); it = ipc_ptr_cache.find(key)) { + if (DevicePtrHandle h = it->second.handle.lock()) { + return h; + } + // Expired but not erased: the last reference is gone and its + // deleter has not freed the mapping yet. A re-import now would + // share that mapping, and the deleter's cuMemFreeAsync would + // unmap it under us (nvbug 5570902). Wait for the release. + ipc_release_cv.wait(lock); } CUdeviceptr ptr; @@ -579,20 +599,29 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co GILReleaseGuard gil; const DeallocationStream& stream = b->deallocation; ContextHandle h_dealloc = deallocation_context(stream); + // Explicit template argument: MSVC cannot deduce it here (C2955). std::unique_lock lock(ipc_import_mutex); - ipc_ptr_cache.unregister_handle(key); + // Erase only this box's entry (importers wait while it + // exists, so no other box can own the key yet). + auto entry = ipc_ptr_cache.find(key); + if (entry != ipc_ptr_cache.end() && entry->second.box == b) { + ipc_ptr_cache.erase(entry); + } cleanup_in_context( h_dealloc, "cuMemFreeAsync", handle_bits(b->resource), [&]() noexcept { return p_cuMemFreeAsync(b->resource, as_cu(stream.h_stream)); // raw: under ipc_import_mutex }, - [&]() noexcept { lock.release()->unlock(); }); + [&]() noexcept { + ipc_release_cv.notify_all(); + lock.release()->unlock(); + }); delete b; } ); // clang-format on DevicePtrHandle h(box, &box->resource); - ipc_ptr_cache.register_handle(key, h); + ipc_ptr_cache[key] = IpcCacheEntry{h, box.get()}; return h; } diff --git a/cuda_core/tests/memory_ipc/test_ipc_import_release_race.py b/cuda_core/tests/memory_ipc/test_ipc_import_release_race.py new file mode 100644 index 00000000000..7e6a19b977a --- /dev/null +++ b/cuda_core/tests/memory_ipc/test_ipc_import_release_race.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stress test: importing a descriptor while its last import is being released. + +``deviceptr_import_ipc()`` shares one handle per descriptor, because the +driver's first free of an imported pointer unmaps every import of it +(nvbug 5570902). Releasing the last handle frees the mapping after its +reference count reaches zero, so an import can arrive while that free is still +pending. The import must wait for the free, not re-import the still-mapped +pointer and lose it to the pending free. + +Threads in one child import, use and close the same descriptor in a loop, so +the last handle is released again and again while other threads import. A +lost mapping shows up as a failed copy; a missed wakeup shows up as a hang, +which the parent's join timeout catches. +""" + +import contextlib +import multiprocessing as mp +import threading +import time + +import pytest +from helpers.child_processes import child_timeout_sec, kill_subprocesses + +from cuda.core import Buffer, Device + +CHILD_TIMEOUT_SEC = child_timeout_sec() +NBYTES = 4096 +THREADS = 4 +DURATION_SEC = 1.0 + +# these tests spawn new processes and files which fails for very many threads; +# child processes and IPC fixtures are shared across workers (#2784); cap at two threads +pytestmark = pytest.mark.parallel_threads_limit(2) + + +def child_main(queue): + device = Device() + device.set_current() + mr = queue.get() + descriptor = queue.get() + # A VirtualMemoryResource rounds the allocation up to its granularity. + nbytes = queue.get() + + barrier = threading.Barrier(THREADS) + errors = [] + iterations = [0] * THREADS + + def worker(index): + Device().set_current() + stream = Device().create_stream() + local = Device().memory_resource.allocate(nbytes, stream=stream) + barrier.wait() + deadline = time.monotonic() + DURATION_SEC + try: + while time.monotonic() < deadline and not errors: + buffer = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) + local.copy_from(buffer, stream=stream) + stream.sync() + buffer.close() + iterations[index] += 1 + except Exception as e: + errors.append(e) + finally: + local.close(stream) + stream.sync() + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + if errors: + raise errors[0] + assert all(iterations), iterations + device.sync() + + +class TestIpcImportReleaseRace: + """Imports that race the release of the last handle must stay usable and must not hang.""" + + @pytest.fixture(autouse=True) + def _set_start_method(self): + # Ensure spawn is used for multiprocessing + with contextlib.suppress(RuntimeError): + mp.set_start_method("spawn", force=True) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_main(self, ipc_device, ipc_memory_resource): + ipc_device.set_current() + mr = ipc_memory_resource + + stream = ipc_device.default_stream + buffer = mr.allocate(NBYTES, stream=stream) + stream.sync() + + try: + queue = mp.Queue() + process = mp.Process(target=child_main, args=(queue,)) + process.start() + queue.put(mr) + queue.put(buffer.ipc_descriptor) + queue.put(buffer.size) + + process.join(timeout=CHILD_TIMEOUT_SEC) + survivors = kill_subprocesses(process) + assert not survivors, "an import racing the last release did not return" + assert process.exitcode == 0, f"child process failed with exit code {process.exitcode}" + finally: + buffer.close(stream) From 8e7cb056ad271abf07dd07c79501faa7b466f86f Mon Sep 17 00:00:00 2001 From: Rui Luo Date: Tue, 17 Nov 2026 08:16:38 +0800 Subject: [PATCH 07/13] test(cuda.core): skip HOST graph allocation before CUDA 13 (#3446) (cherry picked from commit 11cc66621809b3d4eef8bd4702ec082ac15d1576) --- cuda_core/tests/graph/test_graph_definition.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index cd4f5038bd7..0d4f2c8f843 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -927,6 +927,11 @@ def test_alloc_memory_type_host(init_cuda): """HOST graph alloc nodes reconstruct memory_type from the driver, not the Python argument.""" skip_if_mempool_unsupported() from cuda.core._utils.cuda_utils import CUDAError + from cuda.core._utils.version import driver_version + + # CUDA 12 rejects HOST pool properties before graph support is checked. + if driver_version() < (13, 0, 0): + pytest.skip("CUDA drivers before 13.0 reject HOST allocation properties") g = GraphDefinition() try: From c18d573517f40551e2502ba1b19afd9a72460947 Mon Sep 17 00:00:00 2001 From: Rui Luo Date: Tue, 17 Nov 2026 08:17:45 +0800 Subject: [PATCH 08/13] test(cuda.core): query green stream contexts with v2 (#3448) Signed-off-by: Rui Luo (cherry picked from commit 7076a67d91ee4eb6addbce87ba9fcf153d5161a5) --- cuda_core/tests/helpers/contexts.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/helpers/contexts.py b/cuda_core/tests/helpers/contexts.py index f613f0ac336..d1e73de926a 100644 --- a/cuda_core/tests/helpers/contexts.py +++ b/cuda_core/tests/helpers/contexts.py @@ -71,9 +71,17 @@ def assert_device_operations_use_bound_context(device): # Live check: query the driver directly, rather than comparing cached # metadata, to confirm the stream was actually created in the bound # context rather than whatever was ambient. - driver_stream_ctx = handle_return(driver.cuStreamGetCtx(stream.handle)) + if bound_context.is_green: + # The legacy query rejects green streams on CUDA 12.9. In v2, + # pCtx is the primary context; pGreenCtx identifies the green context. + _, driver_green_ctx = handle_return(driver.cuStreamGetCtx_v2(stream.handle)) + assert int(driver_green_ctx) != 0, "The stream must be associated with a green context." + driver_stream_ctx = handle_return(driver.cuCtxFromGreenCtx(driver_green_ctx)) + else: + # Keep the legacy query for regular contexts on older drivers. + driver_stream_ctx = handle_return(driver.cuStreamGetCtx(stream.handle)) assert int(driver_stream_ctx) == int(bound_context.handle), ( - "cuStreamGetCtx reports a context other than the one the stream was created in." + "The driver reports a context other than the one the stream was created in." ) event = device.create_event() From 70d6a9f2fe279951c42d6dd7d389ae9c94c53fc6 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Mon, 16 Nov 2026 18:30:51 -0800 Subject: [PATCH 09/13] test(cuda.core): skip the VMM RDMA policy test on WSL (#3478) GPUDirect RDMA is not supported on WSL, but some drivers report the capability there, and the RDMA-enabled VMM allocation in test_vmm_allocator_policy_configuration then fails. Skip the test on WSL; the capability checks stay as they are. Co-authored-by: Claude Opus 5.5 (cherry picked from commit 36bc530b90f363f86462808faf80d4e36d3f9356) --- cuda_core/tests/test_memory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 5d4aab75e42..ac1099244ab 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -81,7 +81,7 @@ VirtualMemoryLocationType, ) from cuda.core.utils import StridedMemoryView -from cuda_python_test_helpers import IS_WINDOWS +from cuda_python_test_helpers import IS_WINDOWS, IS_WSL CHILD_TIMEOUT_SEC = child_timeout_sec() @@ -1464,6 +1464,9 @@ def test_vmm_allocator_basic_allocation(init_cuda, use_device_object, handle_typ @pytest.mark.agent_authored(model="claude-fable-5-1") +@pytest.mark.skipif( + IS_WSL, reason="GPUDirect RDMA is not supported on WSL, but some drivers report the capability there" +) def test_vmm_allocator_policy_configuration(init_cuda): """Test VMM allocator with different policy configurations. From a4a52d019df5c531fddb7de8852aeda7ab29ab59 Mon Sep 17 00:00:00 2001 From: Rob Parolin Date: Mon, 16 Nov 2026 19:32:24 -0800 Subject: [PATCH 10/13] docs(cuda.core): add 1.3.0 release note for the IPC import/release race fix (#3479) * docs(cuda.core): add 1.3.0 release note for the IPC import/release race fix Follow-up to #3450, which merged without a release note. Co-Authored-By: Claude Opus 5.5 * docs(cuda.core): scope the IPC race release note to memory pool buffers Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit de4c60d049396dd3a520bb269c15ca887b8cd88a) --- cuda_core/docs/source/release/1.3.1-notes.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cuda_core/docs/source/release/1.3.1-notes.rst diff --git a/cuda_core/docs/source/release/1.3.1-notes.rst b/cuda_core/docs/source/release/1.3.1-notes.rst new file mode 100644 index 00000000000..8d573bd3ce0 --- /dev/null +++ b/cuda_core/docs/source/release/1.3.1-notes.rst @@ -0,0 +1,15 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +.. currentmodule:: cuda.core + +``cuda.core`` 1.3.1 Release Notes +================================= + +Fixes and enhancements +---------------------- + +- :meth:`Buffer.from_ipc_descriptor` no longer unmaps a newly imported + memory pool buffer when another thread releases the same descriptor at the + same time. The import now waits for the pending release to finish. + (`#3450 `__) From 7393471790eb40b545dd485f76ed9135e08f1977 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 16 Nov 2026 21:50:27 -0800 Subject: [PATCH 11/13] Fix conditional graph lifetime tests to use supported nodes (#3469) * Fix conditional graph lifetime tests to use supported nodes Signed-off-by: Ralf W. Grosse-Kunstleve * ci: require manual copy-pr-bot approval Signed-off-by: Ralf W. Grosse-Kunstleve --------- Signed-off-by: Ralf W. Grosse-Kunstleve Co-authored-by: Andy Jost (cherry picked from commit b80866b3556717c6e702b9a9d11a7fc4ba62097d) --- .../graph/test_graph_definition_lifetime.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index 5a3a99b65ea..d0cc71797d9 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -213,29 +213,31 @@ def test_reconstructed_body_survives_parent_deletion(init_cuda): @pytest.mark.agent_authored(model="gpt-5.6") def test_destroying_conditional_node_releases_branch_attachments(init_cuda, builder, expected_count): """Destroying a conditional owner releases every branch attachment.""" - graph_def = GraphDefinition() - condition = try_create_condition(graph_def) - branches = builder(graph_def, condition) - callback_refs = [] - - for branch in branches: - - def callback(): - pass - - callback_refs.append(weakref.ref(callback)) - branch.callback(callback) - - del callback, branches - gc.collect() - assert len(callback_refs) == expected_count - assert all(ref() is not None for ref in callback_refs) + dev = Device() + with dev.allocate(4, stream=dev.default_stream) as buf: + dev.default_stream.sync() + dptr = int(buf.handle) + graph_def = GraphDefinition() + condition = try_create_condition(graph_def) + branches = builder(graph_def, condition) + owner_refs = [] + + # Conditional bodies do not permit host callbacks; memset can retain an owner. + for branch in branches: + attachment_owner = _Sentinel() + owner_refs.append(weakref.ref(attachment_owner)) + branch.memset(dptr, 0, 4, dst_owner=attachment_owner) + + del attachment_owner, branches + gc.collect() + assert len(owner_refs) == expected_count + assert all(ref() is not None for ref in owner_refs) - owner = next(node for node in graph_def.nodes() if isinstance(node, ConditionalNode)) - owner.destroy() - del owner + owner = next(node for node in graph_def.nodes() if isinstance(node, ConditionalNode)) + owner.destroy() + del owner - _wait_until(lambda: all(ref() is None for ref in callback_refs)) + _wait_until(lambda: all(ref() is None for ref in owner_refs)) @pytest.mark.parametrize("builder, expected_count", _COND_BUILDERS) @@ -245,7 +247,7 @@ def test_destroying_conditional_node_invalidates_branch_handles(init_cuda, build graph_def = GraphDefinition() condition = try_create_condition(graph_def) branches = builder(graph_def, condition) - branch_nodes = [branch.callback(lambda: None) for branch in branches] + branch_nodes = [branch.empty() for branch in branches] owner = next(node for node in graph_def.nodes() if isinstance(node, ConditionalNode)) owner.destroy() From 6e607fe4ca40f673748610e790d5007755b09045 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 17 Nov 2026 10:28:20 -0800 Subject: [PATCH 12/13] fix(cuda.core): run cuMemPoolSetAccess in relaxed mode in PinnedMemoryResource (#3475) Since #3405, PinnedMemoryResource.__init__ without options calls cuMemPoolSetAccess to grant the current device read/write access to the driver's default pool. The driver refuses that call, and invalidates the capture, while a global-mode capture is active on any thread. Users of PyTorch's torch.cuda.graph (which defaults to global mode) therefore see CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED on construction and CUDA_ERROR_STREAM_CAPTURE_INVALIDATED on end_building. Fix: introduce MP_set_access in _memory_pool.pyx, wrapping cuMemPoolSetAccess in the existing _enter_relaxed_capture_mode / _exit_relaxed_capture_mode helpers, following the identical pattern used by pool attribute reads and writes and trim() since #2834. Call it from _PMR_init instead of the bare with-nogil block. The with-options path (cuMemPoolCreate) is unaffected; that call is still refused under global capture as documented in the 1.3.0 notes. Add tests for the three capture combinations (global+same_thread, global+other_thread, thread_local+same_thread) and a capture-mode restore check modelled on test_dmr_init_restores_capture_mode. Closes #3451 Co-authored-by: Andy Jost (cherry picked from commit d94f19c8bf915a36934f6614d6803311785588db) --- cuda_core/cuda/core/_memory/_memory_pool.pxd | 3 + cuda_core/cuda/core/_memory/_memory_pool.pyx | 18 +++++ .../core/_memory/_pinned_memory_resource.pyi | 3 +- .../core/_memory/_pinned_memory_resource.pyx | 8 +- cuda_core/docs/source/release/1.3.1-notes.rst | 11 +++ cuda_core/tests/test_memory.py | 78 +++++++++++++++++++ 6 files changed, 116 insertions(+), 5 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pxd b/cuda_core/cuda/core/_memory/_memory_pool.pxd index b85e5182e3b..f2117b4034d 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pxd +++ b/cuda_core/cuda/core/_memory/_memory_pool.pxd @@ -36,6 +36,9 @@ cdef int MP_init_current_pool( cdef int MP_raise_release_threshold(_MemPool self) except? -1 +cdef int MP_set_access( + _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count) except? -1 + cdef inline int MP_check_open(_MemPool self) except -1: if not self._h_pool: diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pyx b/cuda_core/cuda/core/_memory/_memory_pool.pyx index 7d071f62238..679e25b4f3c 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pyx +++ b/cuda_core/cuda/core/_memory/_memory_pool.pyx @@ -461,6 +461,24 @@ cdef int MP_raise_release_threshold(_MemPool self) except? -1: return 0 +cdef int MP_set_access( + _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count) except? -1: + """Set device access to the pool. + + Runs in relaxed capture mode, like every pool attribute access (see + ``_enter_relaxed_capture_mode``): the driver refuses cuMemPoolSetAccess, + and invalidates the capture, while a global-mode capture is active on + any thread or a thread-local one on the calling thread. + """ + cdef cydriver.CUstreamCaptureMode mode + cdef cydriver.CUresult err + with nogil: + _enter_relaxed_capture_mode(&mode) + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) + _exit_relaxed_capture_mode(err, &mode) + return 0 + + # Raise an exception if the given stream is capturing. # A result of CU_STREAM_CAPTURE_STATUS_INVALIDATED is considered an error. cdef inline int check_not_capturing(cydriver.CUstream s) except?-1 nogil: diff --git a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyi b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyi index af909bb095a..29f72ea10c1 100644 --- a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyi +++ b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyi @@ -77,7 +77,8 @@ class PinnedMemoryResource(_MemPool): default, so the resource grants the device that is current when it is created read and write access to the pool. Kernels on other devices cannot access its buffers. Without options, the grant applies to the driver's - current pool, which the whole process shares. + current pool, which the whole process shares. Construction without options + does not affect a CUDA graph capture in progress on any thread. To create an IPC-Enabled memory resource (MR) that is capable of sharing allocations between processes, specify ``ipc_enabled=True`` in the initializer diff --git a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx index cbfaa4f7638..305d69c2809 100644 --- a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx @@ -12,11 +12,11 @@ from cuda.core._memory._memory_pool cimport ( MP_check_open, MP_init_create_pool, MP_init_current_pool, + MP_set_access, ) from cuda.core._memory cimport _ipc from cuda.core._memory._ipc cimport IPCAllocationHandle from cuda.core._memory._location cimport cumemlocation_from_id -from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream, Stream_accept from cuda.core._utils.cuda_utils cimport ( check_or_create_options, @@ -105,7 +105,8 @@ cdef class PinnedMemoryResource(_MemPool): default, so the resource grants the device that is current when it is created read and write access to the pool. Kernels on other devices cannot access its buffers. Without options, the grant applies to the driver's - current pool, which the whole process shares. + current pool, which the whole process shares. Construction without options + does not affect a CUDA graph capture in progress on any thread. To create an IPC-Enabled memory resource (MR) that is capable of sharing allocations between processes, specify ``ipc_enabled=True`` in the initializer @@ -328,8 +329,7 @@ cdef inline _PMR_init(PinnedMemoryResource self, options): access.location = cumemlocation_from_id( cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE, dev.device_id) access.flags = cydriver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - with nogil: - HANDLE_RETURN(cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), &access, 1)) + MP_set_access(self, &access, 1) def _deep_reduce_pinned_memory_resource(mr: object) -> tuple[object, ...]: diff --git a/cuda_core/docs/source/release/1.3.1-notes.rst b/cuda_core/docs/source/release/1.3.1-notes.rst index 8d573bd3ce0..938de84e732 100644 --- a/cuda_core/docs/source/release/1.3.1-notes.rst +++ b/cuda_core/docs/source/release/1.3.1-notes.rst @@ -13,3 +13,14 @@ Fixes and enhancements memory pool buffer when another thread releases the same descriptor at the same time. The import now waits for the pending release to finish. (`#3450 `__) + +- :class:`PinnedMemoryResource` construction without options no longer + invalidates a CUDA graph capture in progress on another thread. ``cuda.core`` + 1.3.0 introduced a ``cuMemPoolSetAccess`` call in the constructor (#3405); + the driver refuses that call, and invalidates any concurrent global-mode + capture, unless it runs in relaxed capture mode. The fix applies the same + relaxed-mode wrapper already used for pool attribute reads and writes and + :meth:`~_memory._memory_pool._MemPool.trim`. Captures started with the + default ``begin_building()`` mode are unaffected; global mode is the default + in several external libraries (for example ``torch.cuda.graph``). + (`#3451 `__) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index ac1099244ab..93a2a5f4211 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -3068,6 +3068,84 @@ def use_pool(): mr.close() +@pytest.mark.agent_authored(model="claude-sonnet-4-6") +@pytest.mark.thread_unsafe(reason="a global-mode capture affects every thread") +@pytest.mark.parametrize( + ("mode", "caller"), + # A thread-local capture does not restrict other threads, so pairing it + # with other_thread would pass without relaxed mode. + [("global", "same_thread"), ("global", "other_thread"), ("thread_local", "same_thread")], +) +def test_pmr_construction_during_capture_keeps_capture_valid(init_cuda, mode, caller): + """PinnedMemoryResource() without options does not invalidate a capture on another thread. + + Since #3405 construction calls cuMemPoolSetAccess, which the driver refuses + while a global-mode capture is active on any thread or a thread-local one on + the calling thread. The fix runs that call in relaxed capture mode (#3451). + """ + device = Device() + skip_if_pinned_memory_unsupported(device) + device.set_current() + + mr = None + + def construct(): + device.set_current() + nonlocal mr + mr = PinnedMemoryResource() + + gb = device.create_graph_builder().begin_building(mode=mode) + try: + if caller == "same_thread": + construct() + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + pool.submit(construct).result() + finally: + # Raises if construction invalidated the capture. + gb.end_building() + gb.close() + if mr is not None: + mr.close() + + +@pytest.mark.agent_authored(model="claude-sonnet-4-6") +@pytest.mark.thread_unsafe(reason="a global-mode capture affects every thread") +@pytest.mark.parametrize("mode", ["global", "thread_local"]) +def test_pmr_construction_restores_capture_mode(init_cuda, mode): + """After construction the calling thread is back in its previous capture mode. + + PinnedMemoryResource() switches the thread to relaxed mode around its + cuMemPoolSetAccess call (#3451) and must not leave it there. + """ + + def thread_capture_mode(): + # Exchange the mode with relaxed twice to read without changing it. + relaxed = driver.CUstreamCaptureMode.CU_STREAM_CAPTURE_MODE_RELAXED + err, current = driver.cuThreadExchangeStreamCaptureMode(relaxed) + assert err == driver.CUresult.CUDA_SUCCESS + err, _ = driver.cuThreadExchangeStreamCaptureMode(current) + assert err == driver.CUresult.CUDA_SUCCESS + return current + + device = Device() + skip_if_pinned_memory_unsupported(device) + device.set_current() + + gb = device.create_graph_builder().begin_building(mode=mode) + mr = None + try: + before = thread_capture_mode() + mr = PinnedMemoryResource() + assert thread_capture_mode() == before + finally: + # Raises if construction invalidated the capture. + gb.end_building() + gb.close() + if mr is not None: + mr.close() + + def test_mempool_attributes_repr(memory_resource_factory): """Test the repr of memory pool attributes for all memory resource types.""" MR, MRops = memory_resource_factory From ebc0680bbd9ca7d944999e0aa56eb48e2f5562f9 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Tue, 17 Nov 2026 16:30:32 -0500 Subject: [PATCH 13/13] cuda.core: carry the cuda-bindings floors through minimal-sysctk12/13 extras (#3460) * cuda.core: carry the cuda-bindings floors through minimal-sysctk extras The floors lived only in the cu12/cu13 extras, so a package that depends on plain cuda-core never passed them to pip or uv, and an older cuda-bindings failed only at import (#3459). Declare the floors in new minimal-sysctk12/13 extras, include them from cu12/cu13, and depend on cuda-bindings without a version. The floor parser, the CI wheel-floor reader, the pre-commit hook and the docs read the new extras. * cuda.core: read cu floors through the minimal-sysctk self-reference Drop the unversioned cuda-bindings from the base dependencies, so a plain cuda-core install keeps asking for an extra. Keep the floor readers on the cu extras, as before, and teach them to follow a cuda-core[...] self-reference and to skip an unversioned cuda-bindings[all]. The cu layout then yields the floor written in minimal-sysctk, and the existing tests of the cu form stay as they are. New tests cover the self-reference. --------- Co-authored-by: Andy Jost (cherry picked from commit ad38141935c21d99f2bf6104f51a438f617a20e6) --- ci/tools/cuda_core_bindings_floor.py | 43 ++++++++---- .../tests/test_cuda_core_bindings_floor.py | 35 ++++++++++ cuda_core/AGENTS.md | 14 ++-- cuda_core/DESCRIPTION.rst | 2 + cuda_core/cuda/core/_bindings_floor.py | 42 ++++++++++-- cuda_core/docs/source/install.rst | 20 ++++-- cuda_core/docs/source/release/1.3.1-notes.rst | 15 +++++ cuda_core/docs/source/support.rst | 7 +- cuda_core/pyproject.toml | 31 +++++++-- cuda_core/tests/test_bindings_floor.py | 65 +++++++++++++++++++ toolshed/check_cuda_core_bindings_floor.py | 41 ++++++++---- 11 files changed, 266 insertions(+), 49 deletions(-) diff --git a/ci/tools/cuda_core_bindings_floor.py b/ci/tools/cuda_core_bindings_floor.py index 5f4dadc8cf0..a458a16d32a 100644 --- a/ci/tools/cuda_core_bindings_floor.py +++ b/ci/tools/cuda_core_bindings_floor.py @@ -14,10 +14,12 @@ from the checkout, so a nightly job that tests a wheel built from another commit reads that wheel's floor. -The floors are declared once, by the `cu12`/`cu13` extras in -cuda_core/pyproject.toml, and the wheel's METADATA carries them as -`Requires-Dist: cuda-bindings[all]<14,>=13.4.1; extra == "cu13"`. This script -reads that line. +The floors are declared once, by the `minimal-sysctk12`/`minimal-sysctk13` +extras in cuda_core/pyproject.toml, which the `cu12`/`cu13` extras include. The +wheel's METADATA carries them as +`Requires-Dist: cuda-bindings<14,>=13.4.1; extra == "minimal-sysctk13"` and +`Requires-Dist: cuda-core[minimal-sysctk13]; extra == "cu13"`. This script +reads the floor of the `cu` extra, following that self-reference. """ from __future__ import annotations @@ -28,21 +30,38 @@ import zipfile from pathlib import Path -# The cuda-bindings requirement of one `cu` extra in METADATA. The name -# is followed by `[`, an operator or whitespace so that other names do not match. +# A cuda-bindings requirement of one extra in METADATA. The name is followed by +# `[`, an operator or whitespace so that other names do not match. _REQUIRES_DIST_RE = re.compile( - r"^Requires-Dist:\s*cuda-bindings(?=[\[<>=!~\s])(?P[^;]*);\s*extra\s*==\s*['\"]cu(?P\d+)['\"]" + r"^Requires-Dist:\s*cuda-bindings(?=[\[<>=!~\s])(?P[^;]*);\s*extra\s*==\s*['\"](?P[^'\"]+)['\"]" +) +# A self-reference of one extra, such as `cuda-core[minimal-sysctk13]; extra == "cu13"`. +_SELF_REFERENCE_RE = re.compile( + r"^Requires-Dist:\s*cuda-core\s*\[(?P[^\]]*)\]\s*;\s*extra\s*==\s*['\"](?P[^'\"]+)['\"]" ) _FLOOR_RE = re.compile(r">=\s*(\d+)\.(\d+)\.(\d+)") +def _bindings_requirements(metadata: str, extra: str, seen: tuple[str, ...] = ()) -> list[tuple[str, str]]: + """(line, specifier) of each versioned cuda-bindings requirement of `extra`, following self-references.""" + seen = (*seen, extra) + found = [] + for line in metadata.splitlines(): + if (m := _REQUIRES_DIST_RE.match(line)) is not None and m.group("extra") == extra: + # An unversioned requirement such as `cuda-bindings[all]` only selects extras of cuda-bindings. + if re.sub(r"\[[^\]]*\]", "", m.group("spec")).strip(): + found.append((line, m.group("spec"))) + elif (m := _SELF_REFERENCE_RE.match(line)) is not None and m.group("extra") == extra: + for name in (name.strip() for name in m.group("extras").split(",")): + if name and name not in seen: + found.extend(_bindings_requirements(metadata, name, seen)) + return found + + def floor_from_metadata(metadata: str, major: int) -> str: """The floor that the `cu` extra of a wheel's METADATA pins cuda-bindings to.""" - for line in metadata.splitlines(): - m = _REQUIRES_DIST_RE.match(line) - if m is None or int(m.group("major")) != major: - continue - floor = _FLOOR_RE.search(m.group("spec")) + for line, spec in _bindings_requirements(metadata, f"cu{major}"): + floor = _FLOOR_RE.search(spec) if floor is None: raise SystemExit(f"METADATA pins cuda-bindings without a floor for the cu{major} extra: {line.strip()!r}") if int(floor.group(1)) != major: diff --git a/ci/tools/tests/test_cuda_core_bindings_floor.py b/ci/tools/tests/test_cuda_core_bindings_floor.py index 5f4b18dc97d..86a9b220ff1 100644 --- a/ci/tools/tests/test_cuda_core_bindings_floor.py +++ b/ci/tools/tests/test_cuda_core_bindings_floor.py @@ -69,6 +69,41 @@ def test_ignores_lookalike_names_and_other_extras(): assert tool.floor_from_metadata(metadata, 13) == "13.4.1" +# The layout in which cu includes minimal-sysctk, which pins the floor. +SELF_REFERENCING_METADATA = """\ +Metadata-Version: 2.4 +Name: cuda-core +Version: 1.3.1 +Requires-Dist: cuda-pathfinder>=1.4.2 +Requires-Dist: numpy +Provides-Extra: minimal-sysctk12 +Requires-Dist: cuda-bindings<13,>=12.9.8; extra == "minimal-sysctk12" +Provides-Extra: minimal-sysctk13 +Requires-Dist: cuda-bindings<14,>=13.5.0; extra == "minimal-sysctk13" +Provides-Extra: cu12 +Requires-Dist: cuda-core[minimal-sysctk12]; extra == "cu12" +Requires-Dist: cuda-bindings[all]; extra == "cu12" +Requires-Dist: cuda-toolkit==12.*; extra == "cu12" +Provides-Extra: cu13 +Requires-Dist: cuda-core[minimal-sysctk13]; extra == "cu13" +Requires-Dist: cuda-bindings[all]; extra == "cu13" +Requires-Dist: cuda-toolkit==13.*; extra == "cu13" +""" + + +@pytest.mark.agent_authored(model="claude-opus-5-5") +@pytest.mark.parametrize(("major", "floor"), [(12, "12.9.8"), (13, "13.5.0")]) +def test_follows_the_self_reference_of_a_cu_extra(tmp_path, major, floor): + assert tool.floor_from_wheel(_wheel(tmp_path, SELF_REFERENCING_METADATA), major) == floor + + +@pytest.mark.agent_authored(model="claude-opus-5-5") +def test_rejects_a_self_reference_to_an_extra_without_cuda_bindings(): + metadata = 'Requires-Dist: cuda-core[minimal-sysctk13]; extra == "cu13"\nRequires-Dist: cuda-bindings[all]; extra == "cu13"\n' + with pytest.raises(SystemExit, match="no cuda-bindings requirement for the cu13 extra"): + tool.floor_from_metadata(metadata, 13) + + @pytest.mark.agent_authored(model="claude-fable-5-1") def test_rejects_a_missing_extra(tmp_path): with pytest.raises(SystemExit, match="no cuda-bindings requirement for the cu11 extra"): diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index a97f37edcfa..afa7ddb0d0e 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -74,15 +74,18 @@ Nothing else moves a floor. A new `cuda-bindings` release does not, and a `cuda.core` release does not. Do not raise a floor to the newest release. To bump: -1. Edit the `cuda-bindings` pin in the `cu12` or `cu13` extra in - `pyproject.toml`. That is the only version to type. +1. Edit the `cuda-bindings` pin in the `minimal-sysctk12` or + `minimal-sysctk13` extra in `pyproject.toml`. That is the only version to + type. The `cu12` and `cu13` extras include these extras and carry no + version of their own. 2. Check `ci/versions.yml`. The `build` pin is the current major and the `prev_build` pin is the prior major. Neither toolkit pin may sit below its floor's major.minor, or CI builds a configuration that the build rejects. A toolkit ahead of the floor is the bump window described below. The pre-commit hook `check-cuda-core-bindings-floor` (`toolshed/check_cuda_core_bindings_floor.py`) checks the pins. It also - checks that no documentation page spells a floor out by hand. + validates the extras and checks that no documentation page spells a floor + out by hand. 3. Add to the release notes an entry that names the new floors. The support-policy table in `docs/source/support.rst` reads the floors and the release version at docs-build time. It needs no edit. @@ -119,8 +122,9 @@ the literal floor, until the floor is bumped. The order is: is normally its first release. A `cuda.core` release must not ship while the window is open. In the window, -the `cu` extra can resolve to a `cuda-bindings` that fails the header -check at import, and CI does not test the literal floor. +the `minimal-sysctk` and `cu` extras can resolve to a +`cuda-bindings` that fails the header check at import, and CI does not test +the literal floor. ## Testing expectations diff --git a/cuda_core/DESCRIPTION.rst b/cuda_core/DESCRIPTION.rst index 9759aed8731..136ad8fa635 100644 --- a/cuda_core/DESCRIPTION.rst +++ b/cuda_core/DESCRIPTION.rst @@ -26,4 +26,6 @@ Installation pip install cuda-core[cu12] +Use ``[cu13]`` for CUDA 13. With a CUDA Toolkit installed on the system, ``[minimal-sysctk12]`` and ``[minimal-sysctk13]`` install only ``cuda-bindings``. + Please refer to the `installation instructions `_ for different ways of installing ``cuda.core``, including building from source. diff --git a/cuda_core/cuda/core/_bindings_floor.py b/cuda_core/cuda/core/_bindings_floor.py index 9fd71f6b0af..b7782d67829 100644 --- a/cuda_core/cuda/core/_bindings_floor.py +++ b/cuda_core/cuda/core/_bindings_floor.py @@ -14,9 +14,13 @@ https://github.com/NVIDIA/cuda-python/issues/2783 and the support policy in the documentation. -Exactly one place declares the floors: the ``cu12`` and ``cu13`` extras in -``pyproject.toml``. Each extra pins ``cuda-bindings>=,<``. -Everything else derives from them through :func:`floors_from_extras`: +Exactly one place declares the floors: the ``minimal-sysctk12`` and +``minimal-sysctk13`` extras in ``pyproject.toml``. Each pins +``cuda-bindings>=,<``. The ``cu12`` and ``cu13`` extras +include them through a ``cuda-core[minimal-sysctk]`` self-reference, so +that pip and uv see the floor whichever extra a dependent requests. Everything +else reads the floor of each ``cu`` extra, following that +self-reference, through :func:`floors_from_extras`: - The build backend, ``build_hooks.py``, checks the installed cuda-bindings against the floor. It checks that the ``cuda.h`` that it compiles against is @@ -28,6 +32,8 @@ - The documentation reads the floors into substitutions in ``docs/source/conf.py``. - ``toolshed/check_cuda_core_bindings_floor.py``, a pre-commit hook, checks that the CI toolkit pins in ``ci/versions.yml`` sit in the minors of the floors. + It also validates the extras and checks that each ``cu`` extra includes + its ``minimal-sysctk`` extra. The build and the tools above load this module by file path. It therefore uses the standard library only and must not import anything from ``cuda``. @@ -56,6 +62,8 @@ _REQUIREMENT_RE = re.compile( r"^\s*cuda-bindings(?=[\[<>=!~;\s]|$)\s*(?:\[[^\]]*\])?\s*(?P[^;]*?)\s*(?:;.*)?$" ) +# A self-reference such as `cuda-core[minimal-sysctk13]`, up to an optional environment marker. +_SELF_REFERENCE_RE = re.compile(r"^\s*cuda-core\s*\[(?P[^\]]*)\]\s*(?:;.*)?$") _FLOOR_SPEC_RE = re.compile(r"^>=(\d+)\.(\d+)\.(\d+)$") # The upper bound: `<14`, `<14.0` or `<14.0.0`, or the equivalent `==13.*`. _UPPER_SPEC_RE = re.compile(r"^<(\d+)(?:\.0)*$") @@ -92,7 +100,10 @@ def floors_from_extras(extras: Mapping[str, Sequence[str]]) -> dict[int, tuple[i """The floor per CUDA major, from the ``cu`` extras of ``pyproject.toml``. ``extras`` is the parsed ``[project.optional-dependencies]`` table. Each - ``cu`` extra must list exactly one cuda-bindings requirement with two + ``cu`` extra must list exactly one versioned cuda-bindings requirement, + either itself or through a ``cuda-core[]`` self-reference such as + ``cuda-core[minimal-sysctk]``. An unversioned requirement such as + ``cuda-bindings[all]`` does not count. The requirement has two specifiers, in either order. One is the floor, ``>=..``. The other is an upper bound that excludes the next major, ``<`` or ``==.*``. Anything else raises ValueError with the name of the extra, so @@ -100,12 +111,16 @@ def floors_from_extras(extras: Mapping[str, Sequence[str]]) -> dict[int, tuple[i the floor. """ floors: dict[int, tuple[int, int, int]] = {} - for extra, requirements in extras.items(): + for extra in extras: m = _EXTRA_RE.match(extra) if m is None: continue major = int(m.group(1)) - matches = [(r, rm) for r in requirements if (rm := _REQUIREMENT_RE.match(r)) is not None] + matches = [ + (r, rm) + for r in _expand_self_references(extras, extra) + if (rm := _REQUIREMENT_RE.match(r)) is not None and rm.group("specifiers") + ] if len(matches) != 1: raise ValueError( f"the {extra!r} extra must list exactly one cuda-bindings requirement, found {len(matches)}" @@ -130,6 +145,21 @@ def floors_from_extras(extras: Mapping[str, Sequence[str]]) -> dict[int, tuple[i return dict(sorted(floors.items())) +def _expand_self_references(extras: Mapping[str, Sequence[str]], extra: str, seen: tuple[str, ...] = ()) -> list[str]: + """The requirements of ``extra``, with each ``cuda-core[...]`` replaced by the requirements of the extras it names.""" + seen = (*seen, extra) + requirements = [] + for requirement in extras.get(extra, ()): + m = _SELF_REFERENCE_RE.match(requirement) + if m is None: + requirements.append(requirement) + continue + for name in (name.strip() for name in m.group("extras").split(",")): + if name and name not in seen: + requirements.extend(_expand_self_references(extras, name, seen)) + return requirements + + def _confined_major(specifier: str) -> int | None: """The one major a specifier confines cuda-bindings to: ``<14`` -> 13, ``==13.*`` -> 13, else None.""" if (m := _UPPER_SPEC_RE.match(specifier)) is not None: diff --git a/cuda_core/docs/source/install.rst b/cuda_core/docs/source/install.rst index 7e67fadad8d..4b98f33afe0 100644 --- a/cuda_core/docs/source/install.rst +++ b/cuda_core/docs/source/install.rst @@ -47,21 +47,31 @@ Installing from PyPI ``cuda.core`` works with ``cuda-bindings`` (part of ``cuda-python``) 12 or 13 at or above the release's floor for that major. See :ref:`cuda-core-bindings-floor`. The ``cu12`` and ``cu13`` -extras install a compatible version. Test dependencies now use the ``cuda-toolkit`` metapackage for improved dependency resolution. For example with CUDA 12: +extras install a compatible version together with the CUDA Toolkit wheels. Test dependencies now use the ``cuda-toolkit`` metapackage for improved dependency resolution. For example with CUDA 12: .. code-block:: console $ pip install cuda-core[cu12] -and likewise use ``[cu13]`` for CUDA 13. +and likewise use ``[cu13]`` for CUDA 13. The ``minimal-sysctk12`` and ``minimal-sysctk13`` +extras install only a compatible ``cuda-bindings``, for use with a CUDA Toolkit installed on the +system: + +.. code-block:: console + + $ pip install cuda-core[minimal-sysctk12] If you upgrade ``cuda-core`` alone, the installed ``cuda-bindings`` can be older than the new release requires. ``import cuda.core`` then reports the required version and the ``pip`` command that installs it. -Do not pin ``cuda-bindings`` in your own requirements. Depend on ``cuda-core`` with the ``cu12`` -or ``cu13`` extra, and let ``cuda-core`` choose the ``cuda-bindings`` version. A pin of your own -can conflict with the floor of a later ``cuda-core`` release. +Do not pin ``cuda-bindings`` in your own requirements. In a package that depends on +``cuda-core``, request ``cuda-core[minimal-sysctk12]`` or ``cuda-core[minimal-sysctk13]`` from +your own CUDA 12 and CUDA 13 extras, or ``cuda-core[cu12]`` or ``cuda-core[cu13]`` to also pull +the CUDA Toolkit wheels, and let ``cuda-core`` choose the ``cuda-bindings`` version. A plain +``cuda-core`` requirement does not pass the floor to pip or uv: they can install an older +``cuda-bindings`` next to it, ``pip check`` does not report it, and ``import cuda.core`` fails. +A pin of your own can conflict with the floor of a later ``cuda-core`` release. Installing from Conda (conda-forge) diff --git a/cuda_core/docs/source/release/1.3.1-notes.rst b/cuda_core/docs/source/release/1.3.1-notes.rst index 938de84e732..01a2e587fc8 100644 --- a/cuda_core/docs/source/release/1.3.1-notes.rst +++ b/cuda_core/docs/source/release/1.3.1-notes.rst @@ -6,6 +6,21 @@ ``cuda.core`` 1.3.1 Release Notes ================================= +Highlights +---------- + +- New ``minimal-sysctk12`` and ``minimal-sysctk13`` extras install only ``cuda-bindings`` at or + above this release's floor for that CUDA major, without the CUDA Toolkit wheels that ``cu12`` + and ``cu13`` add. The ``cu12`` and ``cu13`` extras include them. Packages that depend on + ``cuda-core`` should request one of these extras, so that pip and uv see the floor; with a + plain ``cuda-core`` requirement, an older ``cuda-bindings`` is installed and fails at import. + See :ref:`cuda-core-bindings-floor`. + (`#3459 `__) + +- ``cuda-core`` 1.3.0 is yanked from PyPI. It declared its floors only in the ``cu12`` and + ``cu13`` extras, so a resolver that cannot use 1.3.1 could fall back to it next to an older + ``cuda-bindings``. + Fixes and enhancements ---------------------- diff --git a/cuda_core/docs/source/support.rst b/cuda_core/docs/source/support.rst index 08670da95a1..4fc7507a511 100644 --- a/cuda_core/docs/source/support.rst +++ b/cuda_core/docs/source/support.rst @@ -73,9 +73,10 @@ For each supported CUDA major version, each ``cuda-core`` release declares a min ``cuda-bindings`` version, its *floor*. The floor is the oldest ``cuda-bindings`` release that meets two conditions. It was generated from the ``cuda.h`` major.minor that the published ``cuda-core`` wheels are built against. It provides every ``cuda-bindings`` API and fix that -``cuda-core`` needs. The ``cu12`` and ``cu13`` extras of ``cuda-core`` in ``pyproject.toml`` -declare the floors of the current release. The build, the import-time check, this page, and CI -all read them from there. +``cuda-core`` needs. The ``minimal-sysctk12`` and ``minimal-sysctk13`` extras of ``cuda-core`` +in ``pyproject.toml`` declare the floors of the current release, and the ``cu12`` and ``cu13`` +extras include them. The build, the import-time check, this page, and CI all read them from +there. pip and uv see a floor only when ``cuda-core`` is installed with one of these extras. .. list-table:: ``cuda-bindings`` floors :header-rows: 1 diff --git a/cuda_core/pyproject.toml b/cuda_core/pyproject.toml index f3ea7e35deb..151dd0c8a0c 100644 --- a/cuda_core/pyproject.toml +++ b/cuda_core/pyproject.toml @@ -54,14 +54,18 @@ dependencies = [ "numpy", ] -# The cuda-bindings pins below are the cuda-bindings *floors* of this release, one -# per CUDA major. This is the only place that declares them. build_hooks.py, the -# import-time check, the docs and CI read them from here (see -# cuda/core/_bindings_floor.py and the "Bumping the cuda-bindings floor" -# checklist in AGENTS.md). Keep the form `>=,<`. +# The cuda-bindings pins in the minimal-sysctk extras are the cuda-bindings +# *floors* of this release, one per CUDA major. This is the only place that +# declares them. Keep the form `>=,<`. The cu extras +# include them, so that pip and uv see the floor whichever extra a dependent +# requests. build_hooks.py, the import-time check, the docs and CI read each +# floor through the cu extra (see cuda/core/_bindings_floor.py and the +# "Bumping the cuda-bindings floor" checklist in AGENTS.md). [project.optional-dependencies] -cu12 = ["cuda-bindings[all]>=12.9.8,<13", "cuda-toolkit==12.*"] -cu13 = ["cuda-bindings[all]>=13.5.0,<14", "cuda-toolkit==13.*"] +minimal-sysctk12 = ["cuda-bindings>=12.9.8,<13"] +minimal-sysctk13 = ["cuda-bindings>=13.5.0,<14"] +cu12 = ["cuda-core[minimal-sysctk12]", "cuda-bindings[all]", "cuda-toolkit==12.*"] +cu13 = ["cuda-core[minimal-sysctk13]", "cuda-bindings[all]", "cuda-toolkit==13.*"] [dependency-groups] # Keep build in sync with build-system.requires; install local cuda-pathfinder separately. @@ -102,6 +106,19 @@ conflicts = [ { extra = "cu12" }, { extra = "cu13" }, ], + # uv locks every pair of extras unless told otherwise, and cu includes minimal-sysctk. + [ + { extra = "minimal-sysctk12" }, + { extra = "minimal-sysctk13" }, + ], + [ + { extra = "cu12" }, + { extra = "minimal-sysctk13" }, + ], + [ + { extra = "cu13" }, + { extra = "minimal-sysctk12" }, + ], [ { group = "test-cu12" }, { group = "test-cu13" }, diff --git a/cuda_core/tests/test_bindings_floor.py b/cuda_core/tests/test_bindings_floor.py index 05b218722f3..958713dca92 100644 --- a/cuda_core/tests/test_bindings_floor.py +++ b/cuda_core/tests/test_bindings_floor.py @@ -124,6 +124,46 @@ def test_floors_from_extras_ignores_lookalike_names(): assert floors_from_extras(extras) == {13: (13, 4, 1)} +@pytest.mark.agent_authored(model="claude-opus-5-5") +def test_floors_from_extras_follows_self_references(): + extras = { + "minimal-sysctk12": ["cuda-bindings>=12.9.8,<13"], + "minimal-sysctk13": ["cuda-bindings>=13.5.0,<14"], + "cu12": ["cuda-core[minimal-sysctk12]", "cuda-bindings[all]", "cuda-toolkit==12.*"], + "cu13": ["cuda-core[minimal-sysctk13]", "cuda-bindings[all]", "cuda-toolkit==13.*"], + } + assert floors_from_extras(extras) == {12: (12, 9, 8), 13: (13, 5, 0)} + # A cycle of self-references ends. + cyclic = {"a": ["cuda-core[b]"], "b": ["cuda-core[a,cu13]"], "cu13": ["cuda-core[a]", "cuda-bindings>=13.5.0,<14"]} + assert floors_from_extras(cyclic) == {13: (13, 5, 0)} + + +@pytest.mark.agent_authored(model="claude-opus-5-5") +@pytest.mark.parametrize( + ("extras", "message"), + [ + ( + {"minimal-sysctk13": ["cuda-bindings>=13.5,<14"], "cu13": ["cuda-core[minimal-sysctk13]"]}, + "the 'cu13' extra must pin cuda-bindings as", + ), + ( + { + "minimal-sysctk13": ["cuda-bindings>=13.5.0,<14"], + "cu13": ["cuda-core[minimal-sysctk13]", "cuda-bindings[all]>=13.5.0,<14"], + }, + "exactly one cuda-bindings requirement, found 2", + ), + ( + {"cu13": ["cuda-core[minimal-sysctk13]", "cuda-bindings[all]"]}, + "exactly one cuda-bindings requirement, found 0", + ), + ], +) +def test_floors_from_extras_rejects_bad_self_references(extras, message): + with pytest.raises(ValueError, match=re.escape(message)): + floors_from_extras(extras) + + @pytest.mark.agent_authored(model="claude-fable-5-1") @pytest.mark.parametrize( ("version", "expected"), @@ -359,6 +399,31 @@ def test_a_malformed_extra_fails_the_hook(self, hook, tmp_path, capsys): assert hook.main(["--repo-root", str(tmp_path)]) == 1 assert "pyproject.toml: the 'cu13' extra must pin cuda-bindings" in capsys.readouterr().err + @pytest.mark.agent_authored(model="claude-opus-5-5") + @pytest.mark.parametrize( + "cu13", + [ + '["cuda-bindings[all]>=13.4.1,<14", "cuda-toolkit==13.*"]', + '["cuda-core[minimal-sysctk12]", "cuda-bindings[all]", "cuda-toolkit==13.*"]', + None, + ], + ) + def test_a_cu_extra_without_its_minimal_sysctk_extra_fails_the_hook(self, hook, tmp_path, cu13): + pyproject = tmp_path / "cuda_core" / "pyproject.toml" + pyproject.parent.mkdir() + text = '[project.optional-dependencies]\nminimal-sysctk13 = ["cuda-bindings>=13.4.1,<14"]\n' + if cu13 is not None: + text += f"cu13 = {cu13}\n" + pyproject.write_text(text, encoding="utf-8") + assert hook.cu_extra_problems(tmp_path, {13: (13, 4, 1)}) == [ + f"{hook.PYPROJECT}: the 'cu13' extra must list 'cuda-core[minimal-sysctk13]', so that it carries the floor" + ] + pyproject.write_text( + text.split("cu13 =")[0] + 'cu13 = ["cuda-core[minimal-sysctk13]", "cuda-bindings[all]"]\n', + encoding="utf-8", + ) + assert hook.cu_extra_problems(tmp_path, {13: (13, 4, 1)}) == [] + @pytest.mark.agent_authored(model="claude-fable-5-1") def test_ci_toolkit_pins_must_not_sit_below_the_floors_minor(self, hook): floors = {12: (12, 9, 8), 13: (13, 4, 1)} diff --git a/toolshed/check_cuda_core_bindings_floor.py b/toolshed/check_cuda_core_bindings_floor.py index c01a2d6fea1..d73909a13e6 100644 --- a/toolshed/check_cuda_core_bindings_floor.py +++ b/toolshed/check_cuda_core_bindings_floor.py @@ -3,20 +3,24 @@ """Check that everything that depends on the cuda-bindings floor agrees with it. -The floor of each CUDA major is declared once, by the `cu12`/`cu13` extras in -cuda_core/pyproject.toml (see cuda_core/cuda/core/_bindings_floor.py). Most -consumers read it from there. These constraints cannot be derived from it, so -this script checks them, as a pre-commit hook and from +The floor of each CUDA major is declared once, by the +`minimal-sysctk12`/`minimal-sysctk13` extras in cuda_core/pyproject.toml, which +the `cu12`/`cu13` extras include (see cuda_core/cuda/core/_bindings_floor.py). +Most consumers read it from there. These constraints cannot be derived from it, +so this script checks them, as a pre-commit hook and from cuda_core/tests/test_bindings_floor.py: 1. The extras parse: each `cu` extra pins exactly - `cuda-bindings[...]>=..,<`. -2. ci/versions.yml builds each major against a CUDA Toolkit of at least the + `cuda-bindings[...]>=..,<`, itself or through the + extra that it includes. +2. Each `cu` extra lists `cuda-core[minimal-sysctk]`, so that it carries + the floor to pip and uv. +3. ci/versions.yml builds each major against a CUDA Toolkit of at least the floor's major.minor. A toolkit below the floor's minor cannot build the floor's cuda-bindings, because the build requires the header that cuda-bindings was generated from. A toolkit ahead of the floor is the toolkit-bump window described in cuda_core/AGENTS.md. -3. No documentation page spells a floor out by hand. docs/source/conf.py +4. No documentation page spells a floor out by hand. docs/source/conf.py provides |cuda-bindings-floor-cu12| and |cuda-bindings-floor-cu13|. Release notes are history and are exempt. @@ -51,11 +55,25 @@ def load_floor_module(repo_root: Path): return module +def read_extras(repo_root: Path) -> dict[str, list[str]]: + with open(repo_root / PYPROJECT, "rb") as f: + return tomllib.load(f)["project"]["optional-dependencies"] + + def read_floors(repo_root: Path) -> dict[int, tuple[int, int, int]]: """The floors declared by the pyproject extras. Raises ValueError if they do not parse.""" - with open(repo_root / PYPROJECT, "rb") as f: - extras = tomllib.load(f)["project"]["optional-dependencies"] - return load_floor_module(repo_root).floors_from_extras(extras) + return load_floor_module(repo_root).floors_from_extras(read_extras(repo_root)) + + +def cu_extra_problems(repo_root: Path, floors: dict[int, tuple[int, int, int]]) -> list[str]: + """`cu` extras that do not include `minimal-sysctk`. pip and uv would not see the floor through them.""" + extras = read_extras(repo_root) + problems = [] + for major in floors: + wanted = f"cuda-core[minimal-sysctk{major}]" + if wanted not in (r.replace(" ", "") for r in extras.get(f"cu{major}", [])): + problems.append(f"{PYPROJECT}: the 'cu{major}' extra must list {wanted!r}, so that it carries the floor") + return problems def ci_pin_problems(floors: dict[int, tuple[int, int, int]], versions_yml: str) -> list[str]: @@ -99,7 +117,8 @@ def check(repo_root: Path) -> list[str]: floors = read_floors(repo_root) except ValueError as exc: return [f"{PYPROJECT}: {exc}"] - problems = ci_pin_problems(floors, (repo_root / CI_VERSIONS).read_text(encoding="utf-8")) + problems = cu_extra_problems(repo_root, floors) + problems += ci_pin_problems(floors, (repo_root / CI_VERSIONS).read_text(encoding="utf-8")) problems += docs_problems(repo_root) return problems