From 45576c9b16a86b736ecb7dfb2084f5bb79b469e7 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Tue, 17 Nov 2026 07:28:51 +0000 Subject: [PATCH 1/2] fix(samples): reserve one SM partition instead of two unequal groups The greenContext sample asked the driver for two unequal SM groups with SMResourceOptions(count=(long, critical)). On drivers older than 13.1, cuda.core splits SMs with cuDevSmResourceSplitByCount, which sizes every group from a single min_count and rejects the request with "CUDA 12 SM splitting only supports homogeneous count values". The discovery probe swallowed that exception and reported it as a driver refusal, so auto selection only worked on devices that can express an equal two-way split. On the 14-SM sm_75 part in NVBug 7106432 (7 SMs per side, min_partition_size 2) no such pair exists, so the sample exited 1 and test_samples.py failed. Reserve the critical partition with a single-group request and give the device remainder to the long kernel, which both driver split APIs accept. Probes report the last driver reason instead of hiding it, --split keeps its shape (CRITICAL is reserved and must be aligned, LONG is the remainder), and a device that cannot host two SM partitions exits with the sample waiver code instead of failing. That also brings the sample in line with the negotiated EXIT_WAIVED convention the other samples follow. Extend test_sample_verification.py with two GPU-free tests that drive the sample's split selection against the count-API rules on a 14-SM device and on one too small to split. --- .../example_tests/test_sample_verification.py | 83 ++++++++++ samples/cuda_core/greenContext/README.md | 29 ++-- .../cuda_core/greenContext/greenContext.py | 153 +++++++++--------- 3 files changed, 182 insertions(+), 83 deletions(-) diff --git a/cuda_core/tests/example_tests/test_sample_verification.py b/cuda_core/tests/example_tests/test_sample_verification.py index 4130632c8d3..6dca813907f 100644 --- a/cuda_core/tests/example_tests/test_sample_verification.py +++ b/cuda_core/tests/example_tests/test_sample_verification.py @@ -4,6 +4,8 @@ from __future__ import annotations import ast +import importlib.util +import types from pathlib import Path import numpy as np @@ -17,6 +19,7 @@ "glInteropFluid", "glInteropMipmapLod", "glInteropPlasma", + "greenContext", "ipcMemoryPool", "launchConfigTuning", "memoryResources", @@ -141,3 +144,83 @@ def test_verification_exception_is_reported_as_sample_failure(tmp_path: Path) -> assert result.status == "FAIL" assert result.return_code != 0 + + +def _load_green_context_sample(): + sample_path = REPO_ROOT / "samples" / "cuda_core" / "greenContext" / "greenContext.py" + spec = importlib.util.spec_from_file_location("greenContext_sample", sample_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _Partition: + def __init__(self, sm_count: int) -> None: + self.sm_count = sm_count + + +class _CountApiSM: + """Stand-in for ``SMResource`` behind ``cuDevSmResourceSplitByCount``. + + Mirrors the two rules that matter here: equal-sized groups from one + ``min_count``, and sizes that are multiples of the device's minimum SM + partition size. The greenContext sample cannot ask for two different group + sizes, so a probe of it must never reach the unequal-count branch. + """ + + def __init__(self, sm_count: int, min_partition_size: int) -> None: + self.sm_count = sm_count + self.min_partition_size = min_partition_size + + def split(self, options, *, dry_run: bool = False): + counts = options.count if isinstance(options.count, tuple) else (options.count,) + if len(set(counts)) > 1: + raise RuntimeError("CUDA 12 SM splitting only supports homogeneous count values") + if any(count % self.min_partition_size for count in counts): + raise RuntimeError("CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION") + if sum(counts) > self.sm_count: + raise RuntimeError("CUDA_ERROR_INVALID_RESOURCE_CONFIGURATION") + return [_Partition(count) for count in counts], _Partition(self.sm_count - sum(counts)) + + +@pytest.mark.agent_authored(model="deepseek/deepseek-v4.1-flash-1017") +def test_green_context_auto_split_without_equal_two_way_split() -> None: + """Regression: NVBug 7106432 - 14 SMs cannot form an equal two-way split.""" + sm = _CountApiSM(sm_count=14, min_partition_size=2) + + found, reason = _load_green_context_sample()._find_working_split(sm) + + assert reason is None + assert found is not None + long_count, critical_count, _granularity = found + assert long_count + critical_count == sm.sm_count + assert long_count > 0 + assert critical_count % sm.min_partition_size == 0 + + +@pytest.mark.agent_authored(model="deepseek/deepseek-v4.1-flash-1017") +def test_green_context_device_without_usable_split_waives() -> None: + """A device without a usable two-partition split is a waiver, not a failure.""" + sample = _load_green_context_sample() + sm = _CountApiSM(sm_count=2, min_partition_size=2) + device = types.SimpleNamespace(resources=types.SimpleNamespace(sm=sm)) + + found, _reason = sample._find_working_split(sm) + assert found is None + + with pytest.raises(SystemExit) as exc_info: + sample.parse_split(None, device) + assert exc_info.value.code == sample.EXIT_WAIVED + + +@pytest.mark.agent_authored(model="deepseek/deepseek-v4.1-flash-1017") +def test_green_context_explicit_split_must_take_the_remainder() -> None: + """``--split A,B`` is accepted only when A is what B leaves on the device.""" + sample = _load_green_context_sample() + device = types.SimpleNamespace(resources=types.SimpleNamespace(sm=_CountApiSM(14, 2))) + + assert sample.parse_split("12,2", device) == (12, 2) + + with pytest.raises(SystemExit) as exc_info: + sample.parse_split("10,2", device) + assert exc_info.value.code == 1 diff --git a/samples/cuda_core/greenContext/README.md b/samples/cuda_core/greenContext/README.md index f602a0349b0..ac5121493d6 100644 --- a/samples/cuda_core/greenContext/README.md +++ b/samples/cuda_core/greenContext/README.md @@ -41,8 +41,9 @@ partition. The reference row lets you separate those two effects: - Querying a device's SM resources via `Device.resources.sm` and reading `sm_count`, `min_partition_size`, `coscheduled_alignment` -- Splitting an `SMResource` into disjoint partitions with - `sm.split(SMResourceOptions(count=(A, B)))` +- Splitting an `SMResource` into disjoint partitions: reserve one with + `sm.split(SMResourceOptions(count=B))` and use the returned remainder + for the other - Creating a green context from an SM partition via `Device.create_context(ContextOptions(resources=[group]))` - Creating a non-blocking stream on a green context with @@ -61,8 +62,8 @@ partition. The reference row lets you separate those two effects: ### From `cuda.core` - `Device.resources.sm` - the device's SM-type device resource -- `SMResource.split(SMResourceOptions(count=(A, B)))` - partition SMs - into disjoint groups (plus an optional remainder) +- `SMResource.split(SMResourceOptions(count=B))` - partition SMs by + reserving `B` SMs and keeping the returned remainder - `Device.create_context(ContextOptions(resources=[sm_group]))` - create a green context provisioned with a specific SM partition - `Context.is_green` / `Context.resources` - introspect a green @@ -111,13 +112,17 @@ pip install -r requirements.txt The auto-default split reserves a small partition (~16 SMs) for the critical kernel and gives the rest to the long-running kernel. The -exact sizes are chosen by probing the driver with a dry-run `sm.split`, +exact size is chosen by probing the driver with a dry-run `sm.split`, escalating the alignment granularity in powers of two until the driver -accepts the pair. This handles architectures where the driver enforces +accepts it. This handles architectures where the driver enforces stricter alignment (e.g. TPC/GPC-pair alignment on Blackwell) than the reported `min_partition_size`. When that happens the sample prints a `Note:` line with the granularity it landed on. +If the driver accepts no partition at all, the sample reports that the +device cannot host two SM partitions and exits with the sample-waiver +code instead of failing. + ```bash cd samples/cuda_core/greenContext python greenContext.py @@ -151,11 +156,13 @@ python greenContext.py --device 1 ``` --device CUDA device ID (default: 0) --split SM split as 'LONG,CRITICAL', e.g. '112,16'. - Each side must be a multiple of the device's - min_partition_size, and the driver may enforce additional - architecture-specific alignment (e.g. TPC/partition-grid - alignment on Blackwell). Omit --split to auto-select a - driver-accepted split. + CRITICAL SMs are reserved for the critical kernel and must + be a multiple of the device's min_partition_size; the driver + may enforce additional architecture-specific alignment + (e.g. TPC/partition-grid alignment on Blackwell). The + long-running kernel gets every SM the critical partition + leaves behind, so LONG must equal that remainder. + Omit --split to auto-select a driver-accepted split. --delay-us Per-block busy-wait of the delay kernel, in us (default: 2000) --delay-waves Number of waves of the delay kernel on the long partition. Drives the default --delay-blocks (default: 16) diff --git a/samples/cuda_core/greenContext/greenContext.py b/samples/cuda_core/greenContext/greenContext.py index a9145947608..bde4aea15df 100644 --- a/samples/cuda_core/greenContext/greenContext.py +++ b/samples/cuda_core/greenContext/greenContext.py @@ -71,6 +71,10 @@ launch, ) +# Samples report "cannot run here" with this code; the sample runner treats it as +# a waiver rather than a failure. +EXIT_WAIVED = int(os.environ.get("CUDA_PYTHON_SAMPLE_WAIVER_EXIT_CODE", "2")) + # Two CUDA kernels: # 1. The delay kernel spins until `cycles` SM clock ticks have elapsed. # 2. The critical kernel does a small amount of useful work. @@ -127,41 +131,55 @@ def _align_down(n: int, k: int) -> int: return (n // k) * k -def _driver_accepts_split(sm, long_count: int, critical_count: int) -> bool: - if long_count <= 0 or critical_count <= 0: - return False +def _probe_split(sm, critical_count: int) -> tuple[int | None, str | None]: + """ + Dry-run probe: reserve `critical_count` SMs and leave the rest of the device + to the long kernel. + + One group plus the remainder is the only request shape every driver split + API accepts: `cuDevSmResourceSplitByCount`, used on drivers older than + 13.1, sizes all groups from a single `min_count` and rejects unequal counts. + + Returns `(long_count, None)`, or `(None, reason)` if the driver says no. + """ try: - groups, _ = sm.split( - SMResourceOptions(count=(long_count, critical_count)), - dry_run=True, - ) - except Exception: - return False - actual = tuple(g.sm_count for g in groups) - return actual == (long_count, critical_count) + groups, remainder = sm.split(SMResourceOptions(count=critical_count), dry_run=True) + except Exception as e: + # Any rejection removes this candidate only; the catch is broad because + # cuda.core >= 1.0 does not export the rejection type. + return None, f"{type(e).__name__}: {e}" + + if len(groups) != 1: + return None, f"driver created {len(groups)} groups, expected 1" + reserved = groups[0].sm_count + if reserved != critical_count: + return None, f"driver reserved {reserved} SMs, not {critical_count}" + # Prefer the SMs the driver actually left: run_green_context requires the + # long partition to match that remainder. + return remainder.sm_count or (sm.sm_count - reserved), None -def _find_working_split(sm, prefer_critical: int | None = None) -> tuple[int, int, int] | None: +def _find_working_split(sm, prefer_critical: int | None = None) -> tuple[tuple[int, int, int] | None, str | None]: """ Probe the driver for a (long, critical) split it actually accepts. Escalates the alignment granularity from `min_partition_size` upward in - powers of two, requiring BOTH sides to be multiples of the current + powers of two, probing partition sizes that are multiples of the current granularity. This handles architectures where the driver's true allocation granularity is larger than the reported `min_partition_size` (e.g. TPC/GPC-pair alignment on Blackwell: on a 188-SM part `min_partition_size` is 8 but the driver actually requires - each side to be a multiple of 16). + the reserved partition to be a multiple of 16). - Returns (long_count, critical_count, granularity) or None. The - granularity is the smallest power-of-two multiple of - `min_partition_size` at which both sides are aligned and the driver - accepts the pair. + Returns ((long_count, critical_count, granularity), None), or + (None, last driver reason) if no candidate is accepted. The granularity + is the smallest power-of-two multiple of `min_partition_size` at which + the driver accepts a partition. """ total = sm.sm_count min_part = sm.min_partition_size if min_part <= 0: - return None + return None, "device reports no minimum SM partition size" if prefer_critical is None or prefer_critical <= 0: prefer_critical = max(min_part, min(16, total // 8)) @@ -169,6 +187,7 @@ def _find_working_split(sm, prefer_critical: int | None = None) -> tuple[int, in # Escalate granularity in powers of two. The upper bound is half of # `total` because below that we cannot fit two partitions of size # >= granularity. + last_reason = None granularity = min_part while granularity * 2 <= total: base = max(granularity, _align_down(prefer_critical, granularity)) @@ -194,15 +213,14 @@ def push(c: int, granularity: int = granularity, seen: set = seen, candidates: l push(base - step * granularity) for critical in candidates: - long_count = _align_down(total - critical, granularity) - if long_count < granularity: - continue - if _driver_accepts_split(sm, long_count, critical): - return long_count, critical, granularity + long_count, reason = _probe_split(sm, critical) + if long_count is not None: + return (long_count, critical, granularity), None + last_reason = reason granularity *= 2 - return None + return None, last_reason def _format_suggestion(sm, prefer_critical: int | None) -> str | None: @@ -210,7 +228,7 @@ def _format_suggestion(sm, prefer_critical: int | None) -> str | None: Return a '--split A,B' string the driver is known to accept, or None if we couldn't find one. """ - found = _find_working_split(sm, prefer_critical=prefer_critical) + found, _reason = _find_working_split(sm, prefer_critical=prefer_critical) if found is None: return None long_count, critical_count, _granularity = found @@ -221,7 +239,8 @@ def parse_split(arg: str | None, device: Device) -> tuple[int, int]: """ Parse the --split "A,B" CLI argument and validate it against the device. - Returns (long_count, critical_count). + B SMs are reserved for the critical kernel; the long kernel gets every + remaining SM on the device. Returns (long_count, critical_count). """ sm = device.resources.sm total = sm.sm_count @@ -229,25 +248,22 @@ def parse_split(arg: str | None, device: Device) -> tuple[int, int]: if arg is None: # Auto: reserve a small aligned slice for the critical kernel and - # hand the rest (also aligned) to the long-running kernel. We - # can't trust `min_partition_size` alone: on some GPUs (e.g. - # 188-SM Blackwell) the driver requires stricter alignment than - # it reports. Escalate the granularity until the driver accepts - # a pair. + # hand every remaining SM to the long-running kernel. We can't trust + # `min_partition_size` alone: on some GPUs (e.g. 188-SM Blackwell) + # the driver requires stricter alignment than it reports. Escalate + # the granularity until the driver accepts a partition. prefer_critical = max(min_part, min(16, total // 8)) - found = _find_working_split(sm, prefer_critical=prefer_critical) + found, reason = _find_working_split(sm, prefer_critical=prefer_critical) if found is None: print( - "Error: could not find an SM split that the driver accepts " + "Error: could not reserve an SM partition that the driver accepts " f"on this device (total SMs={total}, " f"min_partition_size={min_part})." ) - print( - " The driver enforces architecture-specific alignment " - "rules beyond min_partition_size; try passing an explicit " - "--split." - ) - sys.exit(1) + if reason is not None: + print(f" Last driver response: {reason}") + print(" The device cannot host two SM partitions; skipping the sample.") + sys.exit(EXIT_WAIVED) long_count, critical_count, granularity = found if granularity > min_part: print( @@ -271,10 +287,10 @@ def parse_split(arg: str | None, device: Device) -> tuple[int, int]: errors = [] if long_count <= 0 or critical_count <= 0: errors.append("both partition sizes must be positive") - if long_count % min_part != 0 or critical_count % min_part != 0: - errors.append(f"each size must be a multiple of min_partition_size={min_part}") + if critical_count % min_part != 0: + errors.append(f"the critical partition must be a multiple of min_partition_size={min_part}") if long_count + critical_count > total: - errors.append(f"sum {long_count + critical_count} exceeds device total of {total} SMs") + errors.append(f"LONG + CRITICAL cannot exceed the device total of {total} SMs") if errors: print("Error: invalid --split value:") @@ -287,34 +303,22 @@ def parse_split(arg: str | None, device: Device) -> tuple[int, int]: # Confirm the driver itself accepts the split. The well-known alignment # checks above are necessary but not sufficient on every architecture. - try: - groups, _ = sm.split( - SMResourceOptions(count=(long_count, critical_count)), - dry_run=True, - ) - except Exception as e: - print(f"Error: driver rejected the requested split: {e}") + driver_long, reason = _probe_split(sm, critical_count) + if driver_long is None: + print(f"Error: driver rejected the requested split: {reason}") print( - " The sample's own alignment checks are not exhaustive on " - "every architecture; the driver enforces additional hardware " - "constraints (for example TPC/partition-grid alignment)." + " The driver applies hardware constraints the sample cannot " + "predict (for example TPC/partition-grid alignment)." ) - suggestion = _format_suggestion(sm, prefer_critical=critical_count) - if suggestion is not None: - print(f"Tip: a driver-accepted split on this device is {suggestion}") - sys.exit(1) - - actual = tuple(g.sm_count for g in groups) - if actual != (long_count, critical_count): - print(f"Error: driver adjusted the requested split to {actual}.") - suggestion = _format_suggestion(sm, prefer_critical=critical_count) - if suggestion is not None: - print(f"Tip: a driver-accepted split on this device is {suggestion}") - else: - print(" Pick a different --split, or omit it for the auto default.") - sys.exit(1) + elif driver_long != long_count: + print(f"Error: the driver left {driver_long} SMs for the long partition, not {long_count}.") + else: + return long_count, critical_count - return long_count, critical_count + suggestion = _format_suggestion(sm, prefer_critical=critical_count) + if suggestion is not None: + print(f"Tip: a driver-accepted split on this device is {suggestion}") + sys.exit(1) def compile_kernels(device: Device): @@ -498,9 +502,14 @@ def run_green_context( """Each kernel on its own green context, with disjoint SM partitions.""" long_count, critical_count = split sm = device.resources.sm - groups, _remainder = sm.split(SMResourceOptions(count=(long_count, critical_count))) - assert len(groups) == 2 - long_group, critical_group = groups + groups, remainder = sm.split(SMResourceOptions(count=critical_count)) + assert len(groups) == 1 + critical_group, long_group = groups[0], remainder + if long_group.sm_count != long_count or critical_group.sm_count != critical_count: + raise RuntimeError( + f"driver provided a {long_group.sm_count}+{critical_group.sm_count} SM split, " + f"expected {long_count}+{critical_count}" + ) # Create the large ctx last so it's closed first: order matters only for # ensuring we never try to close a ctx that's currently the thread's @@ -590,7 +599,7 @@ def main(): from cuda.bindings import driver as _d if _d.cuDriverGetVersion()[1] >= 13010: - sys.exit(int(os.environ.get("CUDA_PYTHON_SAMPLE_WAIVER_EXIT_CODE", "2"))) + sys.exit(EXIT_WAIVED) parser = argparse.ArgumentParser( description="Green Context sample using CUDA Core API", From a858343a399fc5ffb97a21e80b0ca3961d896ce6 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Tue, 17 Nov 2026 09:43:34 +0000 Subject: [PATCH 2/2] fix(samples): propagate probe allocation failures in greenContext Catch only candidate rejections while probing SM partitions: a MemoryError comes from cuda.core failing to allocate the split result, so it is a broken environment rather than a refused partition, and must not end up as a sample waiver. Also pin the negotiated waiver code behaviorally: load the sample with a non-default CUDA_PYTHON_SAMPLE_WAIVER_EXIT_CODE and assert that the no-partition path exits with it, and add a test that an allocation failure during probing propagates. Addresses CodeRabbit review on PR #3482. --- .../example_tests/test_sample_verification.py | 21 +++++++++++++++++-- .../cuda_core/greenContext/greenContext.py | 4 ++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cuda_core/tests/example_tests/test_sample_verification.py b/cuda_core/tests/example_tests/test_sample_verification.py index 6dca813907f..4e0b346b4bd 100644 --- a/cuda_core/tests/example_tests/test_sample_verification.py +++ b/cuda_core/tests/example_tests/test_sample_verification.py @@ -199,8 +199,10 @@ def test_green_context_auto_split_without_equal_two_way_split() -> None: @pytest.mark.agent_authored(model="deepseek/deepseek-v4.1-flash-1017") -def test_green_context_device_without_usable_split_waives() -> None: +def test_green_context_device_without_usable_split_waives(monkeypatch) -> None: """A device without a usable two-partition split is a waiver, not a failure.""" + # Non-default code, so a hard-coded exit value cannot pass this test. + monkeypatch.setenv("CUDA_PYTHON_SAMPLE_WAIVER_EXIT_CODE", "77") sample = _load_green_context_sample() sm = _CountApiSM(sm_count=2, min_partition_size=2) device = types.SimpleNamespace(resources=types.SimpleNamespace(sm=sm)) @@ -210,7 +212,22 @@ def test_green_context_device_without_usable_split_waives() -> None: with pytest.raises(SystemExit) as exc_info: sample.parse_split(None, device) - assert exc_info.value.code == sample.EXIT_WAIVED + assert sample.EXIT_WAIVED == 77 + assert exc_info.value.code == 77 + + +@pytest.mark.agent_authored(model="deepseek/deepseek-v4.1-flash-1017") +def test_green_context_probe_propagates_allocation_failure() -> None: + """A failed allocation is a broken environment, not a refused partition.""" + + class _OutOfMemorySM(_CountApiSM): + def split(self, options, *, dry_run: bool = False): + raise MemoryError + + sample = _load_green_context_sample() + + with pytest.raises(MemoryError): + sample._find_working_split(_OutOfMemorySM(14, 2)) @pytest.mark.agent_authored(model="deepseek/deepseek-v4.1-flash-1017") diff --git a/samples/cuda_core/greenContext/greenContext.py b/samples/cuda_core/greenContext/greenContext.py index bde4aea15df..7e14d4e27e7 100644 --- a/samples/cuda_core/greenContext/greenContext.py +++ b/samples/cuda_core/greenContext/greenContext.py @@ -144,6 +144,10 @@ def _probe_split(sm, critical_count: int) -> tuple[int | None, str | None]: """ try: groups, remainder = sm.split(SMResourceOptions(count=critical_count), dry_run=True) + except MemoryError: + # An allocation failure means the process is in trouble, not that the + # driver refused this partition. + raise except Exception as e: # Any rejection removes this candidate only; the catch is broad because # cuda.core >= 1.0 does not export the rejection type.