From ca6dd495e9d43bfc475f0f50db89a61920979f03 Mon Sep 17 00:00:00 2001 From: vxnuaj Date: Wed, 18 Nov 2026 20:14:44 -0800 Subject: [PATCH] Saturate the loop split point when the bound is the index type maximum Signed-off-by: vxnuaj --- .../fix-loop-split-split-point-overflow.md | 4 ++ src/cuda/tile/_passes/loop_split.py | 50 +++++++++++++++---- test/test_loop_split.py | 38 ++++++++++++++ 3 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fix-loop-split-split-point-overflow.md diff --git a/changelog.d/fix-loop-split-split-point-overflow.md b/changelog.d/fix-loop-split-split-point-overflow.md new file mode 100644 index 00000000..9403cca7 --- /dev/null +++ b/changelog.d/fix-loop-split-split-point-overflow.md @@ -0,0 +1,4 @@ +- Fixed loop splitting computing a wrong split point for ``if i > c`` and ``if i <= c`` + when ``c`` is the maximum value of the loop index type. The split point ``c + 1`` + wrapped around, so every iteration took the wrong branch and the kernel returned + incorrect results. diff --git a/src/cuda/tile/_passes/loop_split.py b/src/cuda/tile/_passes/loop_split.py index c9400741..8f3847b4 100644 --- a/src/cuda/tile/_passes/loop_split.py +++ b/src/cuda/tile/_passes/loop_split.py @@ -5,10 +5,13 @@ from collections import defaultdict from typing import Optional, Set, Dict, DefaultDict, Mapping, NamedTuple +from cuda.tile._datatype import bool_, is_signed from cuda.tile._ir.ir import Block, Var, Mapper, IRContext from cuda.tile._ir.control_flow_ops import Loop, IfElse, EndBranch, Continue from cuda.tile._ir.core_ops import TypedConst, Assign -from cuda.tile._ir.arithmetic_ops import RawBinaryArithmeticOperation, RawComparisonOperation +from cuda.tile._ir.arithmetic_ops import RawBinaryArithmeticOperation, RawComparisonOperation, \ + RawWhereOperation +from cuda.tile._ir.type import TileTy class _Condition(NamedTuple): @@ -97,15 +100,12 @@ def _split_loop(loop: Loop, cond: _Condition, if_ops_to_flatten: Set[IfElse], ne split_value = cond.rhs loc = loop.loc if _NEED_TO_ADJUST_RANGE[cond.cmp]: - one_var = new_block.make_temp_var() - new_block.append(TypedConst(value=1, result_vars=(one_var,), loc=loc)) - one_var.set_type(range_dtype) - plus_one_var = new_block.make_temp_var() - new_block.append(RawBinaryArithmeticOperation(fn="add", lhs=split_value, rhs=one_var, - result_vars=(plus_one_var,), - loc=loc)) - plus_one_var.set_type(range_dtype) - split_value = plus_one_var + # For `i > c` and `i <= c` the split point is `c + 1`, which wraps around + # when `c` is the maximum value of the induction variable's type. Saturate + # it to that maximum instead. Because `stop <= max`, the first loop then + # covers the whole range and the second loop is empty, which is exactly + # right since `i <= max` always holds and `i > max` never does. + split_value = _saturating_add_one(split_value, range_dtype, new_block, loc) first_loop_stop = new_block.make_temp_var() new_block.append(RawBinaryArithmeticOperation(fn="min", lhs=loop.stop, rhs=split_value, @@ -134,6 +134,36 @@ def _split_loop(loop: Loop, cond: _Condition, if_ops_to_flatten: Set[IfElse], ne new_block.append(second_loop) +def _saturating_add_one(value: Var, ty: TileTy, block: Block, loc) -> Var: + dtype = ty.dtype + max_value = (1 << (dtype.bitwidth - 1)) - 1 if is_signed(dtype) else (1 << dtype.bitwidth) - 1 + + def const(v): + var = block.make_temp_var() + block.append(TypedConst(value=v, result_vars=(var,), loc=loc)) + var.set_type(ty) + return var + + one_var = const(1) + max_var = const(max_value) + + plus_one_var = block.make_temp_var() + block.append(RawBinaryArithmeticOperation(fn="add", lhs=value, rhs=one_var, + result_vars=(plus_one_var,), loc=loc)) + plus_one_var.set_type(ty) + + is_max_var = block.make_temp_var() + block.append(RawComparisonOperation(fn="eq", lhs=value, rhs=max_var, + result_vars=(is_max_var,), loc=loc)) + is_max_var.set_type(TileTy(bool_)) + + result_var = block.make_temp_var() + block.append(RawWhereOperation(cond=is_max_var, x=max_var, y=plus_one_var, + result_vars=(result_var,), loc=loc)) + result_var.set_type(ty) + return result_var + + def _clone_loop(loop: Loop, new_start: Var, new_stop: Var, new_step: Var, initial_vars: tuple[Var, ...], result_vars: tuple[Var, ...], if_ops_to_flatten: Set[IfElse], branch_to_keep: str, ctx: IRContext) -> Loop: diff --git a/test/test_loop_split.py b/test/test_loop_split.py index 4ce9fa4c..5df1212e 100644 --- a/test/test_loop_split.py +++ b/test/test_loop_split.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +import pytest import torch import cuda.tile as ct @@ -64,3 +65,40 @@ def test_dynamic_split_boundary(): ct.launch(torch.cuda.current_stream(), (1,), dynamic_split_boundary_kernel, (output, 3)) ref = torch.tensor([0, 1, 2, 30, 40, 50, 60], dtype=torch.int32, device="cuda:0") assert_equal(output, ref) + + +@ct.kernel +def split_gt_kernel(output, n: int, c: int): + acc = 0 + for i in range(n): + if i > c: + acc = acc + 1000 + i + else: + acc = acc + i + ct.store(output, (0,), acc) + + +@ct.kernel +def split_le_kernel(output, n: int, c: int): + acc = 0 + for i in range(n): + if i <= c: + acc = acc + 1000 + i + else: + acc = acc + i + ct.store(output, (0,), acc) + + +@pytest.mark.parametrize("kind", ["gt", "le"]) +@pytest.mark.parametrize("c", [-1, 0, 1, 3, 4, 2**31 - 2, 2**31 - 1]) +def test_split_point_at_dtype_max(kind, c): + # The split point for `i > c` and `i <= c` is `c + 1`, which used to wrap + # around when `c` is the int32 maximum and send every iteration down the + # wrong branch. + n = 4 + kernel = split_gt_kernel if kind == "gt" else split_le_kernel + output = torch.zeros((1,), dtype=torch.int32, device="cuda:0") + ct.launch(torch.cuda.current_stream(), (1,), kernel, (output, n, c)) + taken = (lambda i: i > c) if kind == "gt" else (lambda i: i <= c) + ref = sum(1000 + i if taken(i) else i for i in range(n)) + assert_equal(output, torch.tensor([ref], dtype=torch.int32, device="cuda:0"))