Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelog.d/fix-loop-split-split-point-overflow.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
- Fixed loop splitting computing a wrong split point for ``if i > c`` and ``if i <= c``
when ``c`` is the maximum value of the loop index type. The split point ``c + 1``
wrapped around, so every iteration took the wrong branch and the kernel returned
incorrect results.
50 changes: 40 additions & 10 deletions src/cuda/tile/_passes/loop_split.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
from collections import defaultdict
from typing import Optional, Set, Dict, DefaultDict, Mapping, NamedTuple

from cuda.tile._datatype import bool_, is_signed
from cuda.tile._ir.ir import Block, Var, Mapper, IRContext
from cuda.tile._ir.control_flow_ops import Loop, IfElse, EndBranch, Continue
from cuda.tile._ir.core_ops import TypedConst, Assign
from cuda.tile._ir.arithmetic_ops import RawBinaryArithmeticOperation, RawComparisonOperation
from cuda.tile._ir.arithmetic_ops import RawBinaryArithmeticOperation, RawComparisonOperation, \
RawWhereOperation
from cuda.tile._ir.type import TileTy


class _Condition(NamedTuple):
Expand Down Expand Up @@ -97,15 +100,12 @@ def _split_loop(loop: Loop, cond: _Condition, if_ops_to_flatten: Set[IfElse], ne
split_value = cond.rhs
loc = loop.loc
if _NEED_TO_ADJUST_RANGE[cond.cmp]:
one_var = new_block.make_temp_var()
new_block.append(TypedConst(value=1, result_vars=(one_var,), loc=loc))
one_var.set_type(range_dtype)
plus_one_var = new_block.make_temp_var()
new_block.append(RawBinaryArithmeticOperation(fn="add", lhs=split_value, rhs=one_var,
result_vars=(plus_one_var,),
loc=loc))
plus_one_var.set_type(range_dtype)
split_value = plus_one_var
# For `i > c` and `i <= c` the split point is `c + 1`, which wraps around
# when `c` is the maximum value of the induction variable's type. Saturate
# it to that maximum instead. Because `stop <= max`, the first loop then
# covers the whole range and the second loop is empty, which is exactly
# right since `i <= max` always holds and `i > max` never does.
split_value = _saturating_add_one(split_value, range_dtype, new_block, loc)

first_loop_stop = new_block.make_temp_var()
new_block.append(RawBinaryArithmeticOperation(fn="min", lhs=loop.stop, rhs=split_value,
Expand Down Expand Up @@ -134,6 +134,36 @@ def _split_loop(loop: Loop, cond: _Condition, if_ops_to_flatten: Set[IfElse], ne
new_block.append(second_loop)


def _saturating_add_one(value: Var, ty: TileTy, block: Block, loc) -> Var:
dtype = ty.dtype
max_value = (1 << (dtype.bitwidth - 1)) - 1 if is_signed(dtype) else (1 << dtype.bitwidth) - 1

def const(v):
var = block.make_temp_var()
block.append(TypedConst(value=v, result_vars=(var,), loc=loc))
var.set_type(ty)
return var

one_var = const(1)
max_var = const(max_value)

plus_one_var = block.make_temp_var()
block.append(RawBinaryArithmeticOperation(fn="add", lhs=value, rhs=one_var,
result_vars=(plus_one_var,), loc=loc))
plus_one_var.set_type(ty)

is_max_var = block.make_temp_var()
block.append(RawComparisonOperation(fn="eq", lhs=value, rhs=max_var,
result_vars=(is_max_var,), loc=loc))
is_max_var.set_type(TileTy(bool_))

result_var = block.make_temp_var()
block.append(RawWhereOperation(cond=is_max_var, x=max_var, y=plus_one_var,
result_vars=(result_var,), loc=loc))
result_var.set_type(ty)
return result_var


def _clone_loop(loop: Loop, new_start: Var, new_stop: Var, new_step: Var,
initial_vars: tuple[Var, ...], result_vars: tuple[Var, ...],
if_ops_to_flatten: Set[IfElse], branch_to_keep: str, ctx: IRContext) -> Loop:
Expand Down
38 changes: 38 additions & 0 deletions test/test_loop_split.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0

import pytest
import torch

import cuda.tile as ct
Expand Down Expand Up @@ -64,3 +65,40 @@ def test_dynamic_split_boundary():
ct.launch(torch.cuda.current_stream(), (1,), dynamic_split_boundary_kernel, (output, 3))
ref = torch.tensor([0, 1, 2, 30, 40, 50, 60], dtype=torch.int32, device="cuda:0")
assert_equal(output, ref)


@ct.kernel
def split_gt_kernel(output, n: int, c: int):
acc = 0
for i in range(n):
if i > c:
acc = acc + 1000 + i
else:
acc = acc + i
ct.store(output, (0,), acc)


@ct.kernel
def split_le_kernel(output, n: int, c: int):
acc = 0
for i in range(n):
if i <= c:
acc = acc + 1000 + i
else:
acc = acc + i
ct.store(output, (0,), acc)


@pytest.mark.parametrize("kind", ["gt", "le"])
@pytest.mark.parametrize("c", [-1, 0, 1, 3, 4, 2**31 - 2, 2**31 - 1])
def test_split_point_at_dtype_max(kind, c):
# The split point for `i > c` and `i <= c` is `c + 1`, which used to wrap
# around when `c` is the int32 maximum and send every iteration down the
# wrong branch.
n = 4
kernel = split_gt_kernel if kind == "gt" else split_le_kernel
output = torch.zeros((1,), dtype=torch.int32, device="cuda:0")
ct.launch(torch.cuda.current_stream(), (1,), kernel, (output, n, c))
taken = (lambda i: i > c) if kind == "gt" else (lambda i: i <= c)
ref = sum(1000 + i if taken(i) else i for i in range(n))
assert_equal(output, torch.tensor([ref], dtype=torch.int32, device="cuda:0"))
Loading