Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelog.d/mma-scaled-mixed-f8-f4.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
`ct.mma_scaled` now accepts mixed f8/f4 input dtypes (`float8_e4m3fn` or
`float8_e5m2` with `float4_e2m1fn`) using `float8_e8m0fnu` scales and a scaling
block size of 32 on sm_120 and sm_121. Lowering mixed operands requires a
tileiras version with mxf8f6f4 support.
39 changes: 26 additions & 13 deletions src/cuda/tile/_datatype.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,19 +234,39 @@ def _resolve_mma_supported_dtype(x_dtype: DType,
float8_e5m3fnu: (float32, (16, 32))},
}

# Mixed f8/f4 operands lowering to the mxf8f6f4 block-scaled MMA use UE8M0
# scales with a scaling block size of 32 on both sides. Only f8 x f4 pairs are
# supported; same-width mixed f8 pairs are not part of mxf8f6f4.
_mma_scaled_mixed_supported_dtypes = {
frozenset((float8_e4m3fn, float4_e2m1fn)): (float8_e8m0fnu, float32, (32,)),
frozenset((float8_e5m2, float4_e2m1fn)): (float8_e8m0fnu, float32, (32,)),
}


def _resolve_mma_scaled_supported_dtype(x_dtype: DType,
x_scale_dtype: DType,
y_dtype: DType,
y_scale_dtype: DType,
acc_dtype: DType):
if x_dtype != y_dtype:
raise TileTypeError(
f"x and y must have the same dtype, got {x_dtype} and {y_dtype}")
acc_dtype: DType) -> Tuple[DType, Tuple[int, ...]]:
if x_scale_dtype != y_scale_dtype:
raise TileTypeError(
f"x_scale and y_scale must have the same dtype, "
f"got {x_scale_dtype} and {y_scale_dtype}")
if x_dtype != y_dtype:
mixed = _mma_scaled_mixed_supported_dtypes.get(frozenset((x_dtype, y_dtype)))
if mixed is None:
raise TileTypeError(
f"x and y must have the same dtype, got {x_dtype} and {y_dtype}")
expected_scale, expected_acc, scaling_block_sizes = mixed
if x_scale_dtype != expected_scale:
raise TileTypeError(
f"Unsupported scale dtype {x_scale_dtype} for mixed mma_scaled "
f"input dtypes {x_dtype} and {y_dtype}, expected {expected_scale}")
if acc_dtype != expected_acc:
raise TileTypeError(
f"Unsupported acc dtype {acc_dtype} for mma_scaled, "
f"expected {expected_acc}")
return expected_acc, scaling_block_sizes
if x_dtype not in _mma_scaled_supported_dtypes:
candidates = ", ".join(str(d) for d in _mma_scaled_supported_dtypes.keys())
raise TileTypeError(
Expand All @@ -258,19 +278,12 @@ def _resolve_mma_scaled_supported_dtype(x_dtype: DType,
raise TileTypeError(
f"Unsupported scale dtype {x_scale_dtype} for input dtype {x_dtype}, "
f"supported scale dtypes are {candidate_names}")
expected_acc, _ = scale_candidates[x_scale_dtype]
expected_acc, scaling_block_sizes = scale_candidates[x_scale_dtype]
if acc_dtype != expected_acc:
raise TileTypeError(
f"Unsupported acc dtype {acc_dtype} for mma_scaled, "
f"expected {expected_acc}")


def _get_mma_scaled_scaling_block_sizes(data_dtype, scale_dtype) -> Tuple[int, ...]:
assert data_dtype in _mma_scaled_supported_dtypes
scale_candidates = _mma_scaled_supported_dtypes[data_dtype]
assert scale_dtype in scale_candidates
_, scaling_block_sizes = scale_candidates[scale_dtype]
return scaling_block_sizes
return expected_acc, scaling_block_sizes


# =============== Documentation Generator ================
Expand Down
12 changes: 7 additions & 5 deletions src/cuda/tile/_ir/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -2175,7 +2175,8 @@ def generate_bytecode(self, ctx: BytecodeContext) -> bc.Value:


def _verify_scaling_block_size(ty: TileTy, scale_ty: TileTy, k_axis: int,
name: str, scale_name: str):
name: str, scale_name: str,
allowed: Tuple[int, ...]):
shape = ty.shape
dtype = ty.dtype
scale_shape = scale_ty.shape
Expand All @@ -2186,7 +2187,6 @@ def _verify_scaling_block_size(ty: TileTy, scale_ty: TileTy, k_axis: int,
f"{scale_name} shape {scale_shape} is not compatible with {name} shape {shape}. "
f"All dimensions except K axis {k_axis} must match")

allowed = datatype._get_mma_scaled_scaling_block_sizes(ty.dtype, scale_ty.dtype)
scaling_block_size, rem = divmod(shape[k_axis], scale_shape[k_axis])
if rem != 0 or scaling_block_size not in allowed:
raise TileTypeError(
Expand All @@ -2213,12 +2213,14 @@ def mma_scaled_impl(x: Var, x_scale: Var, y: Var, y_scale: Var, acc: Var) -> Var
raise TileTypeError(
f'Expect shape of `{name}` to be 2D or 3D, got {shape}')

datatype._resolve_mma_scaled_supported_dtype(
_, scaling_block_sizes = datatype._resolve_mma_scaled_supported_dtype(
x_ty.dtype, x_scale_ty.dtype,
y_ty.dtype, y_scale_ty.dtype,
acc_ty.dtype)
_verify_scaling_block_size(x_ty, x_scale_ty, k_axis=-1, name="x", scale_name="x_scale")
_verify_scaling_block_size(y_ty, y_scale_ty, k_axis=-2, name="y", scale_name="y_scale")
_verify_scaling_block_size(x_ty, x_scale_ty, k_axis=-1, name="x", scale_name="x_scale",
allowed=scaling_block_sizes)
_verify_scaling_block_size(y_ty, y_scale_ty, k_axis=-2, name="y", scale_name="y_scale",
allowed=scaling_block_sizes)

x_shape, y_shape, _, output_shape = _matmul_broadcast_shape(x_ty.shape, y_ty.shape)
if acc_ty.shape != output_shape:
Expand Down
24 changes: 15 additions & 9 deletions src/cuda/tile/_stub.py
Original file line number Diff line number Diff line change
Expand Up @@ -2094,15 +2094,21 @@ def mma_scaled(x, x_scale, y, y_scale, /, acc) -> Tile:

Supported datatypes and scaling block sizes:

+----------------------------+------------+---------+--------+
| Input (x/y) | Scale | Acc/Out | B |
+============================+============+=========+========+
| f8e4m3fn, f8e5m2 | f8e8m0fnu | f32 | 32 |
+----------------------------+------------+---------+--------+
| f4e2m1fn | f8e8m0fnu | f32 | 16, 32 |
+----------------------------+------------+---------+--------+
| f4e2m1fn | f8e4m3fn | f32 | 16 |
+----------------------------+------------+---------+--------+
+-------------------------------+------------+---------+--------+
| Input (x/y) | Scale | Acc/Out | B |
+===============================+============+=========+========+
| f8e4m3fn, f8e5m2 | f8e8m0fnu | f32 | 32 |
+-------------------------------+------------+---------+--------+
| f4e2m1fn | f8e8m0fnu | f32 | 16, 32 |
+-------------------------------+------------+---------+--------+
| f4e2m1fn | f8e4m3fn | f32 | 16 |
+-------------------------------+------------+---------+--------+
| f8e4m3fn or f8e5m2 with | f8e8m0fnu | f32 | 32 |
| f4e2m1fn (mixed f8/f4) | | | |
+-------------------------------+------------+---------+--------+

Mixed f8/f4 operands target sm_120 and sm_121 and require tileiras support
for mxf8f6f4.

Batch dimensions of x and y are broadcast against each other (same as
:func:`mma`). x_scale's batch dimension must match x's batch exactly,
Expand Down
53 changes: 52 additions & 1 deletion test/test_mma_scaled.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import cuda.tile as ct
from cuda.tile._datatype import float4_e2m1fn, float8_e4m3fn, float8_e5m2, float8_e8m0fnu
from cuda.tile._exception import TileTypeError
from cuda.tile._exception import CompilerExecutionError, TileTypeError
from cuda.tile._bytecode.version import BytecodeVersion
from conftest import requires_tileiras
from util import assert_close, require_blackwell_or_newer, require_rubin_cc107
Expand Down Expand Up @@ -78,6 +78,28 @@ def mma_scaled_general_kernel(X, X_scale, Y, Y_scale, Z,
ct.store(Z, index=(0, 0), tile=acc)


@ct.kernel
def mma_scaled_mixed_f8_f4_kernel(X, X_scale, Y, Y_scale, Z,
tm: ct.Constant[int], tn: ct.Constant[int],
tk: ct.Constant[int], tks: ct.Constant[int],
x_dtype_id: ct.Constant[int],
x_elem_per_byte: ct.Constant[int],
y_dtype_id: ct.Constant[int],
y_elem_per_byte: ct.Constant[int]):
x_dtype = (float4_e2m1fn, float8_e4m3fn, float8_e5m2)[x_dtype_id]
y_dtype = (float4_e2m1fn, float8_e4m3fn, float8_e5m2)[y_dtype_id]

x_bytes = ct.load(X, index=(0, 0), shape=(tm, tk // x_elem_per_byte)).reshape((-1,))
x = ct.unpack_from_bytes(x_bytes, x_dtype).reshape((tm, tk))
y_bytes = ct.load(Y, index=(0, 0), shape=(tk, tn // y_elem_per_byte)).reshape((-1,))
y = ct.unpack_from_bytes(y_bytes, y_dtype).reshape((tk, tn))
x_scale = ct.load(X_scale, index=(0, 0), shape=(tm, tks))
y_scale = ct.load(Y_scale, index=(0, 0), shape=(tks, tn))
acc = ct.load(Z, index=(0, 0), shape=(tm, tn))
acc = ct.mma_scaled(x, x_scale, y, y_scale, acc)
ct.store(Z, index=(0, 0), tile=acc)


@pytest.mark.parametrize("input_dtype", [f8e4m3fn, f8e5m2], ids=str)
def test_mma_scaled_fp8(input_dtype):
m, n, k = 16, 16, 64
Expand Down Expand Up @@ -182,6 +204,35 @@ def test_mma_scaled_general_f4(scale_dtype, scaling_block_size):
assert_close(Z, ref, atol=1e-2, rtol=1e-2)


@pytest.mark.xfail(
raises=CompilerExecutionError,
strict=True,
reason="tileiras does not yet support mixed f8/f4 MMA")
@pytest.mark.parametrize("input_dtype,dtype_id", [
(f8e4m3fn, 1), (f8e5m2, 2),
], ids=str)
def test_mma_scaled_mixed_f8_f4(input_dtype, dtype_id):
m, n, k = 16, 16, 64
scaling_block_size = 32
ks = k // scaling_block_size

X = torch.randn((m, k), device='cuda:0').to(input_dtype)
X_packed = X.view(torch.uint8)
Y_packed = torch.randint(0, 256, (k, n // 2), dtype=torch.uint8, device='cuda:0')
X_scale = torch.randn((m, ks), device='cuda:0').to(f8e8m0fnu)
Y_scale = torch.randn((ks, n), device='cuda:0').to(f8e8m0fnu)
Z = torch.randn((m, n), dtype=f32, device='cuda:0')

ref_X = X.to(f32)
ref_Y = _unpack_f4e2m1fn_to_f32(Y_packed, (k, n))
ref_X_scale = torch.repeat_interleave(X_scale, scaling_block_size, dim=1).to(f32)
ref_Y_scale = torch.repeat_interleave(Y_scale, scaling_block_size, dim=0).to(f32)
ref = (ref_X * ref_X_scale) @ (ref_Y * ref_Y_scale) + Z
ct.launch(torch.cuda.current_stream(), (1,), mma_scaled_mixed_f8_f4_kernel,
(X_packed, X_scale, Y_packed, Y_scale, Z, m, n, k, ks, dtype_id, 1, 0, 2))
assert_close(Z, ref, atol=1e-2, rtol=1e-2)


@require_rubin_cc107()
@requires_tileiras(BytecodeVersion.V_13_4)
@pytest.mark.parametrize("scaling_block_size", [
Expand Down
88 changes: 88 additions & 0 deletions test/test_mma_scaled_dtype.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# SPDX-FileCopyrightText: Copyright (c) <2026> NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

import pytest

from cuda.tile._datatype import (
float4_e2m1fn, float8_e4m3fn, float8_e5m2, float8_e8m0fnu,
float8_e5m3fnu, float16, float32, _resolve_mma_scaled_supported_dtype,
)
from cuda.tile._exception import TileTypeError
from cuda.tile._ir.ops import _verify_scaling_block_size
from cuda.tile._ir.type import TileTy


@pytest.mark.parametrize("x_dtype,y_dtype", [
(float8_e4m3fn, float4_e2m1fn),
(float4_e2m1fn, float8_e4m3fn),
(float8_e5m2, float4_e2m1fn),
(float4_e2m1fn, float8_e5m2),
], ids=str)
def test_mma_scaled_mixed_f8_f4(x_dtype, y_dtype):
acc_dtype, block_sizes = _resolve_mma_scaled_supported_dtype(
x_dtype, float8_e8m0fnu, y_dtype, float8_e8m0fnu, float32)
assert acc_dtype == float32
assert block_sizes == (32,)


@pytest.mark.parametrize("scale_dtype", [
float8_e4m3fn,
float8_e5m3fnu,
], ids=str)
def test_mma_scaled_mixed_f8_f4_rejects_unsupported_scale(scale_dtype):
with pytest.raises(TileTypeError, match="Unsupported scale dtype"):
_resolve_mma_scaled_supported_dtype(
float8_e4m3fn, scale_dtype, float4_e2m1fn, scale_dtype, float32)


def test_mma_scaled_mixed_f8_f4_rejects_mismatched_scales():
with pytest.raises(TileTypeError, match="x_scale and y_scale must have the same dtype"):
_resolve_mma_scaled_supported_dtype(
float8_e4m3fn, float8_e8m0fnu, float4_e2m1fn, float8_e4m3fn, float32)


def test_mma_scaled_mixed_f8_f4_rejects_unsupported_acc():
with pytest.raises(TileTypeError, match="Unsupported acc dtype"):
_resolve_mma_scaled_supported_dtype(
float8_e4m3fn, float8_e8m0fnu, float4_e2m1fn, float8_e8m0fnu, float16)


def test_mma_scaled_same_width_f8_mix_still_rejected():
with pytest.raises(TileTypeError, match="x and y must have the same dtype"):
_resolve_mma_scaled_supported_dtype(
float8_e4m3fn, float8_e8m0fnu, float8_e5m2, float8_e8m0fnu, float32)


def test_mma_scaled_mixed_f8_f4_requires_block_size_32():
x = TileTy(float8_e4m3fn, (16, 64))
x_scale = TileTy(float8_e8m0fnu, (16, 2))
y = TileTy(float4_e2m1fn, (64, 16))
y_scale = TileTy(float8_e8m0fnu, (2, 16))

_verify_scaling_block_size(
x, x_scale, k_axis=-1, name="x", scale_name="x_scale", allowed=(32,))
_verify_scaling_block_size(
y, y_scale, k_axis=-2, name="y", scale_name="y_scale", allowed=(32,))

x_scale_16 = TileTy(float8_e8m0fnu, (16, 4))
y_scale_16 = TileTy(float8_e8m0fnu, (4, 16))
with pytest.raises(TileTypeError, match="scaling block size B = K // K_s"):
_verify_scaling_block_size(
x, x_scale_16, k_axis=-1, name="x", scale_name="x_scale", allowed=(32,))
with pytest.raises(TileTypeError, match="scaling block size B = K // K_s"):
_verify_scaling_block_size(
y, y_scale_16, k_axis=-2, name="y", scale_name="y_scale", allowed=(32,))


def test_mma_scaled_same_dtype_f4_still_allows_block_size_16():
_, block_sizes = _resolve_mma_scaled_supported_dtype(
float4_e2m1fn, float8_e8m0fnu,
float4_e2m1fn, float8_e8m0fnu,
float32)
assert block_sizes == (16, 32)

x = TileTy(float4_e2m1fn, (16, 64))
x_scale = TileTy(float8_e8m0fnu, (16, 4))
_verify_scaling_block_size(
x, x_scale, k_axis=-1, name="x", scale_name="x_scale", allowed=block_sizes)
Loading