diff --git a/changelog.d/mma-scaled-mixed-f8-f4.md b/changelog.d/mma-scaled-mixed-f8-f4.md new file mode 100644 index 00000000..c9bab27f --- /dev/null +++ b/changelog.d/mma-scaled-mixed-f8-f4.md @@ -0,0 +1,4 @@ +`ct.mma_scaled` now accepts mixed f8/f4 input dtypes (`float8_e4m3fn` or +`float8_e5m2` with `float4_e2m1fn`) using `float8_e8m0fnu` scales and a scaling +block size of 32 on sm_120 and sm_121. Lowering mixed operands requires a +tileiras version with mxf8f6f4 support. diff --git a/src/cuda/tile/_datatype.py b/src/cuda/tile/_datatype.py index 8ae87972..58add191 100644 --- a/src/cuda/tile/_datatype.py +++ b/src/cuda/tile/_datatype.py @@ -234,19 +234,39 @@ def _resolve_mma_supported_dtype(x_dtype: DType, float8_e5m3fnu: (float32, (16, 32))}, } +# Mixed f8/f4 operands lowering to the mxf8f6f4 block-scaled MMA use UE8M0 +# scales with a scaling block size of 32 on both sides. Only f8 x f4 pairs are +# supported; same-width mixed f8 pairs are not part of mxf8f6f4. +_mma_scaled_mixed_supported_dtypes = { + frozenset((float8_e4m3fn, float4_e2m1fn)): (float8_e8m0fnu, float32, (32,)), + frozenset((float8_e5m2, float4_e2m1fn)): (float8_e8m0fnu, float32, (32,)), +} + def _resolve_mma_scaled_supported_dtype(x_dtype: DType, x_scale_dtype: DType, y_dtype: DType, y_scale_dtype: DType, - acc_dtype: DType): - if x_dtype != y_dtype: - raise TileTypeError( - f"x and y must have the same dtype, got {x_dtype} and {y_dtype}") + acc_dtype: DType) -> Tuple[DType, Tuple[int, ...]]: if x_scale_dtype != y_scale_dtype: raise TileTypeError( f"x_scale and y_scale must have the same dtype, " f"got {x_scale_dtype} and {y_scale_dtype}") + if x_dtype != y_dtype: + mixed = _mma_scaled_mixed_supported_dtypes.get(frozenset((x_dtype, y_dtype))) + if mixed is None: + raise TileTypeError( + f"x and y must have the same dtype, got {x_dtype} and {y_dtype}") + expected_scale, expected_acc, scaling_block_sizes = mixed + if x_scale_dtype != expected_scale: + raise TileTypeError( + f"Unsupported scale dtype {x_scale_dtype} for mixed mma_scaled " + f"input dtypes {x_dtype} and {y_dtype}, expected {expected_scale}") + if acc_dtype != expected_acc: + raise TileTypeError( + f"Unsupported acc dtype {acc_dtype} for mma_scaled, " + f"expected {expected_acc}") + return expected_acc, scaling_block_sizes if x_dtype not in _mma_scaled_supported_dtypes: candidates = ", ".join(str(d) for d in _mma_scaled_supported_dtypes.keys()) raise TileTypeError( @@ -258,19 +278,12 @@ def _resolve_mma_scaled_supported_dtype(x_dtype: DType, raise TileTypeError( f"Unsupported scale dtype {x_scale_dtype} for input dtype {x_dtype}, " f"supported scale dtypes are {candidate_names}") - expected_acc, _ = scale_candidates[x_scale_dtype] + expected_acc, scaling_block_sizes = scale_candidates[x_scale_dtype] if acc_dtype != expected_acc: raise TileTypeError( f"Unsupported acc dtype {acc_dtype} for mma_scaled, " f"expected {expected_acc}") - - -def _get_mma_scaled_scaling_block_sizes(data_dtype, scale_dtype) -> Tuple[int, ...]: - assert data_dtype in _mma_scaled_supported_dtypes - scale_candidates = _mma_scaled_supported_dtypes[data_dtype] - assert scale_dtype in scale_candidates - _, scaling_block_sizes = scale_candidates[scale_dtype] - return scaling_block_sizes + return expected_acc, scaling_block_sizes # =============== Documentation Generator ================ diff --git a/src/cuda/tile/_ir/ops.py b/src/cuda/tile/_ir/ops.py index 1e4608b8..9ae3b834 100644 --- a/src/cuda/tile/_ir/ops.py +++ b/src/cuda/tile/_ir/ops.py @@ -2175,7 +2175,8 @@ def generate_bytecode(self, ctx: BytecodeContext) -> bc.Value: def _verify_scaling_block_size(ty: TileTy, scale_ty: TileTy, k_axis: int, - name: str, scale_name: str): + name: str, scale_name: str, + allowed: Tuple[int, ...]): shape = ty.shape dtype = ty.dtype scale_shape = scale_ty.shape @@ -2186,7 +2187,6 @@ def _verify_scaling_block_size(ty: TileTy, scale_ty: TileTy, k_axis: int, f"{scale_name} shape {scale_shape} is not compatible with {name} shape {shape}. " f"All dimensions except K axis {k_axis} must match") - allowed = datatype._get_mma_scaled_scaling_block_sizes(ty.dtype, scale_ty.dtype) scaling_block_size, rem = divmod(shape[k_axis], scale_shape[k_axis]) if rem != 0 or scaling_block_size not in allowed: raise TileTypeError( @@ -2213,12 +2213,14 @@ def mma_scaled_impl(x: Var, x_scale: Var, y: Var, y_scale: Var, acc: Var) -> Var raise TileTypeError( f'Expect shape of `{name}` to be 2D or 3D, got {shape}') - datatype._resolve_mma_scaled_supported_dtype( + _, scaling_block_sizes = datatype._resolve_mma_scaled_supported_dtype( x_ty.dtype, x_scale_ty.dtype, y_ty.dtype, y_scale_ty.dtype, acc_ty.dtype) - _verify_scaling_block_size(x_ty, x_scale_ty, k_axis=-1, name="x", scale_name="x_scale") - _verify_scaling_block_size(y_ty, y_scale_ty, k_axis=-2, name="y", scale_name="y_scale") + _verify_scaling_block_size(x_ty, x_scale_ty, k_axis=-1, name="x", scale_name="x_scale", + allowed=scaling_block_sizes) + _verify_scaling_block_size(y_ty, y_scale_ty, k_axis=-2, name="y", scale_name="y_scale", + allowed=scaling_block_sizes) x_shape, y_shape, _, output_shape = _matmul_broadcast_shape(x_ty.shape, y_ty.shape) if acc_ty.shape != output_shape: diff --git a/src/cuda/tile/_stub.py b/src/cuda/tile/_stub.py index 15541598..48390026 100644 --- a/src/cuda/tile/_stub.py +++ b/src/cuda/tile/_stub.py @@ -2094,15 +2094,21 @@ def mma_scaled(x, x_scale, y, y_scale, /, acc) -> Tile: Supported datatypes and scaling block sizes: - +----------------------------+------------+---------+--------+ - | Input (x/y) | Scale | Acc/Out | B | - +============================+============+=========+========+ - | f8e4m3fn, f8e5m2 | f8e8m0fnu | f32 | 32 | - +----------------------------+------------+---------+--------+ - | f4e2m1fn | f8e8m0fnu | f32 | 16, 32 | - +----------------------------+------------+---------+--------+ - | f4e2m1fn | f8e4m3fn | f32 | 16 | - +----------------------------+------------+---------+--------+ + +-------------------------------+------------+---------+--------+ + | Input (x/y) | Scale | Acc/Out | B | + +===============================+============+=========+========+ + | f8e4m3fn, f8e5m2 | f8e8m0fnu | f32 | 32 | + +-------------------------------+------------+---------+--------+ + | f4e2m1fn | f8e8m0fnu | f32 | 16, 32 | + +-------------------------------+------------+---------+--------+ + | f4e2m1fn | f8e4m3fn | f32 | 16 | + +-------------------------------+------------+---------+--------+ + | f8e4m3fn or f8e5m2 with | f8e8m0fnu | f32 | 32 | + | f4e2m1fn (mixed f8/f4) | | | | + +-------------------------------+------------+---------+--------+ + + Mixed f8/f4 operands target sm_120 and sm_121 and require tileiras support + for mxf8f6f4. Batch dimensions of x and y are broadcast against each other (same as :func:`mma`). x_scale's batch dimension must match x's batch exactly, diff --git a/test/test_mma_scaled.py b/test/test_mma_scaled.py index ee4cc9ad..a6ffe008 100644 --- a/test/test_mma_scaled.py +++ b/test/test_mma_scaled.py @@ -9,7 +9,7 @@ import cuda.tile as ct from cuda.tile._datatype import float4_e2m1fn, float8_e4m3fn, float8_e5m2, float8_e8m0fnu -from cuda.tile._exception import TileTypeError +from cuda.tile._exception import CompilerExecutionError, TileTypeError from cuda.tile._bytecode.version import BytecodeVersion from conftest import requires_tileiras from util import assert_close, require_blackwell_or_newer, require_rubin_cc107 @@ -78,6 +78,28 @@ def mma_scaled_general_kernel(X, X_scale, Y, Y_scale, Z, ct.store(Z, index=(0, 0), tile=acc) +@ct.kernel +def mma_scaled_mixed_f8_f4_kernel(X, X_scale, Y, Y_scale, Z, + tm: ct.Constant[int], tn: ct.Constant[int], + tk: ct.Constant[int], tks: ct.Constant[int], + x_dtype_id: ct.Constant[int], + x_elem_per_byte: ct.Constant[int], + y_dtype_id: ct.Constant[int], + y_elem_per_byte: ct.Constant[int]): + x_dtype = (float4_e2m1fn, float8_e4m3fn, float8_e5m2)[x_dtype_id] + y_dtype = (float4_e2m1fn, float8_e4m3fn, float8_e5m2)[y_dtype_id] + + x_bytes = ct.load(X, index=(0, 0), shape=(tm, tk // x_elem_per_byte)).reshape((-1,)) + x = ct.unpack_from_bytes(x_bytes, x_dtype).reshape((tm, tk)) + y_bytes = ct.load(Y, index=(0, 0), shape=(tk, tn // y_elem_per_byte)).reshape((-1,)) + y = ct.unpack_from_bytes(y_bytes, y_dtype).reshape((tk, tn)) + x_scale = ct.load(X_scale, index=(0, 0), shape=(tm, tks)) + y_scale = ct.load(Y_scale, index=(0, 0), shape=(tks, tn)) + acc = ct.load(Z, index=(0, 0), shape=(tm, tn)) + acc = ct.mma_scaled(x, x_scale, y, y_scale, acc) + ct.store(Z, index=(0, 0), tile=acc) + + @pytest.mark.parametrize("input_dtype", [f8e4m3fn, f8e5m2], ids=str) def test_mma_scaled_fp8(input_dtype): m, n, k = 16, 16, 64 @@ -182,6 +204,35 @@ def test_mma_scaled_general_f4(scale_dtype, scaling_block_size): assert_close(Z, ref, atol=1e-2, rtol=1e-2) +@pytest.mark.xfail( + raises=CompilerExecutionError, + strict=True, + reason="tileiras does not yet support mixed f8/f4 MMA") +@pytest.mark.parametrize("input_dtype,dtype_id", [ + (f8e4m3fn, 1), (f8e5m2, 2), +], ids=str) +def test_mma_scaled_mixed_f8_f4(input_dtype, dtype_id): + m, n, k = 16, 16, 64 + scaling_block_size = 32 + ks = k // scaling_block_size + + X = torch.randn((m, k), device='cuda:0').to(input_dtype) + X_packed = X.view(torch.uint8) + Y_packed = torch.randint(0, 256, (k, n // 2), dtype=torch.uint8, device='cuda:0') + X_scale = torch.randn((m, ks), device='cuda:0').to(f8e8m0fnu) + Y_scale = torch.randn((ks, n), device='cuda:0').to(f8e8m0fnu) + Z = torch.randn((m, n), dtype=f32, device='cuda:0') + + ref_X = X.to(f32) + ref_Y = _unpack_f4e2m1fn_to_f32(Y_packed, (k, n)) + ref_X_scale = torch.repeat_interleave(X_scale, scaling_block_size, dim=1).to(f32) + ref_Y_scale = torch.repeat_interleave(Y_scale, scaling_block_size, dim=0).to(f32) + ref = (ref_X * ref_X_scale) @ (ref_Y * ref_Y_scale) + Z + ct.launch(torch.cuda.current_stream(), (1,), mma_scaled_mixed_f8_f4_kernel, + (X_packed, X_scale, Y_packed, Y_scale, Z, m, n, k, ks, dtype_id, 1, 0, 2)) + assert_close(Z, ref, atol=1e-2, rtol=1e-2) + + @require_rubin_cc107() @requires_tileiras(BytecodeVersion.V_13_4) @pytest.mark.parametrize("scaling_block_size", [ diff --git a/test/test_mma_scaled_dtype.py b/test/test_mma_scaled_dtype.py new file mode 100644 index 00000000..728307ef --- /dev/null +++ b/test/test_mma_scaled_dtype.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) <2026> NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from cuda.tile._datatype import ( + float4_e2m1fn, float8_e4m3fn, float8_e5m2, float8_e8m0fnu, + float8_e5m3fnu, float16, float32, _resolve_mma_scaled_supported_dtype, +) +from cuda.tile._exception import TileTypeError +from cuda.tile._ir.ops import _verify_scaling_block_size +from cuda.tile._ir.type import TileTy + + +@pytest.mark.parametrize("x_dtype,y_dtype", [ + (float8_e4m3fn, float4_e2m1fn), + (float4_e2m1fn, float8_e4m3fn), + (float8_e5m2, float4_e2m1fn), + (float4_e2m1fn, float8_e5m2), +], ids=str) +def test_mma_scaled_mixed_f8_f4(x_dtype, y_dtype): + acc_dtype, block_sizes = _resolve_mma_scaled_supported_dtype( + x_dtype, float8_e8m0fnu, y_dtype, float8_e8m0fnu, float32) + assert acc_dtype == float32 + assert block_sizes == (32,) + + +@pytest.mark.parametrize("scale_dtype", [ + float8_e4m3fn, + float8_e5m3fnu, +], ids=str) +def test_mma_scaled_mixed_f8_f4_rejects_unsupported_scale(scale_dtype): + with pytest.raises(TileTypeError, match="Unsupported scale dtype"): + _resolve_mma_scaled_supported_dtype( + float8_e4m3fn, scale_dtype, float4_e2m1fn, scale_dtype, float32) + + +def test_mma_scaled_mixed_f8_f4_rejects_mismatched_scales(): + with pytest.raises(TileTypeError, match="x_scale and y_scale must have the same dtype"): + _resolve_mma_scaled_supported_dtype( + float8_e4m3fn, float8_e8m0fnu, float4_e2m1fn, float8_e4m3fn, float32) + + +def test_mma_scaled_mixed_f8_f4_rejects_unsupported_acc(): + with pytest.raises(TileTypeError, match="Unsupported acc dtype"): + _resolve_mma_scaled_supported_dtype( + float8_e4m3fn, float8_e8m0fnu, float4_e2m1fn, float8_e8m0fnu, float16) + + +def test_mma_scaled_same_width_f8_mix_still_rejected(): + with pytest.raises(TileTypeError, match="x and y must have the same dtype"): + _resolve_mma_scaled_supported_dtype( + float8_e4m3fn, float8_e8m0fnu, float8_e5m2, float8_e8m0fnu, float32) + + +def test_mma_scaled_mixed_f8_f4_requires_block_size_32(): + x = TileTy(float8_e4m3fn, (16, 64)) + x_scale = TileTy(float8_e8m0fnu, (16, 2)) + y = TileTy(float4_e2m1fn, (64, 16)) + y_scale = TileTy(float8_e8m0fnu, (2, 16)) + + _verify_scaling_block_size( + x, x_scale, k_axis=-1, name="x", scale_name="x_scale", allowed=(32,)) + _verify_scaling_block_size( + y, y_scale, k_axis=-2, name="y", scale_name="y_scale", allowed=(32,)) + + x_scale_16 = TileTy(float8_e8m0fnu, (16, 4)) + y_scale_16 = TileTy(float8_e8m0fnu, (4, 16)) + with pytest.raises(TileTypeError, match="scaling block size B = K // K_s"): + _verify_scaling_block_size( + x, x_scale_16, k_axis=-1, name="x", scale_name="x_scale", allowed=(32,)) + with pytest.raises(TileTypeError, match="scaling block size B = K // K_s"): + _verify_scaling_block_size( + y, y_scale_16, k_axis=-2, name="y", scale_name="y_scale", allowed=(32,)) + + +def test_mma_scaled_same_dtype_f4_still_allows_block_size_16(): + _, block_sizes = _resolve_mma_scaled_supported_dtype( + float4_e2m1fn, float8_e8m0fnu, + float4_e2m1fn, float8_e8m0fnu, + float32) + assert block_sizes == (16, 32) + + x = TileTy(float4_e2m1fn, (16, 64)) + x_scale = TileTy(float8_e8m0fnu, (16, 4)) + _verify_scaling_block_size( + x, x_scale, k_axis=-1, name="x", scale_name="x_scale", allowed=block_sizes)