Skip to content

Commit 9b3d724

Browse files
committed
[bugfix]: fix FlashInfer backend selection and validation for Wan (hao-ai-lab#1799)
1 parent 6b8afb6 commit 9b3d724

6 files changed

Lines changed: 181 additions & 4 deletions

File tree

‎fastvideo/attention/backends/flashinfer.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ class FlashInferMetadata(AttentionMetadata):
3636

3737
class FlashInferMetadataBuilder(AttentionMetadataBuilder):
3838

39+
def __init__(self) -> None:
40+
pass
41+
3942
def prepare(self):
4043
pass
4144

@@ -81,6 +84,9 @@ def _mask_for_sample(attn_mask: torch.Tensor, sample: int, query_len: int, key_l
8184
if mask.dim() == 3:
8285
mask = mask[sample]
8386
elif mask.dim() == 4:
87+
if mask.shape[1] != 1:
88+
raise ValueError("FLASHINFER does not support per-head attention masks; "
89+
f"expected [B, 1, Q, K], got {attn_mask.shape}")
8490
mask = mask[sample, 0]
8591
else:
8692
raise ValueError(f"Unsupported FLASHINFER attention mask shape: {attn_mask.shape}")

‎fastvideo/models/wan/causal_transformer.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import fastvideo.envs as envs
1919
from fastvideo.attention import (DistributedAttention, LocalAttention)
20+
from fastvideo.attention.selector import effective_attention_backend
2021
from fastvideo.models.wan.config import WanVideoConfig
2122
from fastvideo.distributed.communication_op import (sequence_model_parallel_all_gather)
2223
from fastvideo.distributed.parallel_state import (get_sp_parallel_rank, get_sp_world_size)
@@ -378,12 +379,18 @@ def forward(
378379
class CausalWanTransformer3DModel(BaseDiT):
379380
_fsdp_shard_conditions = WanVideoConfig()._fsdp_shard_conditions
380381
_compile_conditions = WanVideoConfig()._compile_conditions
381-
_supported_attention_backends = WanVideoConfig()._supported_attention_backends
382+
_supported_attention_backends = tuple(
383+
backend for backend in WanVideoConfig()._supported_attention_backends
384+
if backend != AttentionBackendEnum.FLASHINFER)
382385
param_names_mapping = WanVideoConfig().param_names_mapping
383386
reverse_param_names_mapping = WanVideoConfig().reverse_param_names_mapping
384387
lora_param_names_mapping = WanVideoConfig().lora_param_names_mapping
385388

386389
def __init__(self, config: WanVideoConfig, hf_config: dict[str, Any]) -> None:
390+
if effective_attention_backend(config) == AttentionBackendEnum.FLASHINFER:
391+
raise NotImplementedError(
392+
"FLASHINFER is supported only for dense Wan attention; causal Wan is not supported. "
393+
"Use FLASH_ATTN or TORCH_SDPA for causal Wan.")
387394
super().__init__(config=config, hf_config=hf_config)
388395

389396
inner_dim = config.num_attention_heads * config.attention_head_dim

‎fastvideo/models/wan/config.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
from typing import Literal
66

77
from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig
8+
from fastvideo.platforms import AttentionBackendEnum
9+
10+
_WAN_ATTENTION_BACKENDS = DiTArchConfig._supported_attention_backends + (AttentionBackendEnum.FLASHINFER, )
811

912

1013
def is_blocks(n: str, m) -> bool:
@@ -13,6 +16,7 @@ def is_blocks(n: str, m) -> bool:
1316

1417
@dataclass
1518
class WanVideoArchConfig(DiTArchConfig):
19+
_supported_attention_backends: tuple[AttentionBackendEnum, ...] = _WAN_ATTENTION_BACKENDS
1620
_fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks])
1721

1822
param_names_mapping: dict = field(

‎fastvideo/models/wan/transformer.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,8 @@ def __init__(self,
296296
causal=False,
297297
supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN,
298298
AttentionBackendEnum.TORCH_SDPA,
299-
AttentionBackendEnum.ATTN_QAT_INFER))
299+
AttentionBackendEnum.ATTN_QAT_INFER,
300+
AttentionBackendEnum.FLASHINFER))
300301

301302
def forward(self, x: torch.Tensor, context: torch.Tensor, context_lens: int):
302303
r"""

‎fastvideo/tests/attention/test_flashinfer_backend.py‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ def _sdpa_reference(query: torch.Tensor,
7878
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
7979
@pytest.mark.parametrize("head_size", [64, 128, 256])
8080
def test_flashinfer_real_cuda_kernel_matches_sdpa(dtype: torch.dtype, head_size: int) -> None:
81-
"""Launch the real single-GPU FlashInfer kernel for every supported head size."""
8281
_require_flashinfer_cuda()
8382
torch.manual_seed(0)
8483
device = torch.device("cuda", torch.cuda.current_device())
@@ -98,7 +97,6 @@ def test_flashinfer_real_cuda_kernel_matches_sdpa(dtype: torch.dtype, head_size:
9897

9998
@pytest.mark.parametrize("mode", ["causal", "cross_gqa", "causal_padding"])
10099
def test_flashinfer_real_cuda_kernel_attention_modes(mode: str) -> None:
101-
"""Exercise native causal, GQA/cross-attention, and combined custom masks."""
102100
_require_flashinfer_cuda()
103101
torch.manual_seed(1)
104102
device = torch.device("cuda", torch.cuda.current_device())
@@ -138,3 +136,117 @@ def test_flashinfer_real_cuda_kernel_attention_modes(mode: str) -> None:
138136

139137
torch.cuda.synchronize(device)
140138
torch.testing.assert_close(actual, expected, atol=3e-2, rtol=3e-2)
139+
140+
141+
def test_metadata_builder_instantiates_through_backend_api() -> None:
142+
from fastvideo.attention.backends.abstract import AttentionMetadataBuilder
143+
from fastvideo.attention.backends.flashinfer import FlashInferBackend
144+
builder = FlashInferBackend.get_builder_cls()()
145+
assert isinstance(builder, AttentionMetadataBuilder)
146+
builder.prepare()
147+
mask = torch.ones(2, 3, dtype=torch.bool)
148+
metadata = builder.build(current_timestep=7, attn_mask=mask)
149+
assert metadata.current_timestep == 7
150+
assert metadata.attn_mask is mask
151+
152+
153+
@pytest.mark.parametrize("shared_head", [False, True])
154+
@pytest.mark.parametrize("additive", [False, True])
155+
def test_shared_masks_preserve_each_sample(shared_head, additive) -> None:
156+
mask = torch.tensor([[[True, False, True], [False, True, True]],
157+
[[False, True, False], [True, True, False]]])
158+
source = torch.where(mask, 0.0, float("-inf")) if additive else mask
159+
if shared_head:
160+
source = source[:, None]
161+
for sample in range(2):
162+
torch.testing.assert_close(_mask_for_sample(source, sample, 2, 3), mask[sample])
163+
164+
165+
def test_per_head_mask_rejected_before_kernel(monkeypatch) -> None:
166+
prefill = types.ModuleType("flashinfer.prefill")
167+
def unexpected_kernel(*args, **kwargs):
168+
pytest.fail("per-head mask reached FlashInfer kernel")
169+
prefill.single_prefill_with_kv_cache = unexpected_kernel
170+
monkeypatch.setitem(sys.modules, "flashinfer.prefill", prefill)
171+
query = torch.randn(2, 3, 4, 64, dtype=torch.bfloat16)
172+
metadata = FlashInferMetadata(current_timestep=0, attn_mask=torch.ones(2, 4, 3, 3, dtype=torch.bool))
173+
impl = FlashInferImpl(num_heads=4, head_size=64, causal=False, softmax_scale=0.125)
174+
with pytest.raises(ValueError, match="per-head"):
175+
impl.forward(query, query, query, metadata)
176+
177+
178+
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
179+
@pytest.mark.parametrize("shared_head", [False, True])
180+
def test_shared_mask_cuda_matches_flash_attention(dtype, shared_head) -> None:
181+
_require_flashinfer_cuda()
182+
flash_attn = pytest.importorskip("flash_attn")
183+
torch.manual_seed(19)
184+
q = torch.randn(2, 16, 2, 64, device="cuda", dtype=dtype)
185+
k, v = torch.randn_like(q), torch.randn_like(q)
186+
mask = torch.zeros(2, 16, 16, device="cuda", dtype=torch.bool)
187+
mask[0, :, :11] = True
188+
mask[1, :, :7] = True
189+
metadata = FlashInferMetadata(current_timestep=0, attn_mask=mask[:, None] if shared_head else mask)
190+
impl = FlashInferImpl(num_heads=2, head_size=64, causal=False, softmax_scale=0.2)
191+
actual = impl.forward(q, k, v, metadata)
192+
expected = torch.cat([
193+
flash_attn.flash_attn_func(q[i:i + 1], k[i:i + 1, :length], v[i:i + 1, :length], softmax_scale=0.2)
194+
for i, length in enumerate([11, 7])
195+
])
196+
tolerance = 3e-2 if dtype == torch.bfloat16 else 1e-2
197+
torch.testing.assert_close(actual, expected, atol=tolerance, rtol=tolerance)
198+
199+
200+
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
201+
@pytest.mark.parametrize("i2v", [False, True])
202+
def test_wan_attention_executes_flashinfer(monkeypatch, request, dtype, i2v) -> None:
203+
_require_flashinfer_cuda()
204+
request.getfixturevalue("distributed_setup")
205+
import flashinfer.prefill as prefill
206+
import fastvideo.envs as envs
207+
from flash_attn import flash_attn_func
208+
from fastvideo.forward_context import set_forward_context
209+
from fastvideo.models.wan.config import WanVideoConfig
210+
from fastvideo.models.wan.transformer import WanTransformerBlock
211+
from fastvideo.platforms import AttentionBackendEnum
212+
213+
real_kernel = prefill.single_prefill_with_kv_cache
214+
calls = []
215+
tolerance = 3e-2 if dtype == torch.bfloat16 else 1e-2
216+
def checked_kernel(q, k, v, **kwargs):
217+
actual = real_kernel(q, k, v, **kwargs)
218+
expected = flash_attn_func(q[None], k[None], v[None],
219+
softmax_scale=kwargs["sm_scale"], causal=kwargs["causal"])[0]
220+
torch.testing.assert_close(actual, expected, atol=tolerance, rtol=tolerance)
221+
calls.append((q.shape, k.shape))
222+
return actual
223+
monkeypatch.setattr(prefill, "single_prefill_with_kv_cache", checked_kernel)
224+
with envs.FASTVIDEO_ATTENTION_BACKEND.override("FLASHINFER"), torch.device("cuda"):
225+
block = WanTransformerBlock(
226+
dim=256, ffn_dim=512, num_heads=2, cross_attn_norm=True,
227+
added_kv_proj_dim=256 if i2v else None,
228+
supported_attention_backends=WanVideoConfig()._supported_attention_backends,
229+
).to(dtype=dtype).eval()
230+
assert block.attn1.backend == AttentionBackendEnum.FLASHINFER
231+
assert block.attn2.attn.backend == AttentionBackendEnum.FLASHINFER
232+
for param in block.parameters():
233+
torch.nn.init.normal_(param, std=0.02)
234+
q = torch.randn(1, 16, 2, 128, device="cuda", dtype=dtype)
235+
k, v = torch.randn_like(q), torch.randn_like(q)
236+
x = torch.randn(1, 16, 256, device="cuda", dtype=dtype)
237+
context = torch.randn(1, 264 if i2v else 7, 256, device="cuda", dtype=dtype)
238+
with torch.inference_mode(), set_forward_context(current_timestep=0, attn_metadata=None):
239+
self_output, _ = block.attn1(q, k, v)
240+
cross_output = block.attn2(x, context, None)
241+
assert self_output.shape == q.shape
242+
assert cross_output.shape == x.shape
243+
assert len(calls) == (3 if i2v else 2)
244+
245+
246+
def test_wan_cuda_skip_precedes_distributed_setup(monkeypatch):
247+
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
248+
class Request:
249+
def getfixturevalue(self, name):
250+
pytest.fail("distributed setup ran before the CUDA availability check")
251+
with pytest.raises(pytest.skip.Exception, match="NVIDIA CUDA GPU"):
252+
test_wan_attention_executes_flashinfer(monkeypatch, Request(), torch.float16, False)

‎fastvideo/tests/attention/test_flashinfer_cuda_dispatch.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,50 @@ def test_flashinfer_warns_on_dtype_cast(monkeypatch, caplog) -> None:
7373
with caplog.at_level("WARNING"):
7474
CudaPlatformBase.get_attn_backend_cls(AttentionBackendEnum.FLASHINFER, 128, torch.float32)
7575
assert any("cast" in record.message for record in caplog.records)
76+
77+
78+
def test_real_wan_config_reaches_flashinfer_dispatch(monkeypatch, env_overrides):
79+
import fastvideo.envs as envs
80+
import fastvideo.platforms as platforms
81+
from fastvideo.attention.selector import get_attn_backend
82+
from fastvideo.attention.backends.flashinfer import FlashInferBackend
83+
from fastvideo.models.wan.config import WanVideoConfig
84+
_patch_capability(monkeypatch, supported=True)
85+
_install_fake_flashinfer(monkeypatch)
86+
monkeypatch.setattr(platforms, "_current_platform", CudaPlatformBase())
87+
env_overrides.enter_context(envs.FASTVIDEO_ATTENTION_BACKEND.override("FLASHINFER"))
88+
backend = get_attn_backend(128, torch.bfloat16,
89+
supported_attention_backends=WanVideoConfig()._supported_attention_backends)
90+
assert backend is FlashInferBackend
91+
92+
93+
def test_wan_capabilities_preserve_defaults_and_other_models():
94+
from fastvideo.configs.models.dits.base import DiTArchConfig
95+
from fastvideo.models.wan.config import WanVideoArchConfig
96+
before = DiTArchConfig()._supported_attention_backends
97+
after = WanVideoArchConfig()._supported_attention_backends
98+
assert after == (*before, AttentionBackendEnum.FLASHINFER)
99+
assert AttentionBackendEnum.FLASHINFER not in before
100+
101+
102+
@pytest.mark.parametrize("backend", [AttentionBackendEnum.FLASHINFER,
103+
AttentionBackendEnum.FLASH_ATTN,
104+
AttentionBackendEnum.TORCH_SDPA])
105+
def test_causal_wan_rejects_flashinfer_before_weight_allocation(monkeypatch, backend):
106+
from fastvideo.attention.selector import _component_attention_backend_scope
107+
from fastvideo.models.wan import causal_transformer
108+
from fastvideo.models.wan.config import WanVideoConfig
109+
class WeightAllocationReached(Exception):
110+
pass
111+
def stop_at_first_layer(*args, **kwargs):
112+
raise WeightAllocationReached
113+
monkeypatch.setattr(causal_transformer, "PatchEmbed", stop_at_first_layer)
114+
model_cls = causal_transformer.CausalWanTransformer3DModel
115+
assert AttentionBackendEnum.FLASHINFER not in model_cls._supported_attention_backends
116+
with _component_attention_backend_scope(backend):
117+
if backend == AttentionBackendEnum.FLASHINFER:
118+
with pytest.raises(NotImplementedError, match="causal Wan is not supported"):
119+
model_cls(WanVideoConfig(), {})
120+
else:
121+
with pytest.raises(WeightAllocationReached):
122+
model_cls(WanVideoConfig(), {})

0 commit comments

Comments
 (0)