Skip to content

qwen2_5

Classes

Functions:

fastvideo.models.encoders.qwen2_5.repeat_kv 

repeat_kv(hidden_states: Tensor, n_rep: int) -> Tensor

This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)

Source code in fastvideo/models/encoders/qwen2_5.py 
25
26
27
28
29
30
31
32
33
34
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
    """
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    """
    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
    if n_rep == 1:
        return hidden_states
    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)

fastvideo.models.encoders.qwen2_5.rotate_half 

rotate_half(x)

Rotates half the hidden dims of the input.

Source code in fastvideo/models/encoders/qwen2_5.py 
76
77
78
79
80
def rotate_half(x):
    """Rotates half the hidden dims of the input."""
    x1 = x[..., : x.shape[-1] // 2]
    x2 = x[..., x.shape[-1] // 2 :]
    return torch.cat((-x2, x1), dim=-1)