SigLIP Vision Encoder for FastVideo.
SigLIP (Sigmoid Loss for Language-Image Pre-training) is similar to CLIP but uses sigmoid loss instead of contrastive loss, and doesn't use a CLS token.
Classes
Bases: Module
Multi-headed attention for SigLIP.
Source code in fastvideo/models/encoders/siglip.py
| def __init__(
self,
arch_config: SiglipVisionArchConfig,
enable_scale: bool = True,
is_causal: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.arch_config = arch_config
self.embed_dim = arch_config.hidden_size
self.num_heads = arch_config.num_attention_heads
self.head_dim = self.embed_dim // self.num_heads
if self.head_dim * self.num_heads != self.embed_dim:
raise ValueError(
f"embed_dim must be divisible by num_heads "
f"(got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})."
)
self.scale = self.head_dim**-0.5 if enable_scale else None
self.dropout = arch_config.attention_dropout
self.qkv_proj = QKVParallelLinear(
hidden_size=self.embed_dim,
head_size=self.head_dim,
total_num_heads=self.num_heads,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.out_proj = RowParallelLinear(
input_size=self.embed_dim,
output_size=self.embed_dim,
quant_config=quant_config,
prefix=f"{prefix}.out_proj",
)
self.tp_size = get_tp_world_size()
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
self.attn = LocalAttention(
self.num_heads_per_partition,
self.head_dim,
self.num_heads_per_partition,
softmax_scale=self.scale,
causal=is_causal,
supported_attention_backends=arch_config._supported_attention_backends,
)
|
Methods:
forward(hidden_states: Tensor)
Input shape: Batch x Time x Channel
Source code in fastvideo/models/encoders/siglip.py
| def forward(self, hidden_states: torch.Tensor):
"""Input shape: Batch x Time x Channel"""
qkv_states, _ = self.qkv_proj(hidden_states)
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
query_states = query_states.reshape(
query_states.shape[0], query_states.shape[1],
self.num_heads_per_partition, self.head_dim
)
key_states = key_states.reshape(
key_states.shape[0], key_states.shape[1],
self.num_heads_per_partition, self.head_dim
)
value_states = value_states.reshape(
value_states.shape[0], value_states.shape[1],
self.num_heads_per_partition, self.head_dim
)
attn_output = self.attn(query_states, key_states, value_states)
attn_output = attn_output.reshape(
attn_output.shape[0], attn_output.shape[1],
self.num_heads_per_partition * self.head_dim
)
attn_output, _ = self.out_proj(attn_output)
return attn_output, None
|
Bases: Module
SigLIP encoder consisting of transformer layers.
Source code in fastvideo/models/encoders/siglip.py
| def __init__(
self,
arch_config: SiglipVisionArchConfig,
enable_scale: bool = True,
is_causal: bool = False,
quant_config: QuantizationConfig | None = None,
num_hidden_layers_override: int | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.arch_config = arch_config
if num_hidden_layers_override is None:
num_hidden_layers = arch_config.num_hidden_layers
else:
num_hidden_layers = num_hidden_layers_override
self.layers = nn.ModuleList([
SiglipEncoderLayer(
arch_config=arch_config,
enable_scale=enable_scale,
is_causal=is_causal,
quant_config=quant_config,
prefix=f"{prefix}.layers.{layer_idx}",
)
for layer_idx in range(num_hidden_layers)
])
|
Bases: Module
SigLIP vision embeddings - similar to CLIP but without class embedding.
Source code in fastvideo/models/encoders/siglip.py
| def __init__(self, arch_config: SiglipVisionArchConfig):
super().__init__()
self.arch_config = arch_config
self.embed_dim = arch_config.hidden_size
self.image_size = arch_config.image_size
self.patch_size = arch_config.patch_size
# SigLIP uses valid padding, so non-divisible sizes work (edge pixels are ignored)
self.patch_embedding = nn.Conv2d(
in_channels=arch_config.num_channels,
out_channels=self.embed_dim,
kernel_size=self.patch_size,
stride=self.patch_size,
padding="valid", # SigLIP uses valid padding
)
# Integer division - with valid padding, edge pixels are ignored
self.num_patches = (self.image_size // self.patch_size) ** 2
self.num_positions = self.num_patches
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
self.register_buffer(
"position_ids",
torch.arange(self.num_positions).expand((1, -1)),
persistent=False,
)
|
Bases: ImageEncoder
SigLIP Vision Model for FastVideo.
Source code in fastvideo/models/encoders/siglip.py
| def __init__(self, config: SiglipVisionConfig) -> None:
super().__init__(config)
self.config = config
self.vision_model = SiglipVisionTransformer(
config=config,
quant_config=config.quant_config,
num_hidden_layers_override=config.num_hidden_layers_override,
require_post_norm=config.require_post_norm,
prefix=f"{config.prefix}.vision_model",
)
|
Functions: