--- title: t5gemma ---

Config for the T5-Gemma encoder used by daVinci-MagiHuman.

The reference pipeline uses transformers.models.t5gemma.T5GemmaEncoderModel on google/t5gemma-9b-9b-ul2. That is a gated Google repository, so the encoder weights are not bundled inside GAIR/daVinci-MagiHuman; they are loaded from the T5-Gemma HF repo directly.

Encoder shape (verified from google/t5gemma-9b-9b-ul2/config.json): layers=42, hidden=3584, heads=16, kv_heads=8, head_dim=256, intermediate=14336, rope_theta=10000.0, max_pos=8192, layer_types alternate sliding_attention / full_attention.

fastvideo.configs.models.encoders.t5gemma.TextEncoderArchConfig

TextEncoderArchConfig(stacked_params_mapping: list[tuple[str, str, str]] = list(), architectures: list[str] = (lambda: [])(), _supported_attention_backends: tuple[AttentionBackendEnum, ...] = (AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TORCH_SDPA), output_hidden_states: bool = False, use_return_dict: bool = True, vocab_size: int = 0, hidden_size: int = 0, num_hidden_layers: int = 0, num_attention_heads: int = 0, pad_token_id: int = 0, eos_token_id: int = 0, text_len: int = 0, hidden_state_skip_layer: int = 0, decoder_start_token_id: int = 0, output_past: bool = True, scalable_attention: bool = True, tie_word_embeddings: bool = False, tokenizer_kwargs: dict[str, Any] = dict(), _fsdp_shard_conditions: list = (lambda: [])(), require_processor: bool = False)

fastvideo.configs.models.encoders.t5gemma.TextEncoderConfig

TextEncoderConfig(arch_config: ArchConfig = TextEncoderArchConfig(), prefix: str = '', quant_config: QuantizationConfig | None = None, lora_config: Any | None = None, is_chat_model: bool = False, treat_empty_as_dot: bool = False, *, _resolved_attention_backend: AttentionBackendEnum | None = None, chat_template_enable_thinking: bool = False)

fastvideo.configs.models.encoders.t5gemma.T5GemmaEncoderArchConfig

T5GemmaEncoderArchConfig(stacked_params_mapping: list[tuple[str, str, str]] = list(), architectures: list[str] = (lambda: ['T5GemmaEncoderModel'])(), _supported_attention_backends: tuple[AttentionBackendEnum, ...] = (AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TORCH_SDPA), output_hidden_states: bool = False, use_return_dict: bool = True, vocab_size: int = 256000, hidden_size: int = 3584, num_hidden_layers: int = 42, num_attention_heads: int = 16, pad_token_id: int = 0, eos_token_id: int = 1, text_len: int = 640, hidden_state_skip_layer: int = 0, decoder_start_token_id: int = 0, output_past: bool = True, scalable_attention: bool = True, tie_word_embeddings: bool = False, tokenizer_kwargs: dict[str, Any] = dict(), _fsdp_shard_conditions: list = list(), require_processor: bool = False, num_key_value_heads: int = 8, head_dim: int = 256, intermediate_size: int = 14336, max_position_embeddings: int = 8192, rope_theta: float = 10000.0, t5gemma_model_path: str = 'google/t5gemma-9b-9b-ul2', t5gemma_dtype: str = 'bfloat16')
Source
@dataclass
class T5GemmaEncoderArchConfig(TextEncoderArchConfig):
    architectures: list[str] = field(default_factory=lambda: ["T5GemmaEncoderModel"])

    hidden_size: int = 3584
    num_hidden_layers: int = 42
    num_attention_heads: int = 16
    num_key_value_heads: int = 8
    head_dim: int = 256
    intermediate_size: int = 14336
    max_position_embeddings: int = 8192
    rope_theta: float = 10000.0
    vocab_size: int = 256000

    # MagiHuman fixes prompt embed length at 640 via pad_or_trim.
    text_len: int = 640

    pad_token_id: int = 0
    eos_token_id: int = 1

    # Path to the upstream gated repo. When set, the FastVideo loader will
    # pull the encoder directly via `T5GemmaEncoderModel.from_pretrained`.
    t5gemma_model_path: str = "google/t5gemma-9b-9b-ul2"
    t5gemma_dtype: str = "bfloat16"

    # The HF T5-Gemma encoder is lazy-loaded on first forward (see
    # `fastvideo/models/encoders/t5gemma.py`), so no FastVideo-owned
    # submodules exist at FSDP-apply time. An empty list makes
    # `shard_model()` log a warning and return cleanly instead of raising
    # "No layer modules were sharded" — sharding of the lazy HF model is
    # the activation pipeline's responsibility.
    _fsdp_shard_conditions: list = field(default_factory=list)

    def __post_init__(self) -> None:
        super().__post_init__()
        # WHY: upstream `t5_gemma_model.py:25` tokenizes without
        # padding/max_length, then `prompt_process.py` pad_or_trim-s the
        # encoded states. Keep only tensor return here so
        # MagiHumanLatentPreparationStage can pad/trim post-encode while
        # preserving the real original prompt length.
        self.tokenizer_kwargs.pop("truncation", None)
        self.tokenizer_kwargs.pop("max_length", None)
        self.tokenizer_kwargs.pop("padding", None)

fastvideo.configs.models.encoders.t5gemma.T5GemmaEncoderConfig

T5GemmaEncoderConfig(arch_config: TextEncoderArchConfig = T5GemmaEncoderArchConfig(), prefix: str = 't5gemma', quant_config: QuantizationConfig | None = None, lora_config: Any | None = None, is_chat_model: bool = False, treat_empty_as_dot: bool = False, *, _resolved_attention_backend: AttentionBackendEnum | None = None, chat_template_enable_thinking: bool = False)
Source
@dataclass
class T5GemmaEncoderConfig(TextEncoderConfig):
    arch_config: TextEncoderArchConfig = field(default_factory=T5GemmaEncoderArchConfig)

    prefix: str = "t5gemma"