vision

class VisionEncoderInfo(ABC, Generic[_C]):
def __init__(self, vision_config: _C) -> None:
    super().__init__()

    self.vision_config = vision_config

@abstractmethod
def get_num_image_tokens(
    self,
    *,
    image_width: int,
    image_height: int,
) -> int:
    raise NotImplementedError

@abstractmethod
def get_max_image_tokens(self) -> int:
    raise NotImplementedError

@abstractmethod
def get_image_size(self) -> int:
    raise NotImplementedError

@abstractmethod
def get_patch_size(self) -> int:
    raise NotImplementedError

@abstractmethod
def get_patch_grid_length(self) -> int:
    raise NotImplementedError</code></pre></details>
fastvideo.models.encoders.vision.VisionEncoderInfo.get_num_image_tokens
get_num_image_tokens(*, image_width: int, image_height: int) -> int
Source
@abstractmethod
def get_num_image_tokens(
    self,
    *,
    image_width: int,
    image_height: int,
) -> int:
    raise NotImplementedError
fastvideo.models.encoders.vision.VisionEncoderInfo.get_max_image_tokens
get_max_image_tokens() -> int
Source
@abstractmethod
def get_max_image_tokens(self) -> int:
    raise NotImplementedError
fastvideo.models.encoders.vision.VisionEncoderInfo.get_image_size
get_image_size() -> int
Source
@abstractmethod
def get_image_size(self) -> int:
    raise NotImplementedError
fastvideo.models.encoders.vision.VisionEncoderInfo.get_patch_size
get_patch_size() -> int
Source
@abstractmethod
def get_patch_size(self) -> int:
    raise NotImplementedError
fastvideo.models.encoders.vision.VisionEncoderInfo.get_patch_grid_length
get_patch_grid_length() -> int
Source
@abstractmethod
def get_patch_grid_length(self) -> int:
    raise NotImplementedError
fastvideo.models.encoders.vision.resolve_visual_encoder_outputs
resolve_visual_encoder_outputs(encoder_outputs: torch.Tensor | list[torch.Tensor], feature_sample_layers: list[int] | None, post_layer_norm: torch.nn.LayerNorm | None, max_possible_layers: int) -> torch.Tensor
Given the outputs a visual encoder module that may correspond to the
output of the last layer, or a list of hidden states to be stacked,
handle post normalization and resolve it into a single output tensor.
Parameters

encoder_outputs: torch.Tensor | list[torch.Tensor]
Output of encoder's last layer or all hidden states.
feature_sample_layers: list[int] | None
Optional layer indices to grab from the encoder
outputs; if provided, encoder outputs must be a list.
post_layer_norm: torch.nn.LayerNorm | None
Post norm to apply to the output of the encoder.
max_possible_layers: int
Total layers in the fully loaded visual encoder.

Source
def resolve_visual_encoder_outputs(
    encoder_outputs: torch.Tensor | list[torch.Tensor],
    feature_sample_layers: list[int] | None,
    post_layer_norm: torch.nn.LayerNorm | None,
    max_possible_layers: int,
) -> torch.Tensor:
    """Given the outputs a visual encoder module that may correspond to the
    output of the last layer, or a list of hidden states to be stacked,
    handle post normalization and resolve it into a single output tensor.
Args:
    encoder_outputs: Output of encoder&#x27;s last layer or all hidden states.
    feature_sample_layers: Optional layer indices to grab from the encoder
        outputs; if provided, encoder outputs must be a list.
    post_layer_norm: Post norm to apply to the output of the encoder.
    max_possible_layers: Total layers in the fully loaded visual encoder.

&quot;&quot;&quot;
if feature_sample_layers is None:
    if post_layer_norm is not None:
        return post_layer_norm(encoder_outputs)
    return encoder_outputs

# Get the hidden states corresponding to the layer indices.
# Negative values are relative to the full visual encoder,
# so offset them depending on how many layers were loaded.
# NOTE: this assumes that encoder_outputs is a list containing
# the inputs to the visual encoder, followed by the hidden states
# of each layer.
num_loaded_layers = len(encoder_outputs) - 1
offset = max_possible_layers - num_loaded_layers
hs_pool = &#91;
    encoder_outputs&#91;layer_idx&#93; if layer_idx &gt;= 0 else encoder_outputs&#91;layer_idx + offset&#93;
    for layer_idx in feature_sample_layers
&#93;

# Apply post-norm on the final hidden state if we are using it
uses_last_layer = feature_sample_layers&#91;-1&#93; in (len(hs_pool) - 1, -1)
if post_layer_norm is not None and uses_last_layer:
    hs_pool&#91;-1&#93; = post_layer_norm(encoder_outputs)
return torch.cat(hs_pool, dim=-1)</code></pre></details>