metric

VBench Human Action — UMT ViT-L/16 action classification (Kinetics-400).

Classifies human actions in 16-frame clips. Top-5 predictions with confidence >= 0.85 are compared against the ground-truth action label. Score = 1.0 if match found, 0.0 otherwise.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric`

BaseMetric()

Abstract base class for all eval metrics.

Two execution shapes:

Per-sample (is_set_metric=False, default) — implement :meth:compute. The Evaluator calls it once per input sample and returns one :class:MetricResult per sample.
Set-vs-set (is_set_metric=True) — implement :meth:accumulate (called once per sample to buffer features) and :meth:finalize (called once after all samples to compute the corpus-level result). Use :meth:reset to clear buffers and :meth:merge_from to fold multi-GPU per-worker state together.

Optionally override :meth:setup to eagerly load models. Metrics that chunk along the time dim for memory hardcode their own chunk size in __init__ (see optical_flow for the canonical example). Eval always processes one video per :meth:Evaluator.evaluate call; compute / accumulate receive a single sample, not a batch.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.to`

to(device: str | torch.device) -> BaseMetric

Move metric (and its internal models) to device.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.setup`

setup() -> None

Eagerly load models. Called once by :class:EvalWorker.

Default is a no-op; metrics with no eager state (pixel math, closed-form ops) inherit this. Override only if your metric needs to load weights.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.compute`

compute(sample: dict) -> MetricResult

Per-sample metrics: compute the score for one sample.

sample["video"] is (T, C, H, W) float in [0, 1]. sample["reference"] (if used) has the same shape. Return self._skip(sample, reason) for missing inputs.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.reset`

reset() -> None

Clear accumulator state at the start of each evaluate() call.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.accumulate`

accumulate(sample: dict) -> None

Buffer per-sample features for a corpus-level metric.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.finalize`

finalize() -> MetricResult

Compute the corpus-level result from buffered state.

`fastvideo.eval.metrics.vbench.human_action.metric.BaseMetric.merge_from`

merge_from(other: BaseMetric) -> None

Multi-GPU: fold another worker's accumulator state into this one.

`fastvideo.eval.metrics.vbench.human_action.metric.register`

register(name: str)

Decorator to register a metric class.

Usage::

@register("ssim")
class SSIMMetric(BaseMetric):
    ...

`fastvideo.eval.metrics.vbench.human_action.metric.MetricResult`

MetricResult(name: str, score: float | None, details: dict[str, Any] = dict())

Standard result container returned by all metrics.

score is None when the metric was skipped (e.g. missing required input). Check details["skipped"] for the reason.

`fastvideo.eval.metrics.vbench.human_action.metric.extract_frames`

extract_frames(video: torch.Tensor, n_frames: int | None = None) -> torch.Tensor

Uniformly sample n_frames from a (T, C, H, W) video tensor.

`fastvideo.eval.metrics.vbench.human_action.metric.HumanActionMetric`

HumanActionMetric()

Source

@register("vbench.human_action")
class HumanActionMetric(BaseMetric):
name = &quot;vbench.human_action&quot;
requires_reference = False
higher_is_better = True
needs_gpu = True
dependencies = &#91;&quot;timm&quot;&#93;

def __init__(self) -&gt; None:
    super().__init__()
    self._model: Any = None

def to(self, device):
    super().to(device)
    if self._model is not None:
        self._model = self._model.to(self.device)
    return self

def setup(self) -&gt; None:
    if self._model is not None:
        return
    from timm.models import create_model
    from fastvideo.eval.models import ensure_checkpoint

    ckpt_path = ensure_checkpoint(
        &quot;umt_l16_kinetics400.pth&quot;,
        source=&quot;OpenGVLab/VBench_Used_Models&quot;,
        filename=&quot;l16_ptk710_ftk710_ftk400_f16_res224.pth&quot;,
    )

    import vbench.third_party.umt.models.modeling_finetune  # noqa: F401

    self._model = create_model(
        &quot;vit_large_patch16_224&quot;,
        pretrained=False,
        num_classes=400,
        all_frames=16,
        tubelet_size=1,
        use_learnable_pos_emb=False,
        fc_drop_rate=0.0,
        drop_rate=0.0,
        drop_path_rate=0.2,
        attn_drop_rate=0.0,
        drop_block_rate=None,
        use_checkpoint=False,
        checkpoint_num=16,
        use_mean_pooling=True,
        init_scale=0.001,
    )
    state_dict = torch.load(ckpt_path, map_location=&quot;cpu&quot;, weights_only=False)
    self._model.load_state_dict(state_dict, strict=False)
    self._model.to(self.device)
    self._model.eval()

@torch.no_grad()
def compute(self, sample: dict) -&gt; MetricResult:
    video = sample&#91;&quot;video&quot;&#93;  # (T, C, H, W) &#91;0, 1&#93;
    text_prompt = sample.get(&quot;text_prompt&quot;)
    if text_prompt is None:
        return self._skip(sample, &quot;missing text_prompt with action labels&quot;)

    cat_dict = _load_cat_dict()

    frames = extract_frames(video, 16)  # (16, C, H, W)
    frames = resize(frames, 256, antialias=True)
    frames = center_crop(frames, 224)
    frames = normalize(frames, mean=&#91;0.485, 0.456, 0.406&#93;, std=&#91;0.229, 0.224, 0.225&#93;)
    # UMT expects (C, T, H, W); add a leading batch dim of 1.
    clip_in = frames.permute(1, 0, 2, 3).unsqueeze(0).to(self.device)

    logits = torch.sigmoid(self._model(clip_in))  # (1, 400)
    top_scores, top_indices = torch.topk(logits&#91;0&#93;, 5)
    top_indices = top_indices.tolist()
    top_scores = top_scores.tolist()

    predictions = &#91;
        cat_dict.get(str(idx), &quot;&quot;) for idx, score in zip(top_indices, top_scores, strict=False) if score &gt;= 0.85
    &#93;
    gt_label = text_prompt.lower().strip()
    match = any(pred == gt_label for pred in predictions)
    return MetricResult(
        name=self.name,
        score=1.0 if match else 0.0,
        details=&#123;
            &quot;predictions&quot;: predictions,
            &quot;ground_truth&quot;: gt_label
        &#125;,
    )</code></pre></details>
fastvideo.eval.metrics.vbench.human_action.metric.HumanActionMetric.to
to(device)
Source
def to(self, device):
    super().to(device)
    if self._model is not None:
        self._model = self._model.to(self.device)
    return self
fastvideo.eval.metrics.vbench.human_action.metric.HumanActionMetric.setup
setup() -> None
Source
def setup(self) -> None:
    if self._model is not None:
        return
    from timm.models import create_model
    from fastvideo.eval.models import ensure_checkpoint
ckpt_path = ensure_checkpoint(
    &quot;umt_l16_kinetics400.pth&quot;,
    source=&quot;OpenGVLab/VBench_Used_Models&quot;,
    filename=&quot;l16_ptk710_ftk710_ftk400_f16_res224.pth&quot;,
)

import vbench.third_party.umt.models.modeling_finetune  # noqa: F401

self._model = create_model(
    &quot;vit_large_patch16_224&quot;,
    pretrained=False,
    num_classes=400,
    all_frames=16,
    tubelet_size=1,
    use_learnable_pos_emb=False,
    fc_drop_rate=0.0,
    drop_rate=0.0,
    drop_path_rate=0.2,
    attn_drop_rate=0.0,
    drop_block_rate=None,
    use_checkpoint=False,
    checkpoint_num=16,
    use_mean_pooling=True,
    init_scale=0.001,
)
state_dict = torch.load(ckpt_path, map_location=&quot;cpu&quot;, weights_only=False)
self._model.load_state_dict(state_dict, strict=False)
self._model.to(self.device)
self._model.eval()</code></pre></details>
fastvideo.eval.metrics.vbench.human_action.metric.HumanActionMetric.compute
compute(sample: dict) -> MetricResult
Source
@torch.no_grad()
def compute(self, sample: dict) -> MetricResult:
    video = sample["video"]  # (T, C, H, W) [0, 1]
    text_prompt = sample.get("text_prompt")
    if text_prompt is None:
        return self._skip(sample, "missing text_prompt with action labels")
cat_dict = _load_cat_dict()

frames = extract_frames(video, 16)  # (16, C, H, W)
frames = resize(frames, 256, antialias=True)
frames = center_crop(frames, 224)
frames = normalize(frames, mean=&#91;0.485, 0.456, 0.406&#93;, std=&#91;0.229, 0.224, 0.225&#93;)
# UMT expects (C, T, H, W); add a leading batch dim of 1.
clip_in = frames.permute(1, 0, 2, 3).unsqueeze(0).to(self.device)

logits = torch.sigmoid(self._model(clip_in))  # (1, 400)
top_scores, top_indices = torch.topk(logits&#91;0&#93;, 5)
top_indices = top_indices.tolist()
top_scores = top_scores.tolist()

predictions = &#91;
    cat_dict.get(str(idx), &quot;&quot;) for idx, score in zip(top_indices, top_scores, strict=False) if score &gt;= 0.85
&#93;
gt_label = text_prompt.lower().strip()
match = any(pred == gt_label for pred in predictions)
return MetricResult(
    name=self.name,
    score=1.0 if match else 0.0,
    details=&#123;
        &quot;predictions&quot;: predictions,
        &quot;ground_truth&quot;: gt_label
    &#125;,
)</code></pre></details>