metric

VBench Color — GRiT dense captioning for color accuracy.

Detects the target object via GRiT and checks if the expected color keyword appears in the object's caption. Score = frames_with_correct_color / frames_with_object_detected.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric`

BaseMetric()

Abstract base class for all eval metrics.

Two execution shapes:

Per-sample (is_set_metric=False, default) — implement :meth:compute. The Evaluator calls it once per input sample and returns one :class:MetricResult per sample.
Set-vs-set (is_set_metric=True) — implement :meth:accumulate (called once per sample to buffer features) and :meth:finalize (called once after all samples to compute the corpus-level result). Use :meth:reset to clear buffers and :meth:merge_from to fold multi-GPU per-worker state together.

Optionally override :meth:setup to eagerly load models. Metrics that chunk along the time dim for memory hardcode their own chunk size in __init__ (see optical_flow for the canonical example). Eval always processes one video per :meth:Evaluator.evaluate call; compute / accumulate receive a single sample, not a batch.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.to`

to(device: str | torch.device) -> BaseMetric

Move metric (and its internal models) to device.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.setup`

setup() -> None

Eagerly load models. Called once by :class:EvalWorker.

Default is a no-op; metrics with no eager state (pixel math, closed-form ops) inherit this. Override only if your metric needs to load weights.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.compute`

compute(sample: dict) -> MetricResult

Per-sample metrics: compute the score for one sample.

sample["video"] is (T, C, H, W) float in [0, 1]. sample["reference"] (if used) has the same shape. Return self._skip(sample, reason) for missing inputs.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.reset`

reset() -> None

Clear accumulator state at the start of each evaluate() call.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.accumulate`

accumulate(sample: dict) -> None

Buffer per-sample features for a corpus-level metric.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.finalize`

finalize() -> MetricResult

Compute the corpus-level result from buffered state.

`fastvideo.eval.metrics.vbench.color.metric.BaseMetric.merge_from`

merge_from(other: BaseMetric) -> None

Multi-GPU: fold another worker's accumulator state into this one.

`fastvideo.eval.metrics.vbench.color.metric.register`

register(name: str)

Decorator to register a metric class.

Usage::

@register("ssim")
class SSIMMetric(BaseMetric):
    ...

`fastvideo.eval.metrics.vbench.color.metric.MetricResult`

MetricResult(name: str, score: float | None, details: dict[str, Any] = dict())

Standard result container returned by all metrics.

score is None when the metric was skipped (e.g. missing required input). Check details["skipped"] for the reason.

`fastvideo.eval.metrics.vbench.color.metric.ColorMetric`

ColorMetric()

Source

@register("vbench.color")
class ColorMetric(BaseMetric):
name = &quot;vbench.color&quot;
requires_reference = False
higher_is_better = True
needs_gpu = True
dependencies = &#91;&quot;detectron2&quot;&#93;

def __init__(self) -&gt; None:
    super().__init__()
    self._model: Any = None

def setup(self) -&gt; None:
    if self._model is not None:
        return
    from fastvideo.eval.metrics.vbench._grit_helper import load_grit_model
    self._model = load_grit_model(self.device)

@torch.no_grad()
def compute(self, sample: dict) -&gt; MetricResult:
    from fastvideo.eval.metrics.vbench._grit_helper import prepare_frames

    video = sample&#91;&quot;video&quot;&#93;  # (T, C, H, W)
    aux = sample.get(&quot;auxiliary_info&quot;) or &#123;&#125;
    if &quot;color&quot; not in aux:
        return self._skip(sample, &quot;missing &#x27;color&#x27; in auxiliary_info&quot;)

    prompt = sample.get(&quot;text_prompt&quot;) or &quot;&quot;
    color_key = aux&#91;&quot;color&quot;&#93;
    # Parse object name: remove &quot;a &quot;, &quot;an &quot;, and the color word
    object_key = prompt.replace(&quot;a &quot;, &quot;&quot;).replace(&quot;an &quot;, &quot;&quot;).replace(color_key, &quot;&quot;).strip()

    frames_np = prepare_frames(video)

    preds = &#91;&#93;
    for frame in frames_np:
        ret = self._model.run_caption_tensor(frame)
        cur_pred = &#91;&#93;
        if len(ret&#91;0&#93;) &lt; 1:
            cur_pred.append(&#91;&quot;&quot;, &quot;&quot;&#93;)
        else:
            for cap_det in ret&#91;0&#93;:
                cur_pred.append(&#91;cap_det&#91;0&#93;, cap_det&#91;2&#93;&#91;0&#93;&#93;)
        preds.append(cur_pred)

    # Score: matching VBench&#x27;s check_generate logic
    cur_object = 0
    cur_object_color = 0
    for frame_pred in preds:
        object_flag = False
        color_flag = False
        for pred in frame_pred:
            if object_key == pred&#91;1&#93;:
                for cq in _COLOR_KEYWORDS:
                    if cq in pred&#91;0&#93;:
                        object_flag = True
                if color_key in pred&#91;0&#93;:
                    color_flag = True
        if color_flag:
            cur_object_color += 1
        if object_flag:
            cur_object += 1

    score = cur_object_color / cur_object if cur_object &gt; 0 else 0.0
    return MetricResult(
        name=self.name,
        score=float(score),
        details=&#123;
            &quot;object_detected&quot;: cur_object,
            &quot;color_correct&quot;: cur_object_color
        &#125;,
    )</code></pre></details>
fastvideo.eval.metrics.vbench.color.metric.ColorMetric.setup
setup() -> None
Source
def setup(self) -> None:
    if self._model is not None:
        return
    from fastvideo.eval.metrics.vbench._grit_helper import load_grit_model
    self._model = load_grit_model(self.device)
fastvideo.eval.metrics.vbench.color.metric.ColorMetric.compute
compute(sample: dict) -> MetricResult
Source
@torch.no_grad()
def compute(self, sample: dict) -> MetricResult:
    from fastvideo.eval.metrics.vbench._grit_helper import prepare_frames
video = sample&#91;&quot;video&quot;&#93;  # (T, C, H, W)
aux = sample.get(&quot;auxiliary_info&quot;) or &#123;&#125;
if &quot;color&quot; not in aux:
    return self._skip(sample, &quot;missing &#x27;color&#x27; in auxiliary_info&quot;)

prompt = sample.get(&quot;text_prompt&quot;) or &quot;&quot;
color_key = aux&#91;&quot;color&quot;&#93;
# Parse object name: remove &quot;a &quot;, &quot;an &quot;, and the color word
object_key = prompt.replace(&quot;a &quot;, &quot;&quot;).replace(&quot;an &quot;, &quot;&quot;).replace(color_key, &quot;&quot;).strip()

frames_np = prepare_frames(video)

preds = &#91;&#93;
for frame in frames_np:
    ret = self._model.run_caption_tensor(frame)
    cur_pred = &#91;&#93;
    if len(ret&#91;0&#93;) &lt; 1:
        cur_pred.append(&#91;&quot;&quot;, &quot;&quot;&#93;)
    else:
        for cap_det in ret&#91;0&#93;:
            cur_pred.append(&#91;cap_det&#91;0&#93;, cap_det&#91;2&#93;&#91;0&#93;&#93;)
    preds.append(cur_pred)

# Score: matching VBench&#x27;s check_generate logic
cur_object = 0
cur_object_color = 0
for frame_pred in preds:
    object_flag = False
    color_flag = False
    for pred in frame_pred:
        if object_key == pred&#91;1&#93;:
            for cq in _COLOR_KEYWORDS:
                if cq in pred&#91;0&#93;:
                    object_flag = True
            if color_key in pred&#91;0&#93;:
                color_flag = True
    if color_flag:
        cur_object_color += 1
    if object_flag:
        cur_object += 1

score = cur_object_color / cur_object if cur_object &gt; 0 else 0.0
return MetricResult(
    name=self.name,
    score=float(score),
    details=&#123;
        &quot;object_detected&quot;: cur_object,
        &quot;color_correct&quot;: cur_object_color
    &#125;,
)</code></pre></details>