fastvideo eval CLI: list registered eval metrics and run them against a set of videos.

This is a thin wrapper around :mod:fastvideo.eval. Heavy lifting (metric loading, GPU handling, batching) lives in :func:fastvideo.eval.create_evaluator.

Examples::

fastvideo eval list
fastvideo eval list --group vbench
fastvideo eval run --videos path/to/videos/*.mp4 \
    --metrics common.ssim --reference path/to/refs/
fastvideo eval run --videos clip.mp4 --metrics vbench.aesthetic_quality \
    --output scores.json

`fastvideo.entrypoints.cli.eval.CLISubcommand`

CLISubcommand()

Base class for CLI subcommands

`fastvideo.entrypoints.cli.eval.CLISubcommand.cmd`

cmd(args: argparse.Namespace) -> None

Execute the command with the given arguments

`fastvideo.entrypoints.cli.eval.CLISubcommand.validate`

validate(args: argparse.Namespace) -> None

Validate the arguments for this command

`fastvideo.entrypoints.cli.eval.CLISubcommand.subparser_init`

subparser_init(subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser

Initialize the subparser for this command

`fastvideo.entrypoints.cli.eval.init_logger`

init_logger(name: str) -> _FastvideoLogger

The main purpose of this function is to ensure that loggers are retrieved in such a way that we can be sure the root fastvideo logger has already been configured.

`fastvideo.entrypoints.cli.eval.FlexibleArgumentParser`

FlexibleArgumentParser(*args, **kwargs)

ArgumentParser that allows both underscore and dash in names.

`fastvideo.entrypoints.cli.eval.FlexibleArgumentParser.parse_args`

parse_args(args = None, namespace = None) -> argparse.Namespace

`fastvideo.entrypoints.cli.eval.FlexibleArgumentParser.parse_known_args`

parse_known_args(args = None, namespace = None) -> tuple[argparse.Namespace, list[str]]

`fastvideo.entrypoints.cli.eval.EvalSubcommand`

EvalSubcommand()

The eval subcommand — entry point for the eval suite.

Source

class EvalSubcommand(CLISubcommand):
    """The ``eval`` subcommand — entry point for the eval suite."""

    def __init__(self) -> None:
        self.name = "eval"
        super().__init__()

    def cmd(self, args: argparse.Namespace) -> None:
        action = getattr(args, "eval_action", None)
        if action == "list":
            _cmd_list(args)
        elif action == "run":
            _cmd_run(args)
        elif action == "v2a":
            _cmd_v2a(args)
        else:
            # Re-print help if no action was given.
            self._parser.print_help()  # type: ignore[attr-defined]

    def validate(self, args: argparse.Namespace) -> None:
        action = getattr(args, "eval_action", None)
        if action == "v2a":
            if args.backend != "av-benchmark":
                raise SystemExit(f"Unsupported V2A evaluation backend: {args.backend}")
            return
        if action != "run":
            return
        if not args.manifest and not args.videos and not args.audios:
            raise SystemExit("`fastvideo eval run` requires --manifest, --videos, or --audios")
        if args.manifest and (args.videos or args.audios or args.reference or args.reference_audio):
            raise SystemExit("--manifest cannot be combined with media path arguments")

    def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
        eval_parser = subparsers.add_parser(
            "eval",
            help="Run video and V2A evaluation metrics",
            usage="fastvideo eval {list,run,v2a} [...]",
        )
        sub = eval_parser.add_subparsers(dest="eval_action", required=False)

        # `eval list`
        list_p = sub.add_parser("list", help="List registered metrics")
        list_p.add_argument("--group", type=str, default=None, help="Filter to a metric group (e.g. 'vbench').")

        # `eval run`
        run_p = sub.add_parser("run", help="Evaluate videos against one or more metrics")
        run_p.add_argument(
            "--manifest",
            type=str,
            default=None,
            help="JSON/JSONL canonical sample manifest (recommended for benchmarks).",
        )
        run_p.add_argument("--videos",
                           type=str,
                           nargs="+",
                           required=False,
                           help="Path, glob, or directory of generated videos.")
        run_p.add_argument("--audios",
                           type=str,
                           nargs="+",
                           required=False,
                           help="Path, glob, or directory of generated audio files.")
        run_p.add_argument("--reference",
                           type=str,
                           default=None,
                           help="Path / glob / dir of reference videos (for paired metrics).")
        run_p.add_argument(
            "--reference-audio",
            type=str,
            nargs="+",
            default=None,
            help="Path, glob, or directory of reference audio files.",
        )
        run_p.add_argument("--metrics", type=str, default="all", help="Comma-separated metric names, or 'all'.")
        run_p.add_argument("--device", type=str, default="cuda", help="Torch device (e.g. 'cuda', 'cuda:0', 'cpu').")
        run_p.add_argument("--num-gpus", type=int, default=1, help="Number of metric replicas for dataset evaluation.")
        run_p.add_argument("--compile", action="store_true", help="Compile supported metric models.")
        run_p.add_argument(
            "--skip-missing-deps",
            action="store_true",
            help="Skip explicitly requested metrics whose optional dependencies are missing.",
        )
        run_p.add_argument(
            "--extract-audio",
            nargs="?",
            const=True,
            default=False,
            help="Extract missing audio tracks; optionally provide a persistent cache directory.",
        )
        run_p.add_argument("--extract-workers", type=int, default=4)
        run_p.add_argument(
            "--text-prompt",
            type=str,
            nargs="*",
            default=None,
            help="Prompt(s) for text-conditioned metrics. One per video.",
        )
        run_p.add_argument("--fps", type=float, default=None, help="Frame-rate annotation passed to fps-aware metrics.")
        run_p.add_argument("--output",
                           type=str,
                           default=None,
                           help="Write results as JSON to this path (default: stdout).")
        run_p.add_argument(
            "--output-format",
            choices=("legacy", "full"),
            default="legacy",
            help="'full' includes corpus metrics and sample identities.",
        )

        # `eval v2a`: isolated corpus-level backends for video-to-audio.
        v2a_p = sub.add_parser("v2a", help="Evaluate V2A audio with a corpus benchmark backend")
        v2a_p.add_argument("--backend", choices=("av-benchmark", ), default="av-benchmark")
        v2a_p.add_argument("--audio-dir", required=True, help="Directory containing generated WAV/FLAC files.")
        v2a_p.add_argument("--gt-cache", required=True, help="Official ground-truth feature cache directory.")
        v2a_p.add_argument(
            "--prediction-cache",
            default=None,
            help="Generated-audio feature cache (default: <audio-dir>/../av_benchmark_cache).",
        )
        v2a_p.add_argument(
            "--output",
            default=None,
            help="Result JSON (default: <audio-dir>/../av_benchmark_results.json).",
        )
        v2a_p.add_argument(
            "--python-executable",
            default=None,
            help="Python from an isolated av-benchmark environment.",
        )
        v2a_p.add_argument("--device", default="cuda")
        v2a_p.add_argument("--batch-size", type=int, default=32)
        v2a_p.add_argument("--num-workers", type=int, default=0)
        v2a_p.add_argument("--audio-length", type=float, default=8.0)
        v2a_p.add_argument("--recompute", action="store_true", help="Re-extract the prediction cache.")
        v2a_p.add_argument("--skip-video-related", action="store_true", help="Skip ImageBind and DeSync.")
        v2a_p.add_argument("--skip-clap", action="store_true", help="Skip LAION/MS CLAP extraction and scores.")
        v2a_p.add_argument(
            "--align-prediction-keys",
            action="store_true",
            help="Align uniquely underscore-prefixed prediction ids to official GT cache ids.",
        )

        # Stash the parser so cmd() can re-print help on no-action.
        self._parser = eval_parser  # type: ignore[attr-defined]
        return cast(FlexibleArgumentParser, eval_parser)

`fastvideo.entrypoints.cli.eval.EvalSubcommand.cmd`

cmd(args: argparse.Namespace) -> None

Source

def cmd(self, args: argparse.Namespace) -> None:
    action = getattr(args, "eval_action", None)
    if action == "list":
        _cmd_list(args)
    elif action == "run":
        _cmd_run(args)
    elif action == "v2a":
        _cmd_v2a(args)
    else:
        # Re-print help if no action was given.
        self._parser.print_help()  # type: ignore[attr-defined]

`fastvideo.entrypoints.cli.eval.EvalSubcommand.validate`

validate(args: argparse.Namespace) -> None

Source

def validate(self, args: argparse.Namespace) -> None:
    action = getattr(args, "eval_action", None)
    if action == "v2a":
        if args.backend != "av-benchmark":
            raise SystemExit(f"Unsupported V2A evaluation backend: {args.backend}")
        return
    if action != "run":
        return
    if not args.manifest and not args.videos and not args.audios:
        raise SystemExit("`fastvideo eval run` requires --manifest, --videos, or --audios")
    if args.manifest and (args.videos or args.audios or args.reference or args.reference_audio):
        raise SystemExit("--manifest cannot be combined with media path arguments")

`fastvideo.entrypoints.cli.eval.EvalSubcommand.subparser_init`

subparser_init(subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser

Source

def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
    eval_parser = subparsers.add_parser(
        "eval",
        help="Run video and V2A evaluation metrics",
        usage="fastvideo eval {list,run,v2a} [...]",
    )
    sub = eval_parser.add_subparsers(dest="eval_action", required=False)

    # `eval list`
    list_p = sub.add_parser("list", help="List registered metrics")
    list_p.add_argument("--group", type=str, default=None, help="Filter to a metric group (e.g. 'vbench').")

    # `eval run`
    run_p = sub.add_parser("run", help="Evaluate videos against one or more metrics")
    run_p.add_argument(
        "--manifest",
        type=str,
        default=None,
        help="JSON/JSONL canonical sample manifest (recommended for benchmarks).",
    )
    run_p.add_argument("--videos",
                       type=str,
                       nargs="+",
                       required=False,
                       help="Path, glob, or directory of generated videos.")
    run_p.add_argument("--audios",
                       type=str,
                       nargs="+",
                       required=False,
                       help="Path, glob, or directory of generated audio files.")
    run_p.add_argument("--reference",
                       type=str,
                       default=None,
                       help="Path / glob / dir of reference videos (for paired metrics).")
    run_p.add_argument(
        "--reference-audio",
        type=str,
        nargs="+",
        default=None,
        help="Path, glob, or directory of reference audio files.",
    )
    run_p.add_argument("--metrics", type=str, default="all", help="Comma-separated metric names, or 'all'.")
    run_p.add_argument("--device", type=str, default="cuda", help="Torch device (e.g. 'cuda', 'cuda:0', 'cpu').")
    run_p.add_argument("--num-gpus", type=int, default=1, help="Number of metric replicas for dataset evaluation.")
    run_p.add_argument("--compile", action="store_true", help="Compile supported metric models.")
    run_p.add_argument(
        "--skip-missing-deps",
        action="store_true",
        help="Skip explicitly requested metrics whose optional dependencies are missing.",
    )
    run_p.add_argument(
        "--extract-audio",
        nargs="?",
        const=True,
        default=False,
        help="Extract missing audio tracks; optionally provide a persistent cache directory.",
    )
    run_p.add_argument("--extract-workers", type=int, default=4)
    run_p.add_argument(
        "--text-prompt",
        type=str,
        nargs="*",
        default=None,
        help="Prompt(s) for text-conditioned metrics. One per video.",
    )
    run_p.add_argument("--fps", type=float, default=None, help="Frame-rate annotation passed to fps-aware metrics.")
    run_p.add_argument("--output",
                       type=str,
                       default=None,
                       help="Write results as JSON to this path (default: stdout).")
    run_p.add_argument(
        "--output-format",
        choices=("legacy", "full"),
        default="legacy",
        help="'full' includes corpus metrics and sample identities.",
    )

    # `eval v2a`: isolated corpus-level backends for video-to-audio.
    v2a_p = sub.add_parser("v2a", help="Evaluate V2A audio with a corpus benchmark backend")
    v2a_p.add_argument("--backend", choices=("av-benchmark", ), default="av-benchmark")
    v2a_p.add_argument("--audio-dir", required=True, help="Directory containing generated WAV/FLAC files.")
    v2a_p.add_argument("--gt-cache", required=True, help="Official ground-truth feature cache directory.")
    v2a_p.add_argument(
        "--prediction-cache",
        default=None,
        help="Generated-audio feature cache (default: <audio-dir>/../av_benchmark_cache).",
    )
    v2a_p.add_argument(
        "--output",
        default=None,
        help="Result JSON (default: <audio-dir>/../av_benchmark_results.json).",
    )
    v2a_p.add_argument(
        "--python-executable",
        default=None,
        help="Python from an isolated av-benchmark environment.",
    )
    v2a_p.add_argument("--device", default="cuda")
    v2a_p.add_argument("--batch-size", type=int, default=32)
    v2a_p.add_argument("--num-workers", type=int, default=0)
    v2a_p.add_argument("--audio-length", type=float, default=8.0)
    v2a_p.add_argument("--recompute", action="store_true", help="Re-extract the prediction cache.")
    v2a_p.add_argument("--skip-video-related", action="store_true", help="Skip ImageBind and DeSync.")
    v2a_p.add_argument("--skip-clap", action="store_true", help="Skip LAION/MS CLAP extraction and scores.")
    v2a_p.add_argument(
        "--align-prediction-keys",
        action="store_true",
        help="Align uniquely underscore-prefixed prediction ids to official GT cache ids.",
    )

    # Stash the parser so cmd() can re-print help on no-action.
    self._parser = eval_parser  # type: ignore[attr-defined]
    return cast(FlexibleArgumentParser, eval_parser)

`fastvideo.entrypoints.cli.eval.cmd_init`

cmd_init() -> list[CLISubcommand]

Source

def cmd_init() -> list[CLISubcommand]:
    return [EvalSubcommand()]