eval
fastvideo eval CLI: list registered eval metrics and run them
against a set of videos.
This is a thin wrapper around :mod:fastvideo.eval. Heavy lifting
(metric loading, GPU handling, batching) lives in
:func:fastvideo.eval.create_evaluator.
Examples::
fastvideo eval list
fastvideo eval list --group vbench
fastvideo eval run --videos path/to/videos/*.mp4 \
--metrics common.ssim --reference path/to/refs/
fastvideo eval run --videos clip.mp4 --metrics vbench.aesthetic_quality \
--output scores.json
fastvideo.entrypoints.cli.eval.CLISubcommand
CLISubcommand()Base class for CLI subcommands
fastvideo.entrypoints.cli.eval.CLISubcommand.cmd
cmd(args: argparse.Namespace) -> NoneExecute the command with the given arguments
fastvideo.entrypoints.cli.eval.CLISubcommand.validate
validate(args: argparse.Namespace) -> NoneValidate the arguments for this command
fastvideo.entrypoints.cli.eval.CLISubcommand.subparser_init
subparser_init(subparsers: argparse._SubParsersAction) -> FlexibleArgumentParserInitialize the subparser for this command
fastvideo.entrypoints.cli.eval.init_logger
init_logger(name: str) -> _FastvideoLoggerThe main purpose of this function is to ensure that loggers are retrieved in such a way that we can be sure the root fastvideo logger has already been configured.
fastvideo.entrypoints.cli.eval.FlexibleArgumentParser
FlexibleArgumentParser(*args, **kwargs)ArgumentParser that allows both underscore and dash in names.
fastvideo.entrypoints.cli.eval.FlexibleArgumentParser.parse_args
parse_args(args = None, namespace = None) -> argparse.Namespacefastvideo.entrypoints.cli.eval.FlexibleArgumentParser.parse_known_args
parse_known_args(args = None, namespace = None) -> tuple[argparse.Namespace, list[str]]fastvideo.entrypoints.cli.eval.EvalSubcommand
EvalSubcommand()The eval subcommand — entry point for the eval suite.
Source
class EvalSubcommand(CLISubcommand):
"""The ``eval`` subcommand — entry point for the eval suite."""
def __init__(self) -> None:
self.name = "eval"
super().__init__()
def cmd(self, args: argparse.Namespace) -> None:
action = getattr(args, "eval_action", None)
if action == "list":
_cmd_list(args)
elif action == "run":
_cmd_run(args)
elif action == "v2a":
_cmd_v2a(args)
else:
# Re-print help if no action was given.
self._parser.print_help() # type: ignore[attr-defined]
def validate(self, args: argparse.Namespace) -> None:
action = getattr(args, "eval_action", None)
if action == "v2a":
if args.backend != "av-benchmark":
raise SystemExit(f"Unsupported V2A evaluation backend: {args.backend}")
return
if action != "run":
return
if not args.manifest and not args.videos and not args.audios:
raise SystemExit("`fastvideo eval run` requires --manifest, --videos, or --audios")
if args.manifest and (args.videos or args.audios or args.reference or args.reference_audio):
raise SystemExit("--manifest cannot be combined with media path arguments")
def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
eval_parser = subparsers.add_parser(
"eval",
help="Run video and V2A evaluation metrics",
usage="fastvideo eval {list,run,v2a} [...]",
)
sub = eval_parser.add_subparsers(dest="eval_action", required=False)
# `eval list`
list_p = sub.add_parser("list", help="List registered metrics")
list_p.add_argument("--group", type=str, default=None, help="Filter to a metric group (e.g. 'vbench').")
# `eval run`
run_p = sub.add_parser("run", help="Evaluate videos against one or more metrics")
run_p.add_argument(
"--manifest",
type=str,
default=None,
help="JSON/JSONL canonical sample manifest (recommended for benchmarks).",
)
run_p.add_argument("--videos",
type=str,
nargs="+",
required=False,
help="Path, glob, or directory of generated videos.")
run_p.add_argument("--audios",
type=str,
nargs="+",
required=False,
help="Path, glob, or directory of generated audio files.")
run_p.add_argument("--reference",
type=str,
default=None,
help="Path / glob / dir of reference videos (for paired metrics).")
run_p.add_argument(
"--reference-audio",
type=str,
nargs="+",
default=None,
help="Path, glob, or directory of reference audio files.",
)
run_p.add_argument("--metrics", type=str, default="all", help="Comma-separated metric names, or 'all'.")
run_p.add_argument("--device", type=str, default="cuda", help="Torch device (e.g. 'cuda', 'cuda:0', 'cpu').")
run_p.add_argument("--num-gpus", type=int, default=1, help="Number of metric replicas for dataset evaluation.")
run_p.add_argument("--compile", action="store_true", help="Compile supported metric models.")
run_p.add_argument(
"--skip-missing-deps",
action="store_true",
help="Skip explicitly requested metrics whose optional dependencies are missing.",
)
run_p.add_argument(
"--extract-audio",
nargs="?",
const=True,
default=False,
help="Extract missing audio tracks; optionally provide a persistent cache directory.",
)
run_p.add_argument("--extract-workers", type=int, default=4)
run_p.add_argument(
"--text-prompt",
type=str,
nargs="*",
default=None,
help="Prompt(s) for text-conditioned metrics. One per video.",
)
run_p.add_argument("--fps", type=float, default=None, help="Frame-rate annotation passed to fps-aware metrics.")
run_p.add_argument("--output",
type=str,
default=None,
help="Write results as JSON to this path (default: stdout).")
run_p.add_argument(
"--output-format",
choices=("legacy", "full"),
default="legacy",
help="'full' includes corpus metrics and sample identities.",
)
# `eval v2a`: isolated corpus-level backends for video-to-audio.
v2a_p = sub.add_parser("v2a", help="Evaluate V2A audio with a corpus benchmark backend")
v2a_p.add_argument("--backend", choices=("av-benchmark", ), default="av-benchmark")
v2a_p.add_argument("--audio-dir", required=True, help="Directory containing generated WAV/FLAC files.")
v2a_p.add_argument("--gt-cache", required=True, help="Official ground-truth feature cache directory.")
v2a_p.add_argument(
"--prediction-cache",
default=None,
help="Generated-audio feature cache (default: <audio-dir>/../av_benchmark_cache).",
)
v2a_p.add_argument(
"--output",
default=None,
help="Result JSON (default: <audio-dir>/../av_benchmark_results.json).",
)
v2a_p.add_argument(
"--python-executable",
default=None,
help="Python from an isolated av-benchmark environment.",
)
v2a_p.add_argument("--device", default="cuda")
v2a_p.add_argument("--batch-size", type=int, default=32)
v2a_p.add_argument("--num-workers", type=int, default=0)
v2a_p.add_argument("--audio-length", type=float, default=8.0)
v2a_p.add_argument("--recompute", action="store_true", help="Re-extract the prediction cache.")
v2a_p.add_argument("--skip-video-related", action="store_true", help="Skip ImageBind and DeSync.")
v2a_p.add_argument("--skip-clap", action="store_true", help="Skip LAION/MS CLAP extraction and scores.")
v2a_p.add_argument(
"--align-prediction-keys",
action="store_true",
help="Align uniquely underscore-prefixed prediction ids to official GT cache ids.",
)
# Stash the parser so cmd() can re-print help on no-action.
self._parser = eval_parser # type: ignore[attr-defined]
return cast(FlexibleArgumentParser, eval_parser)</code></pre></details>
fastvideo.entrypoints.cli.eval.EvalSubcommand.cmd
cmd(args: argparse.Namespace) -> None
Source
def cmd(self, args: argparse.Namespace) -> None:
action = getattr(args, "eval_action", None)
if action == "list":
_cmd_list(args)
elif action == "run":
_cmd_run(args)
elif action == "v2a":
_cmd_v2a(args)
else:
# Re-print help if no action was given.
self._parser.print_help() # type: ignore[attr-defined]
fastvideo.entrypoints.cli.eval.EvalSubcommand.validate
validate(args: argparse.Namespace) -> None
Source
def validate(self, args: argparse.Namespace) -> None:
action = getattr(args, "eval_action", None)
if action == "v2a":
if args.backend != "av-benchmark":
raise SystemExit(f"Unsupported V2A evaluation backend: {args.backend}")
return
if action != "run":
return
if not args.manifest and not args.videos and not args.audios:
raise SystemExit("`fastvideo eval run` requires --manifest, --videos, or --audios")
if args.manifest and (args.videos or args.audios or args.reference or args.reference_audio):
raise SystemExit("--manifest cannot be combined with media path arguments")
fastvideo.entrypoints.cli.eval.EvalSubcommand.subparser_init
subparser_init(subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser
Source
def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
eval_parser = subparsers.add_parser(
"eval",
help="Run video and V2A evaluation metrics",
usage="fastvideo eval {list,run,v2a} [...]",
)
sub = eval_parser.add_subparsers(dest="eval_action", required=False)
# `eval list`
list_p = sub.add_parser("list", help="List registered metrics")
list_p.add_argument("--group", type=str, default=None, help="Filter to a metric group (e.g. 'vbench').")
# `eval run`
run_p = sub.add_parser("run", help="Evaluate videos against one or more metrics")
run_p.add_argument(
"--manifest",
type=str,
default=None,
help="JSON/JSONL canonical sample manifest (recommended for benchmarks).",
)
run_p.add_argument("--videos",
type=str,
nargs="+",
required=False,
help="Path, glob, or directory of generated videos.")
run_p.add_argument("--audios",
type=str,
nargs="+",
required=False,
help="Path, glob, or directory of generated audio files.")
run_p.add_argument("--reference",
type=str,
default=None,
help="Path / glob / dir of reference videos (for paired metrics).")
run_p.add_argument(
"--reference-audio",
type=str,
nargs="+",
default=None,
help="Path, glob, or directory of reference audio files.",
)
run_p.add_argument("--metrics", type=str, default="all", help="Comma-separated metric names, or 'all'.")
run_p.add_argument("--device", type=str, default="cuda", help="Torch device (e.g. 'cuda', 'cuda:0', 'cpu').")
run_p.add_argument("--num-gpus", type=int, default=1, help="Number of metric replicas for dataset evaluation.")
run_p.add_argument("--compile", action="store_true", help="Compile supported metric models.")
run_p.add_argument(
"--skip-missing-deps",
action="store_true",
help="Skip explicitly requested metrics whose optional dependencies are missing.",
)
run_p.add_argument(
"--extract-audio",
nargs="?",
const=True,
default=False,
help="Extract missing audio tracks; optionally provide a persistent cache directory.",
)
run_p.add_argument("--extract-workers", type=int, default=4)
run_p.add_argument(
"--text-prompt",
type=str,
nargs="*",
default=None,
help="Prompt(s) for text-conditioned metrics. One per video.",
)
run_p.add_argument("--fps", type=float, default=None, help="Frame-rate annotation passed to fps-aware metrics.")
run_p.add_argument("--output",
type=str,
default=None,
help="Write results as JSON to this path (default: stdout).")
run_p.add_argument(
"--output-format",
choices=("legacy", "full"),
default="legacy",
help="'full' includes corpus metrics and sample identities.",
)
# `eval v2a`: isolated corpus-level backends for video-to-audio.
v2a_p = sub.add_parser("v2a", help="Evaluate V2A audio with a corpus benchmark backend")
v2a_p.add_argument("--backend", choices=("av-benchmark", ), default="av-benchmark")
v2a_p.add_argument("--audio-dir", required=True, help="Directory containing generated WAV/FLAC files.")
v2a_p.add_argument("--gt-cache", required=True, help="Official ground-truth feature cache directory.")
v2a_p.add_argument(
"--prediction-cache",
default=None,
help="Generated-audio feature cache (default: <audio-dir>/../av_benchmark_cache).",
)
v2a_p.add_argument(
"--output",
default=None,
help="Result JSON (default: <audio-dir>/../av_benchmark_results.json).",
)
v2a_p.add_argument(
"--python-executable",
default=None,
help="Python from an isolated av-benchmark environment.",
)
v2a_p.add_argument("--device", default="cuda")
v2a_p.add_argument("--batch-size", type=int, default=32)
v2a_p.add_argument("--num-workers", type=int, default=0)
v2a_p.add_argument("--audio-length", type=float, default=8.0)
v2a_p.add_argument("--recompute", action="store_true", help="Re-extract the prediction cache.")
v2a_p.add_argument("--skip-video-related", action="store_true", help="Skip ImageBind and DeSync.")
v2a_p.add_argument("--skip-clap", action="store_true", help="Skip LAION/MS CLAP extraction and scores.")
v2a_p.add_argument(
"--align-prediction-keys",
action="store_true",
help="Align uniquely underscore-prefixed prediction ids to official GT cache ids.",
)
# Stash the parser so cmd() can re-print help on no-action.
self._parser = eval_parser # type: ignore[attr-defined]
return cast(FlexibleArgumentParser, eval_parser)</code></pre></details>
fastvideo.entrypoints.cli.eval.cmd_init
cmd_init() -> list[CLISubcommand]
Source
def cmd_init() -> list[CLISubcommand]:
return [EvalSubcommand()]