Skip to content

stages

Inference stages for MMAudio V2A/T2A.

The video sampler intentionally mirrors mmaudio.data.av_utils.read_frames: timestamps are sampled independently at 8 FPS and 25 FPS, and a decoded frame is repeated when the source FPS is lower than a requested sampling rate.

Classes

fastvideo.pipelines.basic.mmaudio.stages.MMAudioDecodingStage 

MMAudioDecodingStage(audio_vae, vocoder)

Bases: PipelineStage

Decode MMAudio latents to a mono 44.1 kHz waveform.

Source code in fastvideo/pipelines/basic/mmaudio/stages.py 
414
415
416
417
def __init__(self, audio_vae, vocoder) -> None:
    super().__init__()
    self.audio_vae = audio_vae
    self.vocoder = vocoder

fastvideo.pipelines.basic.mmaudio.stages.MMAudioDenoisingStage 

MMAudioDenoisingStage(transformer, scheduler)

Bases: PipelineStage

Run MMAudio's forward-time Euler flow with FastVideo's shared scheduler.

Source code in fastvideo/pipelines/basic/mmaudio/stages.py 
368
369
370
371
def __init__(self, transformer, scheduler) -> None:
    super().__init__()
    self.transformer = transformer
    self.scheduler = scheduler

fastvideo.pipelines.basic.mmaudio.stages.MMAudioInputValidationStage

Bases: PipelineStage

Validate audio-generation inputs without invoking video pipeline logic.

fastvideo.pipelines.basic.mmaudio.stages.MMAudioLatentPreparationStage 

MMAudioLatentPreparationStage(transformer)

Bases: PipelineStage

Sample the Gaussian flow prior using a device-local seeded generator.

Source code in fastvideo/pipelines/basic/mmaudio/stages.py 
338
339
340
def __init__(self, transformer) -> None:
    super().__init__()
    self.transformer = transformer

fastvideo.pipelines.basic.mmaudio.stages.MMAudioTextConditioningStage 

MMAudioTextConditioningStage(text_encoder, tokenizer, transformer)

Bases: PipelineStage

Encode positive/negative OpenCLIP token sequences and project conditions.

Source code in fastvideo/pipelines/basic/mmaudio/stages.py 
278
279
280
281
282
def __init__(self, text_encoder, tokenizer, transformer) -> None:
    super().__init__()
    self.text_encoder = text_encoder
    self.tokenizer = tokenizer
    self.transformer = transformer

fastvideo.pipelines.basic.mmaudio.stages.MMAudioVideoConditioningStage 

MMAudioVideoConditioningStage(image_encoder, sync_encoder, transformer)

Bases: PipelineStage

Decode video and run DFN5B/Synchformer with official preprocessing.

Source code in fastvideo/pipelines/basic/mmaudio/stages.py 
174
175
176
177
178
def __init__(self, image_encoder, sync_encoder, transformer) -> None:
    super().__init__()
    self.image_encoder = image_encoder
    self.sync_encoder = sync_encoder
    self.transformer = transformer

Functions:

fastvideo.pipelines.basic.mmaudio.stages.preprocess_mmaudio_video 

preprocess_mmaudio_video(video_path: str | Path, *, duration_s: float, clip_fps: int = 8, sync_fps: int = 25, clip_size: int = 384, sync_size: int = 224) -> tuple[Tensor, Tensor, float]

Return official-format CLIP frames, sync frames, and effective duration.

CLIP output is float32 [T,3,384,384] in [0,1]. Synchformer output is float32 [T,3,224,224] in [-1,1].

Source code in fastvideo/pipelines/basic/mmaudio/stages.py 
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
def preprocess_mmaudio_video(
    video_path: str | Path,
    *,
    duration_s: float,
    clip_fps: int = 8,
    sync_fps: int = 25,
    clip_size: int = 384,
    sync_size: int = 224,
) -> tuple[torch.Tensor, torch.Tensor, float]:
    """Return official-format CLIP frames, sync frames, and effective duration.

    CLIP output is float32 ``[T,3,384,384]`` in ``[0,1]``. Synchformer
    output is float32 ``[T,3,224,224]`` in ``[-1,1]``.
    """
    clip_array, sync_array = _read_frames_at_fps(
        video_path,
        (float(clip_fps), float(sync_fps)),
        start_s=0.0,
        end_s=duration_s,
    )
    clip_frames = torch.from_numpy(clip_array).permute(0, 3, 1, 2)
    sync_frames = torch.from_numpy(sync_array).permute(0, 3, 1, 2)

    clip_transform = v2.Compose([
        v2.Resize((clip_size, clip_size), interpolation=v2.InterpolationMode.BICUBIC),
        v2.ToImage(),
        v2.ToDtype(torch.float32, scale=True),
    ])
    sync_transform = v2.Compose([
        v2.Resize(sync_size, interpolation=v2.InterpolationMode.BICUBIC),
        v2.CenterCrop(sync_size),
        v2.ToImage(),
        v2.ToDtype(torch.float32, scale=True),
        v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ])
    clip_frames = clip_transform(clip_frames)
    sync_frames = sync_transform(sync_frames)

    effective_duration = min(
        duration_s,
        clip_frames.shape[0] / clip_fps,
        sync_frames.shape[0] / sync_fps,
    )
    clip_frames = clip_frames[:int(clip_fps * effective_duration)]
    sync_frames = sync_frames[:int(sync_fps * effective_duration)]
    return clip_frames, sync_frames, effective_duration