Skip to content
FastVideo
Esc
↑↓navigate↵open⌘Jpreview
On this page

pose

Pose processing utilities for HYWorld video generation.

This module provides functions to convert camera poses to model input tensors, including viewmats, intrinsics, and action labels.

Adapted from HY-WorldPlay: https://github.com/Tencent-Hunyuan/HY-WorldPlay

fastvideo.models.dits.hyworld.pose.generate_camera_trajectory_local

generate_camera_trajectory_local(motions)

list of dict

{"forward": 1.0}, {"yaw": np.pi/2}, {"pitch": np.pi/6}, {"right": 1.0} - forward: Translation (Forward or Backward) - yaw: Rotate (Left or Right) - pitch: Rotate (Up or Down) - right: Translation (Right or Left) - third_yaw: Third Perspective Rotate (Left or Right)

fastvideo.models.dits.hyworld.pose.one_hot_to_one_dimension

one_hot_to_one_dimension(one_hot: torch.Tensor) -> torch.Tensor

Convert one-hot action encoding to single dimension labels.

Source
def one_hot_to_one_dimension(one_hot: torch.Tensor) -> torch.Tensor:
    """Convert one-hot action encoding to single dimension labels."""
    return torch.tensor([mapping[tuple(row.tolist())] for row in one_hot])

fastvideo.models.dits.hyworld.pose.parse_pose_string

parse_pose_string(pose_string: str, forward_speed: float = DEFAULT_FORWARD_SPEED, yaw_speed: float = DEFAULT_YAW_SPEED, pitch_speed: float = DEFAULT_PITCH_SPEED) -> list[dict]

Parse pose string to motions list.

Format: "w-3, right-0.5, d-4" - w: forward movement - s: backward movement - a: left movement - d: right movement - up: pitch up rotation - down: pitch down rotation - left: yaw left rotation - right: yaw right rotation - number after dash: duration in frames/latents

Parameters

pose_string: str

Comma-separated pose commands

forward_speed: float

Movement amount per frame

yaw_speed: float

Yaw rotation amount per frame (radians)

pitch_speed: float

Pitch rotation amount per frame (radians)

Returns

list[dict]

List of motion dictionaries for generate_camera_trajectory_local

Source
def parse_pose_string(
    pose_string: str,
    forward_speed: float = DEFAULT_FORWARD_SPEED,
    yaw_speed: float = DEFAULT_YAW_SPEED,
    pitch_speed: float = DEFAULT_PITCH_SPEED,
) -> list[dict]:
    """
    Parse pose string to motions list.

Format: "w-3, right-0.5, d-4"
- w: forward movement
- s: backward movement
- a: left movement
- d: right movement
- up: pitch up rotation
- down: pitch down rotation
- left: yaw left rotation
- right: yaw right rotation
- number after dash: duration in frames/latents

Args:
    pose_string: Comma-separated pose commands
    forward_speed: Movement amount per frame
    yaw_speed: Yaw rotation amount per frame (radians)
    pitch_speed: Pitch rotation amount per frame (radians)

Returns:
    List of motion dictionaries for generate_camera_trajectory_local
"""
motions = []
commands = [cmd.strip() for cmd in pose_string.split(",")]

for cmd in commands:
    if not cmd:
        continue

    parts = cmd.split("-")
    if len(parts) != 2:
        raise ValueError(f"Invalid pose command: {cmd}. Expected format: 'action-duration'")

    action = parts[0].strip()
    try:
        duration = float(parts[1].strip())
    except ValueError:
        raise ValueError(f"Invalid duration in command: {cmd}")

    num_frames = int(duration)

    # Parse action and create motion dicts
    if action == "w":
        # Forward
        for _ in range(num_frames):
            motions.append({"forward": forward_speed})
    elif action == "s":
        # Backward
        for _ in range(num_frames):
            motions.append({"forward": -forward_speed})
    elif action == "a":
        # Left
        for _ in range(num_frames):
            motions.append({"right": -forward_speed})
    elif action == "d":
        # Right
        for _ in range(num_frames):
            motions.append({"right": forward_speed})
    elif action == "up":
        # Pitch up
        for _ in range(num_frames):
            motions.append({"pitch": pitch_speed})
    elif action == "down":
        # Pitch down
        for _ in range(num_frames):
            motions.append({"pitch": -pitch_speed})
    elif action == "left":
        # Yaw left
        for _ in range(num_frames):
            motions.append({"yaw": -yaw_speed})
    elif action == "right":
        # Yaw right
        for _ in range(num_frames):
            motions.append({"yaw": yaw_speed})
    else:
        raise ValueError(f"Unknown action: {action}. "
                         f"Supported actions: w, s, a, d, up, down, left, right")

return motions</code></pre></details>


fastvideo.models.dits.hyworld.pose.pose_string_to_json


pose_string_to_json(pose_string: str, intrinsic: Optional[list[list[float]]] = None) -> dict


Convert pose string to pose JSON format.


Parameters




pose_string: str
Comma-separated pose commands


intrinsic: Optional[list[list[float]]]
Camera intrinsic matrix (default: DEFAULT_INTRINSIC from trajectory)




Returns




dict
Dict with frame indices as keys, containing extrinsic and K (intrinsic) matrices




Source
def pose_string_to_json(
    pose_string: str,
    intrinsic: Optional[list[list[float]]] = None,
) -> dict:
    """
    Convert pose string to pose JSON format.

Args:
    pose_string: Comma-separated pose commands
    intrinsic: Camera intrinsic matrix (default: DEFAULT_INTRINSIC from trajectory)

Returns:
    Dict with frame indices as keys, containing extrinsic and K (intrinsic) matrices
&quot;&quot;&quot;
if intrinsic is None:
    intrinsic = DEFAULT_INTRINSIC

motions = parse_pose_string(pose_string)
poses = generate_camera_trajectory_local(motions)

pose_json = &#123;&#125;
for i, p in enumerate(poses):
    pose_json&#91;str(i)&#93; = &#123;&quot;extrinsic&quot;: p.tolist(), &quot;K&quot;: intrinsic&#125;

return pose_json</code></pre></details>


fastvideo.models.dits.hyworld.pose.pose_to_input


pose_to_input(pose_data: Union[str, dict], latent_num: int, tps: bool = False) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]


Convert pose data to model input tensors.


Parameters




pose_data: Union[str, dict]
One of:
- str ending with '.json': path to JSON file
- str: pose string (e.g., "w-3, right-0.5, d-4")
- dict: pose JSON data


latent_num: int
Number of latents (frames in latent space)


tps: bool
Third-person mode flag




Returns




tuple[torch.Tensor, torch.Tensor, torch.Tensor]
Tuple of (viewmats, intrinsics, action_labels):
- viewmats: World-to-camera matrices [T, 4, 4]
- intrinsics: Normalized camera intrinsics [T, 3, 3]
- action_labels: Action labels for each frame [T]




Source
def pose_to_input(
    pose_data: Union[str, dict],
    latent_num: int,
    tps: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    """
    Convert pose data to model input tensors.

Args:
    pose_data: One of:
        - str ending with &#x27;.json&#x27;: path to JSON file
        - str: pose string (e.g., &quot;w-3, right-0.5, d-4&quot;)
        - dict: pose JSON data
    latent_num: Number of latents (frames in latent space)
    tps: Third-person mode flag

Returns:
    Tuple of (viewmats, intrinsics, action_labels):
        - viewmats: World-to-camera matrices &#91;T, 4, 4&#93;
        - intrinsics: Normalized camera intrinsics &#91;T, 3, 3&#93;
        - action_labels: Action labels for each frame &#91;T&#93;
&quot;&quot;&quot;
# Handle different input types
if isinstance(pose_data, str):
    if pose_data.endswith(&quot;.json&quot;):
        # Load from JSON file
        with open(pose_data, &quot;r&quot;) as f:
            pose_json = json.load(f)
    else:
        # Parse pose string
        pose_json = pose_string_to_json(pose_data)
elif isinstance(pose_data, dict):
    pose_json = pose_data
else:
    raise ValueError(f&quot;Invalid pose_data type: &#123;type(pose_data)&#125;. Expected str or dict.&quot;)

pose_keys = list(pose_json.keys())
latent_num_from_pose = len(pose_keys)
assert latent_num_from_pose == latent_num, (
    f&quot;pose corresponds to &#123;latent_num_from_pose * 4 - 3&#125; frames, num_frames &quot;
    f&quot;must be set to &#123;latent_num_from_pose * 4 - 3&#125; to ensure alignment.&quot;)

intrinsic_list = &#91;&#93;
w2c_list = &#91;&#93;
for i in range(latent_num):
    t_key = pose_keys&#91;i&#93;
    c2w = np.array(pose_json&#91;t_key&#93;&#91;&quot;extrinsic&quot;&#93;)
    w2c = np.linalg.inv(c2w)
    w2c_list.append(w2c)

    # Normalize intrinsics
    intrinsic = np.array(pose_json&#91;t_key&#93;&#91;&quot;K&quot;&#93;)
    intrinsic&#91;0, 0&#93; /= intrinsic&#91;0, 2&#93; * 2
    intrinsic&#91;1, 1&#93; /= intrinsic&#91;1, 2&#93; * 2
    intrinsic&#91;0, 2&#93; = 0.5
    intrinsic&#91;1, 2&#93; = 0.5
    intrinsic_list.append(intrinsic)

w2c_list = np.array(w2c_list)
intrinsic_list = torch.tensor(np.array(intrinsic_list))

# Compute relative camera-to-world transforms
c2ws = np.linalg.inv(w2c_list)
C_inv = np.linalg.inv(c2ws&#91;:-1&#93;)
relative_c2w = np.zeros_like(c2ws)
relative_c2w&#91;0, ...&#93; = c2ws&#91;0, ...&#93;
relative_c2w&#91;1:, ...&#93; = C_inv @ c2ws&#91;1:, ...&#93;

# Initialize one-hot action encodings
trans_one_hot = np.zeros((relative_c2w.shape&#91;0&#93;, 4), dtype=np.int32)
rotate_one_hot = np.zeros((relative_c2w.shape&#91;0&#93;, 4), dtype=np.int32)

move_norm_valid = 0.0001
for i in range(1, relative_c2w.shape&#91;0&#93;):
    move_dirs = relative_c2w&#91;i, :3, 3&#93;  # direction vector
    move_norms = np.linalg.norm(move_dirs)

    if move_norms &gt; move_norm_valid:  # threshold for movement
        move_norm_dirs = move_dirs / move_norms
        angles_rad = np.arccos(move_norm_dirs.clip(-1.0, 1.0))
        trans_angles_deg = angles_rad * (180.0 / np.pi)  # convert to degrees
    else:
        trans_angles_deg = np.zeros(3)

    R_rel = relative_c2w&#91;i, :3, :3&#93;
    r = R.from_matrix(R_rel)
    rot_angles_deg = r.as_euler(&quot;xyz&quot;, degrees=True)

    # Determine movement and rotation actions
    if move_norms &gt; move_norm_valid:  # threshold for movement
        if (not tps) or (tps and abs(rot_angles_deg&#91;1&#93;) &lt; 5e-2 and abs(rot_angles_deg&#91;0&#93;) &lt; 5e-2):
            if trans_angles_deg&#91;2&#93; &lt; 60:
                trans_one_hot&#91;i, 0&#93; = 1  # forward
            elif trans_angles_deg&#91;2&#93; &gt; 120:
                trans_one_hot&#91;i, 1&#93; = 1  # backward

            if trans_angles_deg&#91;0&#93; &lt; 60:
                trans_one_hot&#91;i, 2&#93; = 1  # right
            elif trans_angles_deg&#91;0&#93; &gt; 120:
                trans_one_hot&#91;i, 3&#93; = 1  # left

    if rot_angles_deg&#91;1&#93; &gt; 5e-2:
        rotate_one_hot&#91;i, 0&#93; = 1  # right
    elif rot_angles_deg&#91;1&#93; &lt; -5e-2:
        rotate_one_hot&#91;i, 1&#93; = 1  # left

    if rot_angles_deg&#91;0&#93; &gt; 5e-2:
        rotate_one_hot&#91;i, 2&#93; = 1  # up
    elif rot_angles_deg&#91;0&#93; &lt; -5e-2:
        rotate_one_hot&#91;i, 3&#93; = 1  # down

trans_one_hot = torch.tensor(trans_one_hot)
rotate_one_hot = torch.tensor(rotate_one_hot)

# Convert one-hot to single-dimension labels
trans_one_label = one_hot_to_one_dimension(trans_one_hot)
rotate_one_label = one_hot_to_one_dimension(rotate_one_hot)
action_one_label = trans_one_label * 9 + rotate_one_label

return (
    torch.as_tensor(w2c_list),
    torch.as_tensor(intrinsic_list),
    action_one_label,
)</code></pre></details>


fastvideo.models.dits.hyworld.pose.camera_center_normalization


camera_center_normalization(w2c: np.ndarray) -> np.ndarray


Normalize camera centers relative to the first camera.


Source
def camera_center_normalization(w2c: np.ndarray) -> np.ndarray:
    """Normalize camera centers relative to the first camera."""
    c2w = np.linalg.inv(w2c)
    C0_inv = np.linalg.inv(c2w[0])
    c2w_aligned = np.array([C0_inv @ C for C in c2w])
    return np.linalg.inv(c2w_aligned)


fastvideo.models.dits.hyworld.pose.parse_pose_string_to_actions


parse_pose_string_to_actions(pose_string: str, fps: int = 24) -> list[dict]


Parse pose string to frame-level action timeline.


Format: pose string uses latent counts, where:
- 1 latent = 4 frames
- Special rule: first frame of entire video is extra (frame 0)
- Example: "w-4,d-4" means:
  - w-4: forward for frames 0-16 (17 frames total: 1 extra + 44)
  - d-4: right for frames 17-32 (16 frames total: 44)


Parameters




pose_string: str
Comma-separated pose commands (e.g., "w-4,d-4")


fps: int
Frames per second for video (default: 24)




Returns




list[dict]
List of dicts with action values for each frame




Source
def parse_pose_string_to_actions(pose_string: str, fps: int = 24) -> list[dict]:
    """
    Parse pose string to frame-level action timeline.

Format: pose string uses latent counts, where:
- 1 latent = 4 frames
- Special rule: first frame of entire video is extra (frame 0)
- Example: &quot;w-4,d-4&quot; means:
  - w-4: forward for frames 0-16 (17 frames total: 1 extra + 4*4)
  - d-4: right for frames 17-32 (16 frames total: 4*4)

Args:
    pose_string: Comma-separated pose commands (e.g., &quot;w-4,d-4&quot;)
    fps: Frames per second for video (default: 24)

Returns:
    List of dicts with action values for each frame
&quot;&quot;&quot;
commands = &#91;cmd.strip() for cmd in pose_string.split(&quot;,&quot;)&#93;

frame_actions = &#91;&#93;
is_first_command = True

for cmd in commands:
    if not cmd:
        continue

    parts = cmd.split(&quot;-&quot;)
    if len(parts) != 2:
        raise ValueError(f&quot;Invalid pose command: &#123;cmd&#125;. Expected format: &#x27;action-duration&#x27;&quot;)

    action = parts&#91;0&#93;.strip()
    try:
        num_latents = int(parts&#91;1&#93;.strip())
    except ValueError:
        raise ValueError(f&quot;Invalid duration in command: &#123;cmd&#125;&quot;)

    # Convert latents to frames
    # First command gets 1 extra frame (the special frame 0)
    if is_first_command:
        num_frames = 1 + num_latents * 4
        is_first_command = False
    else:
        num_frames = num_latents * 4

    # Map action to action values
    action_values = &#123;&quot;forward&quot;: 0, &quot;left&quot;: 0, &quot;yaw&quot;: 0, &quot;pitch&quot;: 0&#125;

    if action == &quot;w&quot;:
        action_values&#91;&quot;forward&quot;&#93; = 1
    elif action == &quot;s&quot;:
        action_values&#91;&quot;forward&quot;&#93; = -1
    elif action == &quot;a&quot;:
        action_values&#91;&quot;left&quot;&#93; = 1
    elif action == &quot;d&quot;:
        action_values&#91;&quot;left&quot;&#93; = -1
    elif action == &quot;up&quot;:
        action_values&#91;&quot;pitch&quot;&#93; = 1
    elif action == &quot;down&quot;:
        action_values&#91;&quot;pitch&quot;&#93; = -1
    elif action == &quot;left&quot;:
        action_values&#91;&quot;yaw&quot;&#93; = -1
    elif action == &quot;right&quot;:
        action_values&#91;&quot;yaw&quot;&#93; = 1
    else:
        raise ValueError(f&quot;Unknown action: &#123;action&#125;&quot;)

    # Add frame-level actions
    for _ in range(num_frames):
        frame_actions.append(action_values.copy())

return frame_actions</code></pre></details>


fastvideo.models.dits.hyworld.pose.compute_latent_num


compute_latent_num(num_frames: int) -> int


Compute the number of latents from number of frames.


Formula: num_frames = (latent_num - 1) * 4 + 1
So: latent_num = (num_frames - 1) // 4 + 1


Parameters




num_frames: int
Number of video frames




Returns




int
Number of latents




Source
def compute_latent_num(num_frames: int) -> int:
    """
    Compute the number of latents from number of frames.

Formula: num_frames = (latent_num - 1) * 4 + 1
So: latent_num = (num_frames - 1) // 4 + 1

Args:
    num_frames: Number of video frames

Returns:
    Number of latents
&quot;&quot;&quot;
return (num_frames - 1) // 4 + 1</code></pre></details>


fastvideo.models.dits.hyworld.pose.compute_num_frames


compute_num_frames(latent_num: int) -> int


Compute the number of frames from number of latents.


Formula: num_frames = (latent_num - 1) * 4 + 1


Parameters




latent_num: int
Number of latents




Returns




int
Number of video frames




Source
def compute_num_frames(latent_num: int) -> int:
    """
    Compute the number of frames from number of latents.

Formula: num_frames = (latent_num - 1) * 4 + 1

Args:
    latent_num: Number of latents

Returns:
    Number of video frames
&quot;&quot;&quot;
return (latent_num - 1) * 4 + 1</code></pre></details>

Was this page helpful?