diff --git a/egomimic/scripts/aria_process/aria_helper.py b/egomimic/scripts/aria_process/aria_helper.py index 14aa7b9e..580b621d 100644 --- a/egomimic/scripts/aria_process/aria_helper.py +++ b/egomimic/scripts/aria_process/aria_helper.py @@ -53,7 +53,8 @@ def zarr_job( dataset_name: str, arm: str, description: str = "", -) -> tuple[Path, Path] | None: + save_mp4: bool = True, +) -> tuple[Path, Path | None] | None: """ Convert one trio to a Zarr dataset. """ @@ -71,9 +72,9 @@ def zarr_job( nthreads=2, debug=False, benchmark=False, - save_mp4=True, + save_mp4=save_mp4, description=description, dataset_name=dataset_name ) - return aria_zarr_main(args) \ No newline at end of file + return aria_zarr_main(args) diff --git a/egomimic/scripts/aria_process/aria_to_zarr.py b/egomimic/scripts/aria_process/aria_to_zarr.py index 3e2178a5..963635c2 100644 --- a/egomimic/scripts/aria_process/aria_to_zarr.py +++ b/egomimic/scripts/aria_process/aria_to_zarr.py @@ -23,7 +23,6 @@ compute_orientation_rotation_matrix, slam_to_rgb, undistort_to_linear, - cpf_to_rgb ) from lerobot.common.datasets.lerobot_dataset import LEROBOT_HOME from projectaria_tools.core import data_provider, mps @@ -518,13 +517,16 @@ def process_episode(episode_path, arm: str, low_res=False, benchmark=False): rgb_timestamps_ns = np.array(stream_timestamps_ns["rgb"]) print(f"[DEBUG] LENGTH BEFORE CLEANING: {len(hand_cartesian_pose)}") - [hand_cartesian_pose, hand_keypoints_pose, head_pose], images, eye_gaze, rgb_timestamps_ns = ( - AriaVRSExtractor.clean_data( - poses=[hand_cartesian_pose, hand_keypoints_pose, head_pose], - images=images, - eye_gaze=eye_gaze, - timestamps=rgb_timestamps_ns - ) + ( + [hand_cartesian_pose, hand_keypoints_pose, head_pose], + images, + eye_gaze, + rgb_timestamps_ns, + ) = AriaVRSExtractor.clean_data( + poses=[hand_cartesian_pose, hand_keypoints_pose, head_pose], + images=images, + eye_gaze=eye_gaze, + timestamps=rgb_timestamps_ns, ) # actions, pose, images = AriaVRSExtractor.clean_data_projection(actions=actions, pose=pose, images=images, arm=arm) print(f"[DEBUG] LENGTH AFTER CLEANING: {len(hand_cartesian_pose)}") @@ -1598,19 +1600,21 @@ def extract_episode( enable_sharding=False, task="", ) - mp4_path = output_dir / f"{episode_name}.mp4" - W, H = 960, 720 - p = start_ffmpeg_mp4(mp4_path, W, H, fps=30, pix_fmt="rgb24") - for video_images in AriaVRSExtractor.iter_images( - episode_path, chunk_length=256, height=H, width=W, focal_mult=3 - ): - for image in video_images: - image = prep_frame(image, H, W) - if image is None: - continue - p.stdin.write(image.tobytes()) - p.stdin.close() - p.wait() + mp4_path = None + if self.save_mp4: + mp4_path = output_dir / f"{episode_name}.mp4" + W, H = 960, 720 + p = start_ffmpeg_mp4(mp4_path, W, H, fps=30, pix_fmt="rgb24") + for video_images in AriaVRSExtractor.iter_images( + episode_path, chunk_length=256, height=H, width=W, focal_mult=3 + ): + for image in video_images: + image = prep_frame(image, H, W) + if image is None: + continue + p.stdin.write(image.tobytes()) + p.stdin.close() + p.wait() return zarr_path, mp4_path def extract_episodes( diff --git a/egomimic/scripts/aria_process/aria_utils.py b/egomimic/scripts/aria_process/aria_utils.py index c91fea82..dc60092c 100644 --- a/egomimic/scripts/aria_process/aria_utils.py +++ b/egomimic/scripts/aria_process/aria_utils.py @@ -27,11 +27,23 @@ def build_camera_matrix(provider, pose_t): return T_world_rgb_camera -def undistort_to_linear(provider, stream_ids, raw_image, camera_label="rgb", height=480, width=640, focal_mult=2): +def undistort_to_linear( + provider, + stream_ids, + raw_image, + camera_label="rgb", + height=480, + width=640, + focal_mult=2, +): camera_label = provider.get_label_from_stream_id(stream_ids[camera_label]) calib = provider.get_device_calibration().get_camera_calib(camera_label) warped = calibration.get_linear_camera_calibration( - height, width, 133.25430222 * focal_mult, camera_label, calib.get_transform_device_camera() + height, + width, + 133.25430222 * focal_mult, + camera_label, + calib.get_transform_device_camera(), ) warped_image = calibration.distort_by_calibration(raw_image, warped, calib) warped_rot = np.rot90(warped_image, k=3) @@ -106,6 +118,7 @@ def slam_to_rgb(provider): return transform + def compute_orientation_rotation_matrix(palm_pose, wrist_pose, palm_normal): x_axis = wrist_pose - palm_pose x_axis = np.ravel(x_axis) / np.linalg.norm(x_axis) @@ -119,6 +132,7 @@ def compute_orientation_rotation_matrix(palm_pose, wrist_pose, palm_normal): rot_matrix = np.column_stack([-1 * x_axis, y_axis, z_axis]) return rot_matrix + def coordinate_frame_to_ypr(x_axis, y_axis, z_axis): rot_matrix = np.column_stack([x_axis, y_axis, z_axis]) rotation = R.from_matrix(rot_matrix) @@ -127,6 +141,7 @@ def coordinate_frame_to_ypr(x_axis, y_axis, z_axis): euler_ypr = np.zeros_like(euler_ypr) return euler_ypr + def cpf_to_rgb(provider): """ Get cpf (eye tracking origin) to rgb camera transform (rotated upright) diff --git a/egomimic/scripts/aria_process/download_and_convert_aria_sql.py b/egomimic/scripts/aria_process/download_and_convert_aria_sql.py new file mode 100644 index 00000000..1acb014c --- /dev/null +++ b/egomimic/scripts/aria_process/download_and_convert_aria_sql.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import shutil +from pathlib import Path + +import pandas as pd + +from egomimic.scripts.aria_process.aria_helper import zarr_job +from egomimic.utils.aws.aws_data_utils import ( + get_boto3_s3_client, + load_env, + s3_sync_to_local, +) +from egomimic.utils.aws.aws_sql import ( + create_default_engine, + episode_hash_to_timestamp_ms, + episode_table_to_df, +) + +RAW_REMOTE_PREFIX = os.environ.get( + "RAW_REMOTE_PREFIX", "s3://rldb/raw_v2/aria/" +).rstrip("/") +BUCKET = os.environ.get("BUCKET", "rldb") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Query the SQL episode table, filter rows, download matching Aria bundles " + "from S3/R2, and convert them into local zarr datasets." + ) + ) + parser.add_argument( + "--download-dir", + type=Path, + required=True, + help="Local root directory for downloaded raw bundles and converted zarrs.", + ) + parser.add_argument( + "--raw-remote-prefix", + type=str, + default=RAW_REMOTE_PREFIX, + help="Remote raw Aria prefix, e.g. s3://rldb/raw_v2/aria", + ) + parser.add_argument( + "--bucket", + type=str, + default=BUCKET, + help="Default bucket to use when --raw-remote-prefix is not a full s3:// URI.", + ) + parser.add_argument( + "--where", + action="append", + default=[], + metavar="COLUMN=VALUE", + help="Exact-match dataframe filter. Repeat as needed.", + ) + parser.add_argument( + "--contains", + action="append", + default=[], + metavar="COLUMN=TEXT", + help="Substring dataframe filter. Repeat as needed.", + ) + parser.add_argument( + "--query", + type=str, + default="", + help="Optional pandas query string applied after the simple filters.", + ) + parser.add_argument( + "--episode-hash", + action="append", + default=[], + help="Explicit episode_hash values to keep. Repeat as needed.", + ) + parser.add_argument( + "--episode-hash-file", + type=Path, + help="Optional text file containing one episode_hash per line.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Maximum number of filtered rows to process.", + ) + parser.add_argument( + "--sort-by", + type=str, + default="episode_hash", + help="Column used to sort the filtered dataframe.", + ) + parser.add_argument( + "--descending", + action="store_true", + help="Sort descending instead of ascending.", + ) + parser.add_argument( + "--arm-override", + choices=["left", "right", "bimanual"], + help="Override arm inference from robot_name.", + ) + parser.add_argument( + "--print-df", + action="store_true", + help="Print the filtered dataframe preview before processing.", + ) + parser.add_argument( + "--save-df", + type=Path, + help="Optional CSV path for the filtered dataframe.", + ) + parser.add_argument( + "--skip-download-existing", + action="store_true", + help="Skip downloading a bundle if the expected local files already exist.", + ) + parser.add_argument( + "--skip-convert-existing", + action="store_true", + help="Skip conversion if the target zarr already exists.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Only show matches and planned paths; do not download or convert.", + ) + parser.add_argument( + "--no-mp4", + action="store_true", + help="Skip mp4 preview generation during zarr conversion.", + ) + parser.add_argument( + "--remove-downloaded-raw", + action="store_true", + help="Remove the downloaded local raw bundle after a successful zarr conversion.", + ) + return parser.parse_args() + + +def parse_s3_uri(uri: str, *, default_bucket: str) -> tuple[str, str]: + uri = uri.strip() + if uri.startswith("s3://"): + rest = uri[len("s3://") :] + bucket, _, prefix = rest.partition("/") + return bucket, prefix.strip("/") + return default_bucket, uri.strip("/") + + +def parse_assignment(raw: str) -> tuple[str, str]: + if "=" not in raw: + raise ValueError(f"Expected COLUMN=VALUE, got: {raw}") + key, value = raw.split("=", 1) + key = key.strip() + if not key: + raise ValueError(f"Missing column name in: {raw}") + return key, value.strip() + + +def maybe_coerce_scalar(value: str): + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + if lowered == "none" or lowered == "null": + return None + try: + if "." in value: + return float(value) + return int(value) + except ValueError: + return value + + +def read_episode_hashes(args: argparse.Namespace) -> list[str]: + hashes = list(args.episode_hash) + if args.episode_hash_file: + hashes.extend( + line.strip() + for line in args.episode_hash_file.read_text().splitlines() + if line.strip() + ) + return hashes + + +def apply_filters(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame: + filtered = df.copy() + + for raw in args.where: + column, value = parse_assignment(raw) + if column not in filtered.columns: + raise KeyError(f"Unknown column for --where: {column}") + filtered = filtered[filtered[column] == maybe_coerce_scalar(value)] + + for raw in args.contains: + column, value = parse_assignment(raw) + if column not in filtered.columns: + raise KeyError(f"Unknown column for --contains: {column}") + filtered = filtered[ + filtered[column].fillna("").astype(str).str.contains(value, regex=False) + ] + + episode_hashes = read_episode_hashes(args) + if episode_hashes: + filtered = filtered[filtered["episode_hash"].isin(episode_hashes)] + + if args.query: + filtered = filtered.query(args.query) + + if args.sort_by: + if args.sort_by not in filtered.columns: + raise KeyError(f"Unknown column for --sort-by: {args.sort_by}") + filtered = filtered.sort_values(args.sort_by, ascending=not args.descending) + + return filtered.reset_index(drop=True) + + +def infer_arm(robot_name: str) -> str: + name = (robot_name or "").lower() + if "left" in name: + return "left" + if "right" in name: + return "right" + return "bimanual" + + +def expected_bundle_paths(bundle_dir: Path, stem: str) -> tuple[Path, Path, Path]: + return ( + bundle_dir / f"{stem}.vrs", + bundle_dir / f"{stem}.json", + bundle_dir / f"mps_{stem}_vrs", + ) + + +def expected_eye_gaze_path(bundle_dir: Path, stem: str) -> Path: + return bundle_dir / f"mps_{stem}_vrs" / "eye_gaze" / "general_eye_gaze.csv" + + +def bundle_exists(bundle_dir: Path, stem: str) -> bool: + vrs_path, json_path, mps_dir = expected_bundle_paths(bundle_dir, stem) + return ( + vrs_path.exists() + and json_path.exists() + and mps_dir.exists() + and (mps_dir / "hand_tracking").exists() + and (mps_dir / "slam").exists() + and expected_eye_gaze_path(bundle_dir, stem).exists() + ) + + +def eye_gaze_exists_in_s3( + *, s3_client, bucket: str, remote_prefix: str, stem: str +) -> bool: + eye_gaze_key = ( + f"{remote_prefix}/mps_{stem}_vrs/eye_gaze/general_eye_gaze.csv".strip("/") + ) + try: + s3_client.head_object(Bucket=bucket, Key=eye_gaze_key) + return True + except Exception: + return False + + +def download_bundle( + *, + s3_client, + bucket: str, + remote_prefix: str, + bundle_dir: Path, + stem: str, +) -> tuple[Path, Path, Path]: + bundle_dir.mkdir(parents=True, exist_ok=True) + file_prefix = f"{remote_prefix}/{stem}".strip("/") + vrs_key = f"{file_prefix}.vrs" + json_key = f"{file_prefix}.json" + mps_key = f"{remote_prefix}/mps_{stem}_vrs".strip("/") + + vrs_path, json_path, mps_dir = expected_bundle_paths(bundle_dir, stem) + + s3_client.download_file(bucket, vrs_key, str(vrs_path)) + s3_client.download_file(bucket, json_key, str(json_path)) + s3_sync_to_local(bucket, mps_key, mps_dir) + + return vrs_path, json_path, mps_dir + + +def dataframe_preview(df: pd.DataFrame) -> str: + if df.empty: + return "" + preview = df[ + [ + c + for c in ["episode_hash", "task", "operator", "robot_name"] + if c in df.columns + ] + ] + return preview.to_string(index=False) + + +def process_rows(df: pd.DataFrame, args: argparse.Namespace) -> list[dict]: + bucket, remote_prefix = parse_s3_uri( + args.raw_remote_prefix, default_bucket=args.bucket + ) + raw_root = args.download_dir / "raw" + zarr_root = args.download_dir / "zarr" + summary: list[dict] = [] + s3_client = get_boto3_s3_client() + if not args.dry_run: + zarr_root.mkdir(parents=True, exist_ok=True) + accepted_count = 0 + + for row in df.itertuples(index=False): + episode_hash = row.episode_hash + stem = str(episode_hash_to_timestamp_ms(episode_hash)) + bundle_dir = raw_root / episode_hash + zarr_path = zarr_root / f"{episode_hash}.zarr" + mp4_path = zarr_root / f"{episode_hash}.mp4" + arm = args.arm_override or infer_arm(getattr(row, "robot_name", "")) + + item = { + "episode_hash": episode_hash, + "timestamp_ms_stem": stem, + "raw_bundle_dir": str(bundle_dir), + "zarr_path": str(zarr_path), + "mp4_path": str(mp4_path), + "arm": arm, + "status": "pending", + } + + try: + if not eye_gaze_exists_in_s3( + s3_client=s3_client, + bucket=bucket, + remote_prefix=remote_prefix, + stem=stem, + ): + item["status"] = "skipped_missing_eye_gaze" + item["error"] = ( + f"Missing mps_{stem}_vrs/eye_gaze/general_eye_gaze.csv in S3" + ) + summary.append(item) + continue + + reuse_existing_bundle = args.skip_download_existing and bundle_exists( + bundle_dir, stem + ) + + if args.limit is not None and accepted_count >= args.limit: + break + + if args.dry_run: + item["status"] = ( + "dry_run_reuse_existing_bundle" + if reuse_existing_bundle + else "dry_run_would_download" + ) + accepted_count += 1 + summary.append(item) + continue + + if not reuse_existing_bundle: + download_bundle( + s3_client=s3_client, + bucket=bucket, + remote_prefix=remote_prefix, + bundle_dir=bundle_dir, + stem=stem, + ) + accepted_count += 1 + + if args.skip_convert_existing and zarr_path.exists(): + item["status"] = "skipped_existing_zarr" + summary.append(item) + continue + + zarr_out, mp4_out = zarr_job( + raw_path=bundle_dir, + output_dir=zarr_root, + dataset_name=episode_hash, + arm=arm, + description=getattr(row, "task_description", "") or "", + save_mp4=not args.no_mp4, + ) + + item["zarr_path"] = str(zarr_out) + item["mp4_path"] = str(mp4_out) if mp4_out is not None else "" + item["status"] = "converted" + if args.remove_downloaded_raw: + shutil.rmtree(bundle_dir, ignore_errors=True) + item["raw_bundle_removed"] = True + except Exception as exc: + item["status"] = "error" + item["error"] = str(exc) + + summary.append(item) + + return summary + + +def main() -> None: + load_env() + args = parse_args() + + engine = create_default_engine() + df = episode_table_to_df(engine) + filtered = apply_filters(df, args) + + print(f"Matched {len(filtered)} episode rows.") + if args.print_df or args.dry_run: + print(dataframe_preview(filtered)) + + if args.save_df: + args.save_df.parent.mkdir(parents=True, exist_ok=True) + filtered.to_csv(args.save_df, index=False) + print(f"Saved filtered dataframe to {args.save_df}") + + episode_hashes = filtered["episode_hash"].tolist() if not filtered.empty else [] + print("Episode hashes:") + print(json.dumps(episode_hashes, indent=2)) + + summary = process_rows(filtered, args) + print("Summary:") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/egomimic/utils/egomimicUtils.py b/egomimic/utils/egomimicUtils.py index be6134af..8427c35c 100644 --- a/egomimic/utils/egomimicUtils.py +++ b/egomimic/utils/egomimicUtils.py @@ -1,13 +1,5 @@ -import subprocess -import numpy as np -import cv2 -import matplotlib.pyplot as plt -import torchvision.transforms.functional as TF -import torch -from scipy.spatial.transform import Rotation -import pytorch_kinematics as pk -import egomimic import os +import subprocess from numbers import Number from pathlib import Path @@ -258,13 +250,13 @@ } ARIA_T_RGB_CPF = np.array( - [ - [-0.99989084, 0.01251132, -0.00786028, 0.05686918], - [-0.01132842, -0.99067146, -0.13580032, 0.00922798], - [-0.009486 , -0.13569645, 0.99070505, -0.01147902], - [0.0, 0.0, 0.0, 1.0] - ] - ) + [ + [-0.99989084, 0.01251132, -0.00786028, 0.05686918], + [-0.01132842, -0.99067146, -0.13580032, 0.00922798], + [-0.009486, -0.13569645, 0.99070505, -0.01147902], + [0.0, 0.0, 0.0, 1.0], + ] +) INTRINSICS = { "base": ARIA_INTRINSICS, @@ -561,12 +553,13 @@ def draw_rotation_text( return image + def render_3d_traj_frames( trajs, labels=None, idx=0, stride=1, - mode="time", # "time" or "rotate" + mode="time", # "time" or "rotate" elev=20, azim_start=0, azim_end=360, @@ -603,7 +596,9 @@ def render_3d_traj_frames( n, t, _ = base.shape for k, a in enumerate(trajs_np): if a.shape != base.shape: - raise ValueError(f"All trajs must share the same shape. traj0={base.shape}, traj{k}={a.shape}") + raise ValueError( + f"All trajs must share the same shape. traj0={base.shape}, traj{k}={a.shape}" + ) if not (0 <= idx < n): raise IndexError(f"idx {idx} out of range for N={n}") @@ -653,13 +648,16 @@ def render_3d_traj_frames( ln.set_data(x, y) ln.set_3d_properties(z) - n_frames = int(abs(azim_end - azim_start)) + 1 if azim_end != azim_start else 360 + n_frames = ( + int(abs(azim_end - azim_start)) + 1 if azim_end != azim_start else 360 + ) azims = np.linspace(azim_start, azim_end, n_frames) def draw_frame(fi): ax.view_init(elev=elev, azim=float(azims[fi])) else: # time mode + def draw_frame(fi): end = fi + 1 start = 0 if tail is None else max(0, end - int(tail)) @@ -672,7 +670,13 @@ def draw_frame(fi): # Render frames into RGB arrays frames = [] canvas = fig.canvas - for fi in range((len(np.linspace(azim_start, azim_end, int(abs(azim_end - azim_start)) + 1)) if mode == "rotate" else t_len)): + for fi in range( + ( + len(np.linspace(azim_start, azim_end, int(abs(azim_end - azim_start)) + 1)) + if mode == "rotate" + else t_len + ) + ): draw_frame(fi) canvas.draw() rgba = np.asarray(canvas.buffer_rgba()) @@ -682,11 +686,12 @@ def draw_frame(fi): plt.close(fig) return frames + def render_3d_traj_frames_NT3( trajs, labels=None, stride=1, - mode="time", # "time" or "rotate" + mode="time", # "time" or "rotate" elev=20, azim_start=0, azim_end=360, @@ -730,7 +735,9 @@ def render_3d_traj_frames_NT3( n_frames, t_chunk, _ = base.shape for k, a in enumerate(trajs_np): if a.shape != base.shape: - raise ValueError(f"All trajs must share shape. traj0={base.shape}, traj{k}={a.shape}") + raise ValueError( + f"All trajs must share shape. traj0={base.shape}, traj{k}={a.shape}" + ) if labels is None: labels = [f"traj{k}" for k in range(len(trajs_np))] @@ -786,7 +793,9 @@ def set_line_from_xyz(ln, xyz): for ln, xyz in zip(lines, xyzs): set_line_from_xyz(ln, xyz) - n_frames_rot = int(abs(azim_end - azim_start)) + 1 if azim_end != azim_start else 360 + n_frames_rot = ( + int(abs(azim_end - azim_start)) + 1 if azim_end != azim_start else 360 + ) azims = np.linspace(azim_start, azim_end, n_frames_rot) def draw_frame(fi): @@ -804,7 +813,7 @@ def draw_frame(fi): else: start_f = max(0, fi + 1 - int(tail)) for ln, a in zip(lines, trajs_np): - xyz = a[ns[start_f:fi + 1]].reshape(-1, 3) # (tail*T, 3) + xyz = a[ns[start_f : fi + 1]].reshape(-1, 3) # (tail*T, 3) set_line_from_xyz(ln, xyz) out_len = n_len @@ -820,10 +829,14 @@ def draw_frame(fi): plt.close(fig) return frames + def xyzw_to_wxyz(xyzw): return np.concatenate([xyzw[..., 3:4], xyzw[..., :3]], axis=-1) -def draw_actions(im, type, color, actions, extrinsics, intrinsics, arm="both", kinematics_solver=None): + +def draw_actions( + im, type, color, actions, extrinsics, intrinsics, arm="both", kinematics_solver=None +): """ args: im: (H, W, C) @@ -1023,6 +1036,7 @@ def ee_orientation_to_cam_frame(ee_orientation_base, T_cam_base): ) return ee_orientation_cam, batched_ypr + def prep_frame(frame: np.ndarray, H: int, W: int) -> np.ndarray | None: if frame is None: print("Frame is None") @@ -1038,22 +1052,35 @@ def prep_frame(frame: np.ndarray, H: int, W: int) -> np.ndarray | None: frame = np.ascontiguousarray(frame) return frame -def start_ffmpeg_mp4(out_path: str, width: int, height: int, fps: int, pix_fmt: str = "rgb24"): + +def start_ffmpeg_mp4( + out_path: str, width: int, height: int, fps: int, pix_fmt: str = "rgb24" +): # pix_fmt: "rgb24" if frames are RGB uint8; use "bgr24" if OpenCV frames cmd = [ "ffmpeg", "-y", - "-f", "rawvideo", - "-vcodec", "rawvideo", - "-pix_fmt", pix_fmt, - "-s", f"{width}x{height}", - "-r", str(fps), - "-i", "pipe:0", + "-f", + "rawvideo", + "-vcodec", + "rawvideo", + "-pix_fmt", + pix_fmt, + "-s", + f"{width}x{height}", + "-r", + str(fps), + "-i", + "pipe:0", "-an", - "-c:v", "libx264", - "-preset", "veryfast", - "-crf", "23", - "-pix_fmt", "yuv420p", # best compatibility + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", # best compatibility out_path, ] return subprocess.Popen(cmd, stdin=subprocess.PIPE) @@ -1294,9 +1321,10 @@ def interpolate_arr_euler(v: np.ndarray, seq_length: int) -> np.ndarray: v: (B, T, 6) or (B, T, 7) [x, y, z, yaw, pitch, roll, (optional) gripper] """ - assert v.ndim == 3 and v.shape[2] in (6, 7), ( - "Input v must be of shape (B, T, 6) or (B, T, 7)" - ) + assert v.ndim == 3 and v.shape[2] in ( + 6, + 7, + ), "Input v must be of shape (B, T, 6) or (B, T, 7)" B, T, D = v.shape new_time = np.linspace(0, 1, seq_length) @@ -1640,7 +1668,7 @@ def get_vector_from_yaw_pitch( return unit_dir else: return unit_dir * depth - + def get_gaze_endpoint(yaw_rads, pitch_rads, depth, T_cam_cpf): """