From dd476bbe3b702979cf04f66b17ec5cc6ad9f107c Mon Sep 17 00:00:00 2001 From: AnikethCheluva Date: Sun, 25 Jan 2026 13:51:47 -0500 Subject: [PATCH 1/2] make config changes From cc12755eae3bbe05540d3745933e83fcb031872d Mon Sep 17 00:00:00 2001 From: Aniketh Cheluva Date: Mon, 26 Jan 2026 11:43:09 -0500 Subject: [PATCH 2/2] branch for pi configs --- egomimic/hydra_configs/data/eva_bc_s3.yaml | 8 +- egomimic/hydra_configs/data/mecka_test.yaml | 6 +- .../hydra_configs/data/obj_cont_cotrain.yaml | 54 +++++ egomimic/hydra_configs/model/pi0.5.yaml | 24 ++- egomimic/hydra_configs/model/pi0.5_mecka.yaml | 57 +++++ egomimic/hydra_configs/train.yaml | 203 ++++++++++-------- egomimic/rldb/utils.py | 2 +- 7 files changed, 254 insertions(+), 100 deletions(-) create mode 100644 egomimic/hydra_configs/data/obj_cont_cotrain.yaml create mode 100644 egomimic/hydra_configs/model/pi0.5_mecka.yaml diff --git a/egomimic/hydra_configs/data/eva_bc_s3.yaml b/egomimic/hydra_configs/data/eva_bc_s3.yaml index c65978f0..2a06e281 100644 --- a/egomimic/hydra_configs/data/eva_bc_s3.yaml +++ b/egomimic/hydra_configs/data/eva_bc_s3.yaml @@ -8,7 +8,7 @@ train_datasets: embodiment: "eva_bimanual" cache_root: "/coc/flash7/scratch/.cache/" filters: - task: "fold clothes" + task: "object in container" local_files_only: True debug: False @@ -20,16 +20,16 @@ valid_datasets: embodiment: "eva_bimanual" cache_root: "/coc/flash7/scratch/.cache/" filters: - task: "fold clothes" + task: "object in container" local_files_only: True debug: False train_dataloader_params: dataset1: batch_size: 32 - num_workers: 10 + num_workers: 6 valid_dataloader_params: dataset1: batch_size: 32 - num_workers: 10 \ No newline at end of file + num_workers: 6 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/mecka_test.yaml b/egomimic/hydra_configs/data/mecka_test.yaml index 03bb217b..a7480650 100644 --- a/egomimic/hydra_configs/data/mecka_test.yaml +++ b/egomimic/hydra_configs/data/mecka_test.yaml @@ -2,25 +2,27 @@ _target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper train_datasets: dataset1: + _target_: rldb.utils.S3RLDBDataset _target_: rldb.utils.S3RLDBDataset bucket_name: "rldb" mode: train local_mode: True embodiment: "mecka_bimanual" filters: - task: "fold_clothes" + task: "organizing_tools" local_files_only: True valid_datasets: dataset1: + _target_: rldb.utils.S3RLDBDataset _target_: rldb.utils.S3RLDBDataset bucket_name: "rldb" mode: valid local_mode: True embodiment: "mecka_bimanual" filters: - task: "fold_clothes" + task: "organizing_tools" local_files_only: True diff --git a/egomimic/hydra_configs/data/obj_cont_cotrain.yaml b/egomimic/hydra_configs/data/obj_cont_cotrain.yaml new file mode 100644 index 00000000..1974a617 --- /dev/null +++ b/egomimic/hydra_configs/data/obj_cont_cotrain.yaml @@ -0,0 +1,54 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper +train_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: eva_right_arm + filters: + task: object in container + local_files_only: true + + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: aria_right_arm + filters: + task: object in container + local_files_only: true + + +valid_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: eva_right_arm + filters: + task: object in container + local_files_only: true + + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: aria_right_arm + filters: + task: object in container + local_files_only: true + +train_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 8 + dataset2: + batch_size: 32 + num_workers: 8 +valid_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 8 + dataset2: + batch_size: 32 + num_workers: 8 \ No newline at end of file diff --git a/egomimic/hydra_configs/model/pi0.5.yaml b/egomimic/hydra_configs/model/pi0.5.yaml index c1af0cc4..7f7d0b34 100644 --- a/egomimic/hydra_configs/model/pi0.5.yaml +++ b/egomimic/hydra_configs/model/pi0.5.yaml @@ -3,24 +3,32 @@ robomimic_model: _target_: egomimic.algo.pi.PI data_schematic: _${data.dataset.data_schematic} camera_transforms: - _target_: egomimic.utils.egomimicUtils.CameraTransforms - intrinsics_key: "base" # change to base_half if using half res - extrinsics_key: "ariaJun7" + eva_right_arm: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "base" # change to base_half if using half res + extrinsics_key: "x5Dec13_2" + aria_right_arm: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "base" # change to base_half if using half res + extrinsics_key: "ariaJun7" ac_keys: - aria_bimanual: "actions_cartesian" - domains: ["aria_bimanual"] + eva_right_arm: "actions_cartesian" + aria_right_arm: "actions_cartesian" + domains: ["eva_right_arm", "aria_right_arm"] action_converters: rules: - ARIA_BIMANUAL: - _target_: egomimic.utils.action_utils.HumanBimanualCartesianEuler + EVA_RIGHT_ARM: + _target_: egomimic.utils.action_utils.RobotRightCartesianEuler + ARIA_RIGHT_ARM: + _target_: egomimic.utils.action_utils.HumanRightCartesianEuler # optional fallback if no match is found fallback: _target_: egomimic.utils.action_utils.BaseActionConverter config: pytorch_training_precision: bfloat16 - pytorch_weight_path: /storage/home/hcoda1/5/rpunamiya6/cedar-dx/rpunamiya6/Projects/EgoVerse/egomimic/algo/pi_checkpoints/pi05_base_pytorch + pytorch_weight_path: /storage/home/hcoda1/5/acheluva3/jcedar/EgoVerse/egomimic/algo/pi_checkpoints/pi05_base_pytorch model: pi05: true action_dim: 32 diff --git a/egomimic/hydra_configs/model/pi0.5_mecka.yaml b/egomimic/hydra_configs/model/pi0.5_mecka.yaml new file mode 100644 index 00000000..c7c34632 --- /dev/null +++ b/egomimic/hydra_configs/model/pi0.5_mecka.yaml @@ -0,0 +1,57 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.pi.PI + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + mecka_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "mecka" # change to base_half if using half res + extrinsics_key: "mecka" + ac_keys: + mecka_bimanual: "actions_cartesian" + domains: ["mecka_bimanual"] + + action_converters: + rules: + MECKA_BIMANUAL: + _target_: egomimic.utils.action_utils.HumanBimanualCartesianEuler + # optional fallback if no match is found + fallback: + _target_: egomimic.utils.action_utils.BaseActionConverter + + config: + pytorch_training_precision: bfloat16 + pytorch_weight_path: /storage/home/hcoda1/5/acheluva3/jcedar/EgoVerse/egomimic/algo/pi_checkpoints/pi05_base_pytorch + model: + pi05: true + action_dim: 32 + action_horizon: 100 + max_token_len: 180 + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Resize + size: 224 + interpolation: 3 + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Resize + size: 224 + interpolation: 3 + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 5e-4 + betas: [0.9, 0.999] + eps: 1e-8 + weight_decay: 0.0 + +scheduler: + _target_: transformers.get_cosine_schedule_with_warmup + _partial_: true + num_warmup_steps: 1000 + num_training_steps: 100000 + num_cycles: 0.5 \ No newline at end of file diff --git a/egomimic/hydra_configs/train.yaml b/egomimic/hydra_configs/train.yaml index 931b780c..f8eb4095 100644 --- a/egomimic/hydra_configs/train.yaml +++ b/egomimic/hydra_configs/train.yaml @@ -1,96 +1,129 @@ - defaults: - - model: hpt_bc_flow_mecka_1B +defaults: + - model: pi0.5 - paths: default - - trainer: ddp + - trainer: ddp_pi - debug: null - logger: wandb - data: mecka_all - callbacks: checkpoints - - override hydra/launcher: submitit_slurm + - override hydra/launcher: submitit_pace - _self_ - name: test - description: test - ckpt_path: null - train: true - eval: false +name: test +description: test +ckpt_path: null +train: true +eval: false - eval_class: - _target_ : egomimic.scripts.evaluation.Eve - mode: real - arm: both - eval_path: "./logs/eval/${name}_${now:%Y-%m-%d_%H-%M-%S}" +eval_class: + _target_ : egomimic.scripts.evaluation.Eve + mode: real + arm: both + eval_path: "./logs/eval/${name}_${now:%Y-%m-%d_%H-%M-%S}" - hydra: - run: - # Dir should be experiment_name/description_{timestamp} - dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} - sweep: - dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} +hydra: + run: + # Dir should be experiment_name/description_{timestamp} + dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} + sweep: + dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} - launch_params: - gpus_per_node: 2 - nodes: 1 +launch_params: + gpus_per_node: 2 + nodes: 1 - data_schematic: # Dynamically fill in these shapes from the dataset - _target_: egomimic.rldb.utils.DataSchematic - norm_mode: quantile - schematic_dict: - eva_bimanual: - front_img_1: #batch key - key_type: camera_keys # key type - lerobot_key: observations.images.front_img_1 # dataset key - right_wrist_img: - key_type: camera_keys - lerobot_key: observations.images.right_wrist_img - left_wrist_img: - key_type: camera_keys - lerobot_key: observations.images.left_wrist_img - ee_pose: - key_type: proprio_keys - lerobot_key: observations.state.ee_pose - joint_positions: - key_type: proprio_keys - lerobot_key: observations.state.joint_positions - actions_joints: - key_type: action_keys - lerobot_key: actions_joints - actions_cartesian: - key_type: action_keys - lerobot_key: actions_cartesian - embodiment: - key_type: metadata_keys - lerobot_key: metadata.embodiment - aria_bimanual: - front_img_1: - key_type: camera_keys - lerobot_key: observations.images.front_img_1 - ee_pose: - key_type: proprio_keys - lerobot_key: observations.state.ee_pose - actions_cartesian: - key_type: action_keys - lerobot_key: actions_cartesian - embodiment: - key_type: metadata_keys - lerobot_key: metadata.embodiment - mecka_bimanual: - front_img_1: - key_type: camera_keys - lerobot_key: observations.images.front_img_1 - ee_pose: - key_type: proprio_keys - lerobot_key: observations.state.ee_pose_cam - actions_cartesian: - key_type: action_keys - lerobot_key: actions_ee_cartesian_cam - embodiment: - key_type: metadata_keys - lerobot_key: metadata.embodiment - viz_img_key: - eva_bimanual: - front_img_1 - aria_bimanual: - front_img_1 - mecka_bimanual: - front_img_1 +data_schematic: # Dynamically fill in these shapes from the dataset + _target_: egomimic.rldb.utils.DataSchematic + norm_mode: quantile + schematic_dict: + eva_right_arm: + base_0_rgb: #batch key + key_type: camera_keys # key type + lerobot_key: observations.images.front_img_1 # dataset key + right_wrist_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.right_wrist_img + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose + actions_joints: + key_type: action_keys + lerobot_key: actions_joints + actions_cartesian: + key_type: action_keys + lerobot_key: actions_cartesian + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + eva_bimanual: + base_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.front_img_1 + right_wrist_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.right_wrist_img + left_wrist_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.left_wrist_img + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose + actions_joints: + key_type: action_keys + lerobot_key: actions_joints + actions_cartesian: + key_type: action_keys + lerobot_key: actions_cartesian + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + aria_right_arm: + base_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.front_img_1 + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose + actions_cartesian: + key_type: action_keys + lerobot_key: actions_cartesian + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + aria_bimanual: + base_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.front_img_1 + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose + actions_cartesian: + key_type: action_keys + lerobot_key: actions_cartesian + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + mecka_bimanual: + base_0_rgb: + key_type: camera_keys + lerobot_key: observations.images.front_img_1 + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose_cam + actions_cartesian: + key_type: action_keys + lerobot_key: actions_ee_cartesian_cam + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + viz_img_key: + eva_right_arm: + base_0_rgb + eva_bimanual: + base_0_rgb + aria_bimanual: + base_0_rgb + aria_right_arm: + base_0_rgb + mecka_bimanual: + base_0_rgb diff --git a/egomimic/rldb/utils.py b/egomimic/rldb/utils.py index 37d4f5f9..ad8f1d65 100644 --- a/egomimic/rldb/utils.py +++ b/egomimic/rldb/utils.py @@ -659,7 +659,7 @@ def __init__( valid_ratio=0.2, temp_root="/storage/project/r-dxu345-0/shared/egoverse_datasets/S3_rldb_data/S3_rldb_data", # "/coc/flash7/scratch/rldb_temp" cache_root="/storage/project/r-dxu345-0/shared/.cache", - filters=None, + filters={}, debug=False, **kwargs, ):