From d6c6c9d6e9cd6a83153da1c19da9e0bd99cecc0b Mon Sep 17 00:00:00 2001 From: AnikethCheluva Date: Tue, 3 Feb 2026 18:50:05 -0500 Subject: [PATCH] hpt configs --- egomimic/hydra_configs/data/__init__.py | 1 + egomimic/hydra_configs/data/mecka_all.yaml | 34 + .../hydra_configs/data/mecka_all_zarr.yaml | 77 ++ .../hydra_configs/data/mecka_cotrain.yaml | 50 ++ .../hydra_configs/data/mecka_flagship.yaml | 71 ++ .../data/mecka_flagship_freeform.yaml | 95 +++ .../data/mecka_human_fold_clothes.yaml | 36 + egomimic/hydra_configs/data/obj_cont_eva.yaml | 44 + egomimic/hydra_configs/data/parse.py | 28 + egomimic/hydra_configs/data/sample.txt | 798 ++++++++++++++++++ .../hydra/launcher/submitit_pace.yaml | 6 +- .../hydra/launcher/submitit_skynet.yaml | 3 +- .../lerobot_data/mecka_test.yaml | 27 +- egomimic/hydra_configs/logger/wandb.yaml | 2 +- egomimic/hydra_configs/model/hpt_all.yaml | 132 +++ .../model/hpt_bc_flow_mecka.yaml | 4 +- .../model/hpt_bc_flow_mecka_1B.yaml | 122 +++ .../model/hpt_bc_flow_mecka_300M.yaml | 123 +++ .../model/hpt_bc_flow_mecka_600M.yaml | 123 +++ .../model/hpt_bc_flow_scale_1B.yaml | 122 +++ .../model/hpt_bc_flow_scale_600M.yaml | 123 +++ .../hpt_cotrain_mecka_flow_shared_head.yaml | 4 +- egomimic/hydra_configs/train.yaml | 176 ++-- egomimic/hydra_configs/train_zarr.yaml | 2 +- egomimic/hydra_configs/trainer/default.yaml | 6 +- egomimic/pl_utils/pl_model.py | 9 + egomimic/rldb/utils.py | 33 +- egomimic/rldb/zarr/zarr_dataset_multi.py | 90 +- external/openpi | 2 +- sync_s3.py | 162 ++++ sync_s3.sbatch | 20 + 31 files changed, 2377 insertions(+), 148 deletions(-) create mode 100644 egomimic/hydra_configs/data/__init__.py create mode 100644 egomimic/hydra_configs/data/mecka_all.yaml create mode 100644 egomimic/hydra_configs/data/mecka_all_zarr.yaml create mode 100644 egomimic/hydra_configs/data/mecka_cotrain.yaml create mode 100644 egomimic/hydra_configs/data/mecka_flagship.yaml create mode 100644 egomimic/hydra_configs/data/mecka_flagship_freeform.yaml create mode 100644 egomimic/hydra_configs/data/mecka_human_fold_clothes.yaml create mode 100644 egomimic/hydra_configs/data/obj_cont_eva.yaml create mode 100644 egomimic/hydra_configs/data/parse.py create mode 100644 egomimic/hydra_configs/data/sample.txt create mode 100644 egomimic/hydra_configs/model/hpt_all.yaml create mode 100644 egomimic/hydra_configs/model/hpt_bc_flow_mecka_1B.yaml create mode 100644 egomimic/hydra_configs/model/hpt_bc_flow_mecka_300M.yaml create mode 100644 egomimic/hydra_configs/model/hpt_bc_flow_mecka_600M.yaml create mode 100644 egomimic/hydra_configs/model/hpt_bc_flow_scale_1B.yaml create mode 100644 egomimic/hydra_configs/model/hpt_bc_flow_scale_600M.yaml create mode 100644 sync_s3.py create mode 100644 sync_s3.sbatch diff --git a/egomimic/hydra_configs/data/__init__.py b/egomimic/hydra_configs/data/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/egomimic/hydra_configs/data/__init__.py @@ -0,0 +1 @@ + diff --git a/egomimic/hydra_configs/data/mecka_all.yaml b/egomimic/hydra_configs/data/mecka_all.yaml new file mode 100644 index 00000000..88b81f0e --- /dev/null +++ b/egomimic/hydra_configs/data/mecka_all.yaml @@ -0,0 +1,34 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper + + +train_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + mode: train + local_mode: true + sample_percent: 0.1 + filters: + lab: "mecka" + embodiment: "mecka_bimanual" + +valid_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + mode: valid + local_mode: true + sample_percent: 0.1 + filters: + lab: "mecka" + embodiment: "mecka_bimanual" + +train_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 8 + +valid_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 8 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/mecka_all_zarr.yaml b/egomimic/hydra_configs/data/mecka_all_zarr.yaml new file mode 100644 index 00000000..b8974fad --- /dev/null +++ b/egomimic/hydra_configs/data/mecka_all_zarr.yaml @@ -0,0 +1,77 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper +train_datasets: + mecka_bimanual: + _target_: egomimic.rldb.zarr.zarr_dataset_multi.MultiDataset._from_resolver + resolver: + _target_: egomimic.rldb.zarr.zarr_dataset_multi.LocalEpisodeResolver + folder_path: /coc/flash7/scratch/egoverseS3ZarrDataset/ + key_map: + observations.images.front_img_1: #batch key + key_type: camera_keys # key type + zarr_key: images.front_1 + right.action_ee_pose: + key_type: action_keys + zarr_key: right.obs_ee_pose + horizon: 30 + left.action_ee_pose: + key_type: action_keys + zarr_key: left.obs_ee_pose + horizon: 30 + right.obs_ee_pose: + key_type: proprio_keys + zarr_key: right.obs_ee_pose + left.obs_ee_pose: + key_type: proprio_keys + zarr_key: left.obs_ee_pose + obs_head_pose: + zarr_key: obs_head_pose + key_type: proprio_keys + transform_list: + _target_: egomimic.rldb.embodiment.human._build_aria_bimanual_transform_list + stride: 1 + # filters: + # episode_hash: ${eval:'__import__("egomimic.hydra_configs.data.parse", fromlist=["episode_hashes_from_sample"]).episode_hashes_from_sample()'} + # episode_hash: ["696a1e7a03b3bab4cd3d4ff9", "696b94c5c25b828c83107b3f"] + mode: train +valid_datasets: + mecka_bimanual: + _target_: egomimic.rldb.zarr.zarr_dataset_multi.MultiDataset._from_resolver + resolver: + _target_: egomimic.rldb.zarr.zarr_dataset_multi.LocalEpisodeResolver + folder_path: /coc/flash7/scratch/egoverseS3ZarrDataset/ + key_map: + observations.images.front_img_1: #batch key + key_type: camera_keys # key type + zarr_key: images.front_1 + right.action_ee_pose: + key_type: action_keys + zarr_key: right.obs_ee_pose + horizon: 30 + left.action_ee_pose: + key_type: action_keys + zarr_key: left.obs_ee_pose + horizon: 30 + right.obs_ee_pose: + key_type: proprio_keys + zarr_key: right.obs_ee_pose + left.obs_ee_pose: + key_type: proprio_keys + zarr_key: left.obs_ee_pose + obs_head_pose: + zarr_key: obs_head_pose + key_type: proprio_keys + transform_list: + _target_: egomimic.rldb.embodiment.human._build_aria_bimanual_transform_list + stride: 1 + # filters: + # episode_hash: ${eval:'__import__("egomimic.hydra_configs.data.parse", fromlist=["episode_hashes_from_sample"]).episode_hashes_from_sample()'} + # episode_hash: ["696a1e7a03b3bab4cd3d4ff9", "696b94c5c25b828c83107b3f"] + mode: valid +train_dataloader_params: + mecka_bimanual: + batch_size: 32 + num_workers: 10 +valid_dataloader_params: + mecka_bimanual: + batch_size: 32 + num_workers: 10 diff --git a/egomimic/hydra_configs/data/mecka_cotrain.yaml b/egomimic/hydra_configs/data/mecka_cotrain.yaml new file mode 100644 index 00000000..e6ce1ab1 --- /dev/null +++ b/egomimic/hydra_configs/data/mecka_cotrain.yaml @@ -0,0 +1,50 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper + +train_datasets: + # dataset1: + # _target_: egomimic.rldb.utils.S3RLDBDataset + # bucket_name: "rldb" + # embodiment: "mecka_bimanual" + # mode: train + # filters: + # task: "potting_soil" + # local_files_only: True + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + embodiment: "aria_right_arm" + mode: train + filters: + task: "object in container" + lab: "song" + local_files_only: True + +valid_datasets: + # dataset1: + # _target_: egomimic.rldb.utils.S3RLDBDataset + # bucket_name: "rldb" + # embodiment: "mecka_bimanual" + # mode: valid + # filters: + # task: "potting_soil" + # local_files_only: True + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + embodiment: "aria_right_arm" + mode: valid + filters: + task: "object in container" + lab: "song" + local_files_only: True + + +train_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 10 + +valid_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 10 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/mecka_flagship.yaml b/egomimic/hydra_configs/data/mecka_flagship.yaml new file mode 100644 index 00000000..18e59a13 --- /dev/null +++ b/egomimic/hydra_configs/data/mecka_flagship.yaml @@ -0,0 +1,71 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper +train_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: eva_bimanual + filters: + task: fold_clothes + local_files_only: true + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: aria_bimanual + filters: + task: fold_clothes_indomain + local_files_only: true + dataset3: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: mecka_bimanual + filters: + task: fold_clothes + local_files_only: true +valid_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: eva_bimanual + filters: + task: fold_clothes + local_files_only: true + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: aria_bimanual + filters: + task: fold_clothes_indomain + local_files_only: true + dataset3: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: mecka_bimanual + filters: + task: fold_clothes + local_files_only: true +train_dataloader_params: + dataset1: + batch_size: 8 + num_workers: 10 + dataset2: + batch_size: 6 + num_workers: 10 + dataset3: + batch_size: 12 + num_workers: 10 +valid_dataloader_params: + dataset1: + batch_size: 8 + num_workers: 10 + dataset2: + batch_size: 6 + num_workers: 10 + dataset3: + batch_size: 12 + num_workers: 10 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/mecka_flagship_freeform.yaml b/egomimic/hydra_configs/data/mecka_flagship_freeform.yaml new file mode 100644 index 00000000..e40618bf --- /dev/null +++ b/egomimic/hydra_configs/data/mecka_flagship_freeform.yaml @@ -0,0 +1,95 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper +train_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: eva_bimanual + filters: + task: fold_clothes + local_files_only: true + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: aria_bimanual + filters: + task: fold_clothes_indomain + local_files_only: true + dataset3: + _target_: egomimic.rldb.utils.MultiRLDBDataset + datasets: + flagship: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: mecka_bimanual + filters: + task: fold_clothes + local_files_only: true + freeform: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: train + embodiment: mecka_bimanual + filters: + task: folding_clothes + local_files_only: true + embodiment: mecka_bimanual +valid_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: eva_bimanual + filters: + task: fold_clothes + local_files_only: true + dataset2: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: aria_bimanual + filters: + task: fold_clothes_indomain + local_files_only: true + dataset3: + _target_: egomimic.rldb.utils.MultiRLDBDataset + datasets: + flagship: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: mecka_bimanual + filters: + task: fold_clothes + local_files_only: true + freeform: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: rldb + mode: valid + embodiment: mecka_bimanual + filters: + task: folding_clothes + local_files_only: true + embodiment: mecka_bimanual +train_dataloader_params: + dataset1: + batch_size: 8 + num_workers: 10 + dataset2: + batch_size: 6 + num_workers: 10 + dataset3: + batch_size: 12 + num_workers: 10 +valid_dataloader_params: + dataset1: + batch_size: 8 + num_workers: 10 + dataset2: + batch_size: 6 + num_workers: 10 + dataset3: + batch_size: 12 + num_workers: 10 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/mecka_human_fold_clothes.yaml b/egomimic/hydra_configs/data/mecka_human_fold_clothes.yaml new file mode 100644 index 00000000..e1edefb6 --- /dev/null +++ b/egomimic/hydra_configs/data/mecka_human_fold_clothes.yaml @@ -0,0 +1,36 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper + +train_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + main_prefix: "mecka" + mode: train + embodiment: "mecka_bimanual" + temp_root: "/coc/flash7/scratch/egoverseS3Dataset" + filters: + lab: "mecka" + task: "fold_clothes" + local_files_only: True + +valid_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + mode: valid + embodiment: "mecka_bimanual" + temp_root: "/coc/flash7/scratch/egoverseS3Dataset" + filters: + lab: "mecka" + task: "fold_clothes" + local_files_only: True + +train_dataloader_params: + dataset1: + batch_size: 64 + num_workers: 10 + +valid_dataloader_params: + dataset1: + batch_size: 64 + num_workers: 10 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/obj_cont_eva.yaml b/egomimic/hydra_configs/data/obj_cont_eva.yaml new file mode 100644 index 00000000..354d1185 --- /dev/null +++ b/egomimic/hydra_configs/data/obj_cont_eva.yaml @@ -0,0 +1,44 @@ +_target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper +use_tokenizer: false + +train_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + temp_root: "/coc/cedarp-dxu345-0/datasets/egoverse" + mode: train + embodiment: "eva_right_arm" + filters: + lab: "rl2" + task: "object in container" + robot_name: "eva_right_arm" + # objects: + # - "green bowl" + # - "caprisun" + local_files_only: True + +valid_datasets: + dataset1: + _target_: egomimic.rldb.utils.S3RLDBDataset + bucket_name: "rldb" + temp_root: "/coc/cedarp-dxu345-0/datasets/egoverse" + mode: valid + embodiment: "eva_right_arm" + filters: + lab: "rl2" + task: "object in container" + robot_name: "eva_right_arm" + # objects: + # - "green bowl" + # - "caprisun" + local_files_only: True + +train_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 8 + +valid_dataloader_params: + dataset1: + batch_size: 32 + num_workers: 8 \ No newline at end of file diff --git a/egomimic/hydra_configs/data/parse.py b/egomimic/hydra_configs/data/parse.py new file mode 100644 index 00000000..9a5ae9da --- /dev/null +++ b/egomimic/hydra_configs/data/parse.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from pathlib import Path + + +def episode_hashes_from_sample(sample_path: str | Path | None = None) -> list[str]: + """ + Load episode hashes from `sample.txt` (one per line). + + Ignores blank lines and lines starting with '#'. + """ + if sample_path is None: + sample_path = Path(__file__).with_name("sample.txt") + else: + sample_path = Path(sample_path) + + hashes: list[str] = [] + seen: set[str] = set() + for line in sample_path.read_text().splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + if s in seen: + continue + seen.add(s) + hashes.append(s) + + return hashes diff --git a/egomimic/hydra_configs/data/sample.txt b/egomimic/hydra_configs/data/sample.txt new file mode 100644 index 00000000..a84899f2 --- /dev/null +++ b/egomimic/hydra_configs/data/sample.txt @@ -0,0 +1,798 @@ +696bc276a8793724a1ef2119 +696da8881b41dc512e994b03 +696be6837ed68fc306df73e4 +696baf675309532923d3f25a +696b8b1f95be6b21592581fb +696d71bbf48d85e4c02f1e35 +696bd22a4efa2e122be603ca +696bd8e75e774190a1c97936 +696bc8b2a8793724a1ef215e +696bd3a97ed68fc306df726e +696dee2d66866953159e99a6 +696bc5a7b261b0a7823a1d4d +696da1856256b37b6b389a08 +696e335f7d01e2361b70edea +696ba041ea14729f866e8c20 +696b50f35fedf8533255da37 +696dab516256b37b6b389a95 +696a32bc7a2fd07226c098f6 +696be0ca33191fac7d9180e8 +696ce4ac0576d930a343fe7f +696ba7c67a2fd07226c09eed +696d9a8da888e5c0e1cb43b0 +696cc1f7fad368fca385f42e +696d2c0d4b2b73eea2f1cd72 +696d912b80f51286a174d23f +696d33068cac7ab9693b11e5 +696d95a44c3d4673e3917432 +696c789938f4b4346c53169f +696c9c0138adf4f372b45143 +696bd11e4c43650cbf6f5eb9 +696da23cf1ecd06ff7254fcd +696be66228ab12fee37356da +696b94c5c25b828c83107b3f +696bd9b0e4eb4a667bf0e066 +696ba71fd99e01dc631f3d2d +696bdb89a6f1d19c01834669 +696bac66afe603760fe1ad15 +696dd05a2ed43ace5a4d4385 +696be4cfc6cb022a9da40504 +696dd12cc5cd8b02c240f8e5 +696db78273a971419ff9fdbd +696ba84f522a3f9be47904b2 +696d967528a92a09a6839cdb +696e53638587a69d44bf7b54 +696ccb4c344d3cc10178a942 +696e84fccf86b2bb0757a778 +696ddc18433fcf31b5691e1c +696dfb5b43f23b8635c5a0b0 +696b77aab717e87cf71f7531 +696e79b3fdf195647b323876 +696e31ce3fd4b5d896cd2263 +696b82ed72239e0ab46348f4 +696bc56343edc8aed548050e +696b4a591e187f32c5eb563b +696be6d9101054681ed97a8f +696bd34a8a0fa4aac9be483a +696baf8612bac97f028599cf +696e7b96114adc11dddb5674 +696b38d2634a1e1f4692736f +696bddffb17075c2c32faa02 +696d5db9ead9a484382f3763 +696d9a6482f4c5e6dfcb43c9 +696d345ef63b90865845f7a9 +696cadc38cac7ab9693b0e6c +696bde7d7e6f94bf34143c9d +696ba9a912bac97f0285994c +696b1f80370c796a030963e8 +696acad484947585c0cb6b48 +696c608438f4b4346c531624 +696bc792c426c607c88b2e6b +696baf2287b9ac9746b299fe +696bd492daa011f7172d22ba +696bd273263039fb3592de3a +696da35664a75ad017733b6a +696ba75da8cba78d3b5a4f96 +696e5ce68db2ad069baadeb4 +696ca8dd1244464a4aac199e +696d128d199912be320e2b95 +696e3399482496d71b0f5400 +696d2df218824db10c3a7c5b +696bbce2d9a2a4e1bf836d59 +696d3303e4c876d0ca244412 +696cc9ac6ed3eb90778f05e2 +696dd0b238fafc2ca5d1189f +696d789ad87661ea56b2b47f +696b2d5bd3cc9dc76c561157 +696ba3dfcd29fb97fb107c48 +696ba65dd2d7cb0bd150ebcc +696bb249c676295b4ae74ac8 +696cbe0f2b50484b18da1ae2 +696d93f8bb9cd59b6d533458 +696ba56fef11876d0045f59e +696e69fd9f6952d8c9d857b5 +696e8635f43c454145204a8f +696b830c02ce4c977624f669 +696d5db253507ad32ee6c842 +696c620a7bc8798b7fe17713 +696ba785fe5c03f9da388615 +696dd7f93d5391e3c577b21f +696bdc5c522a3f9be4790850 +696b2d3b0c52a10600ae7f30 +696baddcb0900198ccf5c0dc +696c62cac0c2d9149de655c6 +696ba7b8c8af0268a6bacccd +696beededa6069828e20d28f +696ba9c6b0b2b22cc0d1f5d1 +696bd40dcca65ef143720317 +696e86a09ec04eb46fe0673e +696c6e0aeaaf0411bfe587dd +696e7ccd9ec04eb46fe0669d +696d9f296b268dede04ca072 +696ba7c57bb51948f9de60b6 +696bc0e2c0aa1c772e76f2ca +696b48b0daa011f7172d1d21 +696b19daf8e8293f615f75eb +696bb4699839477cebe21c3e +696dd56034e6f4bcaf23b1cc +696ba5b91894fc36a07d8d2c +696babf879e3576eb2d4f24f +696bcdfecfb39c7ef2d48d3c +696e977528c744a0fcac8e43 +696e85a017377c6d0e557d16 +696bc711e43294cde98aa534 +696dd5c2dd1a587f40c9b70b +696ac9424e85aa4ea7da60f4 +696da3f0f099d0b0111f1389 +696e53027ce03c4a8965ed33 +696d2df000086fc00b80fec6 +696d971c85537287d8383c5d +696dd0a29603f48ee2e58af1 +696d95fd557bad554d7080b5 +696be03ff53e1ae2cbd87122 +696ba96e9a643f01c844d366 +696b2d8f429b192f087755e1 +696b16493724208fc971de5f +696e83824b013c8444629069 +696ba5aeb17075c2c32fa6d8 +696bdc43eab4449d0aabe972 +696da490840d7d67309c332f +696bd9ac9839477cebe21e6c +696d998840ed685a0dcaca73 +696ce15ac14d3f6041dd6acc +696d9bbdfda6142538506532 +696d91def3fb8768f0f68281 +696d9f79bcbf0c32b4b22af7 +696bdac07ed68fc306df7302 +696bba5da8cba78d3b5a513d +696bd4c54efa2e122be603e5 +696da4ad557bad554d70816e +696be1238a0fa4aac9be4958 +696e82b2ee9b736af1a79628 +696badc5778afed01d13c8df +696ba2cdb261b0a7823a1b74 +696d9b0f53c138b9263a64a8 +696bc41e9a140887948c5f12 +696c602fa199224e8d661f70 +696d987bcde72530e232bef8 +696bdd45abc451024b8271d6 +696e87a04a5fcb629b337ca9 +696bd04f286a12e5ac1a0802 +696da45d82e1ac2de620ad8d +696ba90e76b41c2326097fb8 +696e54357e219e09d76217e4 +696ba24ccd29fb97fb107c2d +696d93e6f4dec6e6d6ff7fa2 +696ba655736a3883c3d67bb5 +696bd3e234614160c6097dcc +696ba303101054681ed97596 +696c9996ead9a484382f3176 +696cfef342dcbdaae6ac0f9d +696bbc179a140887948c5e62 +696bdfe7c8af0268a6bad00c +696c9da47a3764d501282345 +696dc98d3ae226ec1ef35480 +696d3001024933da9b5c9e03 +696e6be1ff40ec10ec63b4fd +696da1e4261c825cae4b2b56 +696bbb16d0bfcc86fe4a68fc +696ba7efa6f1d19c0183434e +696bcb8875020a567fd377ce +696e3cd8946d3894eba26227 +696b374a39424cbffd6509b0 +696da36837dbf74800ad87fe +696b4cdc5e774190a1c9756d +696d940329ce27f261bcf00d +696da972d19968f832c03515 +696d9a1f53c138b9263a6493 +696d9fd9cdaff5670111cdb0 +696dabcd2a96e3bea7a08af6 +696d9e200838d9b98d822d3b +696d0fdafad368fca385f5a5 +696b8135736a3883c3d67a5f +696ba51fc35216997ae3eeb5 +696e53de276bf7d7724c58c6 +696bb3103897ce0bf0acaad8 +696ac84384947585c0cb6b2b +696bd713daa011f7172d22f0 +696b21df4685d5589aefaa98 +696da61716e44e8c77d6ab2d +696ac84b8e5cc0de4613703f +696e758ddc2838e08b0cedcf +696ad1b4915ce3edd0967644 +696bab7e5617babd55e658e7 +696bd8ec9c8ea4b6e4a1f3bb +696bc4a50ac3ef927c48e440 +696bbfe4cca8f940bc789ce0 +696dc1a25d7a7b9a57b7d9a0 +696b3a9e0ac58bb141246a33 +696d8e0930d6b6f26a006d43 +696d9b92fd57c67d83d7c26e +696bab9c1225df5278db7de5 +696d789c46f3fd69cfbea7bd +696d915149652a41264cf3be +696e764f21cd3dcb7f05c42f +696b383b79e3576eb2d4eda9 +696b5b9e017ec1a72078ef25 +696da5ef840d7d67309c334e +696be64b60e10e8f87b58692 +696d939a30d6b6f26a006d9e +696ba64af98fc50273d5d2cb +696b7cf74177e6c8a8387fed +696e8906c19c30de5741276d +696d3177d4f40d10fe08956b +696bbe1a4c43650cbf6f5d36 +696ba98612bac97f02859949 +696bd4f2b717e87cf71f7abd +696bad20d99e01dc631f3dab +696b0dcb0715b2fa1d6490cb +696da42740ed685a0dcacb3f +696bdca786833e290580c467 +696be26c3897ce0bf0acad5f +696aca37f46bb8d1bd8f95a2 +696da888f533d85d69b6a191 +696b9f0a316c07a505ec650e +696bd5478a0fa4aac9be4867 +696da37ed19968f832c034c0 +696ba772f05ef518f87fff11 +696c9dde2bd5bc335fb2a718 +696d71b59683ca5aed32a952 +696e825d5e4e2b7186cbc7e5 +696da69864124db03d5205c2 +696baea964c87f3a4d8c5bb2 +696da7d8ff9123fb363b31be +696d986e6a6524c5b9dbcfc2 +696da3b59984cbf630b0cd36 +696b2d6b72ef8d13aa00464e +696ac7f5b476146af0d5ac31 +696be2eba6f1d19c018346f3 +696ba0e27a2fd07226c09e4c +696da48acb559629bc87257d +696db802819d3d63e28a2d1d +696ca4be024933da9b5c9a1e +696be184b7080af9dcc9451a +696e7542dc2838e08b0cedc8 +696dd0c9f3fb8768f0f6844a +696b9e9467f2110cf0fbdf1e +696da44638fafc2ca5d117cd +696e33062934fcc9b4fff4e5 +696d99ad1b011089c8b7bfdb +696d2e5c911ed385aedae275 +696e2d807d60b0a13a324453 +696d92b07f9c99fea65ee800 +696d0176de00c290bacefe74 +696d789da199224e8d66284c +696d8eb480f51286a174d229 +696b4c1579bc0402722c1fb0 +696ba1e7f38155a6224418df +696ba19f76b41c2326097eec +696e7332e9557a101863898d +696e311f4b3bc2a1bad576ad +696acadac19cc80a4d8afb36 +696b75842c7243cd72c84de2 +696e5f08998e7722610688cd +696ba98e1d3959e2e052af6e +696ba79bbd0145c530d36e56 +696bdb3c60ebbd99811cbf1d +696e32c221cd3dcb7f05c270 +696ba1dd09bf9f435619eb80 +696bd3c0246cb36ca375812c +696dd52e77bf62ada9a01bdc +696ba75c7bb51948f9de60ad +696e852d7d60b0a13a324730 +696d93eb472a5cb8c5bd6b3f +696bb84bd0bfcc86fe4a68c3 +696bdaaa634a1e1f46927a1e +696ba24179188f941930289e +696dabb5103547e961005f75 +696ba3581cafc9f413b5b572 +696da22e71e72197ed90c7f4 +696a20e5715ed751e5e0f76d +696d9d8874fb8917db68166e +696b7a170c52a10600ae81d1 +696ba7ad915ce3edd09678f3 +696bdbb05309532923d3f4b2 +696b82f4dffffb22ce00c12a +696bbb117a2fd07226c0a079 +696e8679a6e0d63375347e07 +696bc278c19cc80a4d8b00da +696ac85d0c1d9ccd5892fc69 +696ba32e4d5f3676bb0aaf10 +696e2d6a16141fc0a6fa26f8 +696bef1037193673e4f89c6c +696bec69a907c8962e22ab4d +696e8c8fdd0cb8d4e520654a +696bcebcb0b2b22cc0d1f7d8 +696b2c5acac8603dbfca79ea +696bd55767d3fa0fb3723c55 +696b95495fedf8533255dc5d +696ca0cf5a4e3449377211ae +696c98a6f63b90865845f31e +696da68ea1cd28d6b07bed16 +696bae5aabc451024b826f14 +696bcd12a6f1d19c0183453b +696d99bf6a6524c5b9dbcfdc +696bac71d9a2a4e1bf836c4c +696bab34d0bfcc86fe4a67fe +696bc0eda907c8962e22a8eb +696b6ff5240b1e4a2d82c492 +696d9a7e925cce50c37eca38 +696ce9cc2bd5bc335fb2a8f4 +696ba1c04177e6c8a838810d +696ac7fc7ae3f760695d8891 +696e7b2e2f5fbbede374bdfc +696da4e2abc3f969d20a6c26 +696ba7ad8688d0ceb96d3b02 +696be13e1ae707d8c0922d5c +696bb83cd3cc9dc76c561714 +696e32baff40ec10ec63b384 +696bd27b75020a567fd37855 +696befef8a74d25e9af39071 +696bce63a2c887c4e90e36d8 +696bab0325368af005ca73d1 +696bac423724208fc971e1fd +696ba694c75cf6e4320e2799 +696bcf3e476e9a6e777fff16 +696bc36cc0aa1c772e76f2fb +696bc662b717e87cf71f79d2 +696bb1cc7ed68fc306df70d8 +696bd251e34991e438ab39ab +696ba2e84a936a4d1328a757 +696e867aa95acfe764b582f3 +696b4aac800cd1498944fc73 +696e3ec7cc5f7c104dea870b +696e8594a95acfe764b582e4 +696da740fd6a4da2c4f2739c +696e857be85937dc7bae4f85 +696e403ecc5f7c104dea8716 +696baf21c70dc8ed89019fc2 +696bddce4efa2e122be6046c +696b994f09b82287df69f479 +696da4afe5201eeff13a0058 +696ba098900932d25e86967d +696e74204a5fcb629b337b7c +696d32b42b50484b18da1df8 +696ba3bc87b9ac9746b29919 +696bdad679bc0402722c262e +696b78981e187f32c5eb584a +696ba71bdaa011f7172d1fdb +696bd9b0675f6396322f9566 +696bb06a4292c4c77991239b +696ce73df63b90865845f552 +696d95af45bc53953e7b2e01 +696bb3efed8ed158122ec5cf +696b791a14764da1c7e181ce +696d9caffdc8cef2b8bd654a +696bd523cd29fb97fb107fc6 +696b9f688a0fa4aac9be4605 +696ba998522a3f9be47904cd +696bd4c570e0519696316002 +696ba1ccfadd8a32efe99da6 +696bc357675f6396322f93e0 +696d2b8dd9f2dbab1eafb813 +696ba5b3daa011f7172d1fba +696c6186c0c2d9149de655a9 +696e885e4ec64da83885c134 +696bd3d8f7b3293476cab9a1 +696e7520657f8b0235e5d769 +696e898ff43c454145204aa9 +696bb97ec75cf6e4320e28fe +696e3102fb96a896b1b15319 +696b7a0ee2eca952034ed520 +696e75906af2aaf916d956ef +696d9e794d2f71b8d51ea2d4 +696db77e62c78a1f9f3dbd28 +696bb3708149bf439389510e +696be5920c52a10600ae87e8 +696e6c58998e77226106891b +696d93b84b4c24f766872fa9 +696ba45a86f71ea8bc69d0f9 +696bab263c35512701de63f0 +696da6d3c8145b08d49e82f0 +696ba6f3bd0145c530d36e46 +696d71c9de28b3a88166eb05 +696bda74e7200945ac1fd13f +696dd4794e52be266e5c903f +696dada11c142afc32b0a7b9 +696ce586296b5bace896d7e8 +696e750bcc5f7c104dea887b +696b19717a53efd0cd3a2625 +696b826070d628f8b0741f88 +696bac76620da154d5b7a90f +696e72bb41eb8180960562a2 +696ce5888cac7ab9693b0fd8 +696da5b4c1956556950adee5 +696c6e09c0c2d9149de655e2 +696b9ec2eab4449d0aabe5de +696d708035131b42cffecfa0 +696cc6c2ead9a484382f32f7 +696d1612ead9a484382f3494 +696dd5fa819d3d63e28a2d65 +696bd32336602141d44d6e7b +696ccc71531c58652b440143 +696c601d2bd5bc335fb2a51c +696b9e5886f71ea8bc69d073 +696ba64fae7ec71f865715a9 +696bafbc894a8c8702a75cbe +696ca65d531c58652b44003b +696b1f6809681473d11f0111 +696bd75b894c2e015aaa1096 +696be36d0ac58bb141247225 +696d8fb2ff9123fb363b2fcb +696e3469b5a66458ce86c587 +696e33797e219e09d7621681 +696ba30ea8cba78d3b5a4f3c +696bd22c101054681ed97932 +696e7b3bd5b9e89f3df82a50 +696d8f9373a971419ff9fcca +696d965e82f4c5e6dfcb4383 +696da5ec265e9641b943c100 +696bc62f883568f6441c830a +696ba8afae7ec71f865715de +696e75b2f6a5ac0b5ab65f4a +696ba4835bfe694453c61498 +696bbc5ec75cf6e4320e2912 +696ba7341894fc36a07d8d4c +696bcbdd0c52a10600ae8698 +696b94b767f2110cf0fbdece +696bdbcd1894fc36a07d90f7 +696bc915476e9a6e777ffe8a +696d5950586e0a17ad22d7b9 +696be41e72239e0ab4634dd6 +696ba09f53fe50fed713734f +696bc4d81894fc36a07d8f4c +696e2fdd5e4e2b7186cbc602 +696bdf6721d8e8ab8ddc855c +696d2910eaaf0411bfe58ce7 +696bb414ae7ec71f86571692 +696ba0243724208fc971e10b +696d9ff27c11eaba88f2def3 +696bacf093818b7b258ec107 +696be2f3e2eca952034eda5b +696dcd6ad724a55c2ff9a6c1 +696beb128a74d25e9af3904d +696ba654c8af0268a6baccb1 +696e341c153584e0fd42c290 +696e2e1d57757f9d4e3a19b8 +696ba4568a0fa4aac9be4677 +696bca3f8910191c50ed9e79 +696ba2e1d9a2a4e1bf836b7a +696d64b56ed3eb90778f0a48 +696d3bda586e0a17ad22d76c +696bacd939dc2438efc84fea +696bbf7975020a567fd37729 +696b9574b261b0a7823a1b15 +696bb816ca28cac2415e5e74 +696b1ec834c9ea293a8829ff +696d99f504c9be98d76d607f +696c605efad368fca385f23f +696b6a6270d628f8b0741e96 +696d9221cb559629bc872423 +696d33d6c0c2d9149de65c09 +696cc5c9960611157f73e478 +696bd7a7590d785c13a76167 +696d2fee296b5bace896da10 +696b68340ac58bb141246bbb +696da604d19968f832c034de +696e33d0482496d71b0f5404 +696d302c4f526320afafb6fe +696bc744fadd8a32efe9a059 +696d9f23b762e60f1f34fce1 +696a1e7a03b3bab4cd3d4ff9 +696d9c169d65f7c2b62e7fe9 +696e2e546d2667c42c6baccf +696ba1a272239e0ab46349ab +696bb8f5c7e4e8ee05bbe09b +696c9b40199912be320e28f6 +696ba73160e10e8f87b583d0 +696d8f03188e5ff22744e4c2 +696bb4f772239e0ab4634b24 +696bbdda8a74d25e9af38df6 +696e84f9276bf7d7724c5a08 +696beb9e7e6f94bf34143d46 +696dc986c7c9f7847ad5de59 +696d979afcf19b7cd6645fe9 +696bddb9b7080af9dcc944e7 +696b36bb80d64f1587ce3607 +696c9af6089658b1b2efaa4e +696e5156cc5f7c104dea8781 +696c9a1a296b5bace896d5af +696e1e16095eb2d2a98efd87 +696cfd3418824db10c3a7ae9 +696bd62e60e10e8f87b585e4 +696b820cdffffb22ce00c115 +696da0e40b1d6d58cb8b20ca +696bac738a0fa4aac9be46e6 +696e85b1b1cc409b12d985d7 +696da7a2d355c133d5ff9a20 +696b8ab2a6f1d19c018341e0 +696e31c843f23b8635c5a168 +696bb0197e6f94bf341438e4 +696bdba55fedf8533255e0c4 +696bd27095be6b21592586a6 +696b9d9b12bac97f0285987f +696bab011ee2a915e19349f2 +696bb3cd60ebbd99811cbcf2 +696da3f4650ccadd255131d6 +696bd3d5c8af0268a6bacf58 +696e88cd4ec64da83885c13f +696bdc8ffadd8a32efe9a1b9 +696bab760f689873a4d2b203 +696bbbc6ae7ec71f86571739 +696bd6ed0d98b1b1a38b151d +696bca650f689873a4d2b393 +696e86ea30d80e9a7b76618d +696bd9c9a6f1d19c01834643 +696bc06f8149bf439389520d +696bdc63ca28cac2415e6062 +696d710a6ed3eb90778f0acb +696bc706f38155a622441b2b +696ba2fea491d5abbd79527f +696d2d7fde00c290bacf0021 +696ba92d0f689873a4d2b1d9 +696bddfdf38155a622441c8e +696bd3fac426c607c88b2fa0 +696bb9c2c35216997ae3f006 +696cadc888af7cc0806366ff +696e8419620f4235f99e507e +696b2d7843d5aa74f8b8d022 +696bd5b0894c2e015aaa1079 +696ba91d1cafc9f413b5b5fc +696bd84c55db5be0d7707213 +696bdd5355db5be0d770727a +696b834907b6c7a3d79463b3 +696ba9f6522a3f9be47904d3 +696bca64e4eb4a667bf0df47 +696bc3a055db5be0d77070c6 +696d973780f51286a174d2b3 +696ba74b778afed01d13c852 +696d9cca683792d4938d418f +696d781a8f609c3c2d37089a +696d327ccb7cb0759a13e8f2 +696b2d84715ed751e5e0fadb +696da61a64a75ad017733b84 +696ba5e121d8e8ab8ddc8110 +696d92738d4e41e87521a468 +696d9aed7b9f7e17cc5772bf +696da5cf4f3681a9d821c26c +696bbf80f53e1ae2cbd86f60 +696b0d5fb38543260932eec7 +696bf47c1894fc36a07d920b +696b9b448a0fa4aac9be45ad +696da2b40e468cf1b7075b0c +696ac863715ed751e5e0f999 +696baa8f3724208fc971e1ed +696b4c1d76b41c2326097c87 +696be24b894c2e015aaa114d +696ac68a7a2fd07226c0999c +696dd09aa888e5c0e1cb44e7 +696ba8e9e34991e438ab3710 +696e75aa43c9ca3b047729ea +696bdcb8590d785c13a761cd +696e4167946d3894eba26273 +696bcb57e34991e438ab3927 +696bdb101894fc36a07d90e7 +696be93c473aede2ca7f0084 +696ba87ee34991e438ab3706 +696d9d50fdc8cef2b8bd6553 +696e742d95d47398e69950f3 +696d95d8cdaff5670111cd07 +696da326de1537c60354d9e3 +696d3276a534b5c917588c14 +696dd0c4be0cef097d487530 +696a319d03b3bab4cd3d51b4 +696bda9e39424cbffd651022 +696bae7637193673e4f89904 +696ccbe6a6fbe50a32562dd5 +696bb925f38155a622441a4d +696d9805e752ff1d8264cade +696bbb1a3c35512701de64e0 +696bb1bc590d785c13a75eca +696be966556c93e45170e089 +696d9fd5d355c133d5ff9972 +696dab0f64124db03d5205f7 +696b8bf45f8beb0b5e06fd29 +696dcbdb33f2abbafee7e6a3 +696cadc32b50484b18da1a64 +696bde936377d0871b6a93b3 +696bd4bf64c87f3a4d8c5e24 +696d2ff542dcbdaae6ac10a9 +696da6ba261c825cae4b2bc4 +696bbe8436602141d44d6d2a +696bb915b717e87cf71f78af +696ba5e079e3576eb2d4f1c6 +696d315585dbcb9294c28451 +696bcbe0e13d37dd4293f58b +696b0ddf2d6f5bba1f851358 +696c6195089658b1b2efa833 +696bd145c2544a8f586253a1 +696b81f809b82287df69f3bf +696d9ce082f4c5e6dfcb4402 +696b660cd2d7cb0bd150e97c +696bd8b7473aede2ca7eff48 +696da44e3cf2a230e1c658e1 +696e4d78d5b9e89f3df82919 +696bd8e8675f6396322f9553 +696d2fe2d95827836b14b81d +696b897239dc2438efc84e3d +696bd327c74b24422e33dc34 +696baa74d1a6773989721d35 +696e71f1a479c0684c886f73 +696d9fdacb3d08a3aa82b897 +696bb73fe4eb4a667bf0ddf6 +696da2d53b7bea24c7815452 +696bd6da0ac58bb141247147 +696bbda893818b7b258ec234 +696ba39e915ce3edd096789e +696be3e3f249daed52be5d0d +696acad4b476146af0d5ac69 +696e1f10b79e9d7860a3b596 +696bab4ba491d5abbd795337 +696e854d556c35e6b30d0288 +696ba50560e10e8f87b58392 +696d9fd23ae226ec1ef353ad +696ba9c80d98b1b1a38b1324 +696b22096b8c25a37ecc764a +696b217fa7f93a4d981fe30c +696e734d283b3b0e6a54c874 +696b0a8f55db5be0d7706cd5 +696e75d08395ff83f67546b7 +696bb2e7246cb36ca3757ea8 +696e551d0ebd516bc2eb4ba5 +696d964f8450426c4f113da6 +696e86d3998e772261068a00 +696da3d169b03d8bffc3f0da +696d989782f4c5e6dfcb43a2 +696d9d120e468cf1b7075aa8 +696bd3b0c19cc80a4d8b017e +696bdb75101054681ed97a01 +696e7527887a0e735b83d1d2 +696da3a0960ada878296ef9f +696b6a575fedf8533255daec +696d0fd56902ce4cbbd7e363 +696da1351b011089c8b7c070 +696b203984947585c0cb6c5e +696b5a063e2d6a68d5268304 +696bc47d6fc0c09c00a97dee +696be1cbf249daed52be5ce4 +696dd0ccfd57c67d83d7c362 +696b93b6cef1149d4f7a18e6 +696bdd7f9ac5bea01f9803de +696d30312ed0529037c4e95e +696ba1a2c19cc80a4d8afe9c +696cad02531c58652b44005b +696e83157046f49217da36d1 +696bcebfd3cc9dc76c561899 +696bd79ccd29fb97fb107ffd +696dcac1fe33c2160dc9e7b6 +696c601818824db10c3a76a1 +696da4a271e72197ed90c82c +696e83cbfa567a6f0c3cdf62 +696ba034894a8c8702a75b52 +696e4dc54b013c8444628e4f +696e73841b679d5d037a5443 +696d9933c93f59bdc6e20d43 +696b6cdc0ac3ef927c48e150 +696da110bbfffe73f206b8c5 +696e6371e23dfd1ea54cedb8 +696ba11328ab12fee3735184 +696d3352c1b587b64f6b4c65 +696ba9b9f53e1ae2cbd86dc7 +696bd310a907c8962e22a9fa +696b15fae1099f88d23797fb +696e737f8a176d6397a7a04b +696e84753fd4b5d896cd24fc +696d76912b50484b18da1ff6 +696bb418ea14729f866e8dec +696b0bc49dd7b9c9b16b35ae +696dcf074e52be266e5c9019 +696b93c5a907c8962e22a689 +696dcf97433fcf31b5691de2 +696da17f188e5ff22744e5b0 +696b77fc8a0fa4aac9be44fd +696ba7560d98b1b1a38b12f6 +696bc73e316c07a505ec67d8 +696ac87b5fedf8533255d7c5 +696be0864efa2e122be6049d +696ba31ef64364413f890c58 +696c788fe270f866a48679fc +696cc6c318824db10c3a79e0 +696da5a9b432c4daee39b60b +696d345d937ebd9d240441c9 +696bd8e0fadd8a32efe9a18f +696bb0c65617babd55e658ff +696bbf73900932d25e86982b +696bbeb5246cb36ca3757f82 +696ba87de13d37dd4293f33c +696dfcba3644d45b8cb8a87c +696ba0ac915ce3edd096785f +696d33e5e51af623f77ad1c7 +696bca9aa8793724a1ef2187 +696bd33babc451024b8270fc +696e4390ec7b7b20f30a6e55 +696d9b1ce75aabd3cbfced18 +696bcafc8a0fa4aac9be4788 +696b4d3dd87f83facc75f56c +696b20ab33191fac7d917b27 +696ba1b612bac97f028598af +696dd0c6e21603c88052575a +696d32c246b26c3b73e584b4 +696b1611cac8603dbfca78fb +696d0a69eaaf0411bfe58bee +696ba1fb07b6c7a3d7946499 +696ba93e86f71ea8bc69d166 +696b37eb80d64f1587ce3617 +696e2c7cc72152e151130c3e +696e7b4496d7c013fcbad036 +696d119e1fe17643f95f9fb9 +696e412cb79e9d7860a3b6a1 +696e8775f04cd7fd4f334578 +696b170c39df305cc536effe +696d9ddabad2864f69af2bec +696b9df109bf9f435619eb25 +696d0e3242dcbdaae6ac0fd4 +696d341774f9afbb0bd88d49 +696e84fb4ec64da83885c0e3 +696dfbafdd0cb8d4e5206191 +696bbffed1a6773989721e98 +696b8c86c6cb022a9da400c0 +696e540d0299a6b3c6c6942b +696d6e607f2429a1932f4cd7 +696ba0a5abc451024b826df7 +696ba0d9a04610abd4bb2f1b +696e3fd3589599cd0790898c +696bab7c1c34188da33fe9a7 +696bca2264c87f3a4d8c5d3d +696e7478c27c731ddfad6650 +696bce279a140887948c5fc9 +696bb4490c52a10600ae8502 +696dcf9069fe7e32c96f1bc3 +696ba113c7e4e8ee05bbdef4 +696c9f6e1fe17643f95f9d4a +696b1567ac451788451c34c5 +696ccb527a3764d50128246b +696e9504f7f281e223e03657 +696bb4a5c35216997ae3efbd +696ce82518824db10c3a7a97 +696e83e3620f4235f99e507a +696c6e77e270f866a48679de +696ba78c5309532923d3f1bc +696b76a7f7b3293476cab573 +696b791086f71ea8bc69cf89 +696e40220299a6b3c6c69349 +696b78cb14764da1c7e181c7 +696da55b74fb8917db6816fb +696e3eb3843b98de1f7e887d +696ce73aa199224e8d66238e +696bc37f6377d0871b6a91a1 +696ba39ac8af0268a6bacc77 +696cc13b2bd5bc335fb2a7f9 +696c97e9bcc428fa4d1b2c9a +696db7fbd07d4cdba345b8f8 +696a22964e85aa4ea7da5e80 +696bcb9c1894fc36a07d8fd5 +696d9b27c8b3144c937f8be1 +696e85d83fd4b5d896cd251f +696d2f312b50484b18da1dad +696bd74239424cbffd650fd9 +696b94737a53efd0cd3a2999 +696bb148c25b828c83107d01 +696b2c35f8e8293f615f764f +696d9cd5f4dec6e6d6ff7ffb +696bb2c9eab4449d0aabe797 +696ce9cb38adf4f372b4531c +696da58de75aabd3cbfcedc7 +696d34a6b6b3247e544e1ca2 +696bb4e7c25b828c83107d4a +696cfb54c14d3f6041dd6b38 +696bc2adcca65ef1437201d7 +696da688840d7d67309c335b +696ba42ac426c607c88b2bfc +696c6e371244464a4aac17f8 +696dfb603488965375fbfc17 +696b5a69e2eca952034ed443 +696bbd6779bc0402722c2418 +696ba983d1a6773989721d20 +696d9f26c1956556950ade4c +696ba8bd1894fc36a07d8d6d +696e3997ee9b736af1a7941f diff --git a/egomimic/hydra_configs/hydra/launcher/submitit_pace.yaml b/egomimic/hydra_configs/hydra/launcher/submitit_pace.yaml index 2d9cd957..cdfb0374 100644 --- a/egomimic/hydra_configs/hydra/launcher/submitit_pace.yaml +++ b/egomimic/hydra_configs/hydra/launcher/submitit_pace.yaml @@ -5,13 +5,13 @@ _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher # Slurm configuration name: ${hydra.job.name} # Default job name -partition: "gpu-h200" # Slurm partition +# partition: "gpu-h200" # Slurm partition account: "gts-dxu345-rl2" # Slurm account -cpus_per_task: 8 # Number of CPUs per task (max 4:1 CPU:GPU ratio) +cpus_per_task: 4 # Number of CPUs per task (max 4:1 CPU:GPU ratio) nodes: ${launch_params.nodes} # Number of nodes tasks_per_node: ${launch_params.gpus_per_node} # Use variable for tasks per node gres: "gpu:h200:${eval:'${launch_params.gpus_per_node} * ${launch_params.nodes}'}" # GPU type and count (h100 for H100 GPUs) -qos: "short" # Slurm QoS +qos: "inferno" # Slurm QoS mem_per_gpu: 250G timeout_min: 2880 # Timeout in minutes (48 hours) # exclude: "protocol, puma" # Nodes to exclude diff --git a/egomimic/hydra_configs/hydra/launcher/submitit_skynet.yaml b/egomimic/hydra_configs/hydra/launcher/submitit_skynet.yaml index 4f989ced..ca83317d 100644 --- a/egomimic/hydra_configs/hydra/launcher/submitit_skynet.yaml +++ b/egomimic/hydra_configs/hydra/launcher/submitit_skynet.yaml @@ -17,5 +17,4 @@ qos: "short" timeout_min: 2880 # 80 hours additional_parameters: - requeue: true - gpus-per-node: "a40:1" \ No newline at end of file + requeue: true \ No newline at end of file diff --git a/egomimic/hydra_configs/lerobot_data/mecka_test.yaml b/egomimic/hydra_configs/lerobot_data/mecka_test.yaml index 7f190339..09542a82 100644 --- a/egomimic/hydra_configs/lerobot_data/mecka_test.yaml +++ b/egomimic/hydra_configs/lerobot_data/mecka_test.yaml @@ -2,28 +2,41 @@ _target_: egomimic.pl_utils.pl_data_utils.MultiDataModuleWrapper train_datasets: dataset1: - _target_: rldb.utils.RLDBDataset - repo_id: "mecka_test" + _target_: rldb.utils.S3RLDBDataset + bucket_name: "rldb" mode: train + local_mode: True embodiment: "mecka_bimanual" - root: "/coc/flash7/acheluva3/EgoVerse/mecka_demo" + filters: + task: "organizing_tools" local_files_only: True + valid_datasets: dataset1: - _target_: rldb.utils.RLDBDataset - repo_id: "mecka_test" + _target_: rldb.utils.S3RLDBDataset + bucket_name: "rldb" mode: valid + local_mode: True embodiment: "mecka_bimanual" - root: "/coc/flash7/acheluva3/EgoVerse/mecka_demo" + filters: + task: "organizing_tools" local_files_only: True + + train_dataloader_params: dataset1: batch_size: 32 num_workers: 10 + # dataset2: + # batch_size: 32 + # num_workers: 10 valid_dataloader_params: dataset1: batch_size: 32 - num_workers: 10 \ No newline at end of file + num_workers: 10 + # dataset2: + # batch_size: 32 + # num_workers: 10 \ No newline at end of file diff --git a/egomimic/hydra_configs/logger/wandb.yaml b/egomimic/hydra_configs/logger/wandb.yaml index 6f574bd5..654b6b98 100644 --- a/egomimic/hydra_configs/logger/wandb.yaml +++ b/egomimic/hydra_configs/logger/wandb.yaml @@ -7,7 +7,7 @@ wandb: offline: False id: "${name}_${description}_${now:%Y-%m-%d_%H-%M-%S}" # pass correct id to resume experiment! anonymous: null # enable anonymous logging - project: "zarr_test" + project: "everse_mecka_hpt_ossification" log_model: False # upload lightning ckpts prefix: "" # a string to put at the beginning of metric keys entity: "rl2-group" # set to name of your wandb team diff --git a/egomimic/hydra_configs/model/hpt_all.yaml b/egomimic/hydra_configs/model/hpt_all.yaml new file mode 100644 index 00000000..ad674ba2 --- /dev/null +++ b/egomimic/hydra_configs/model/hpt_all.yaml @@ -0,0 +1,132 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.hpt.HPT + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + aria_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "base" # change to base_half if using half res + extrinsics_key: "ariaJun7" + eva_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "base" # change to base_half if using half res + extrinsics_key: "x5Dec13_2" + mecka_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "mecka" # change to base_half if using half res + extrinsics_key: "mecka" + + diffusion: true + 6dof: true + + ac_keys: + mecka_bimanual: "actions_cartesian" + eva_bimanual: "actions_cartesian" + aria_bimanual: "actions_cartesian" + + trunk: + embed_dim: 256 + num_blocks: 16 + num_heads: 8 + token_postprocessing: "action_token" + observation_horizon: 1 + action_horizon: 64 + no_trunk: false + use_domain_embedding: true + drop_path: 0.1 + weight_init_style: "pytorch" + + multitask: false + pretrained: false + pretrained_checkpoint: "" # TODO + reverse_kl_samples: 8 + + domains: ["mecka_bimanual", "eva_bimanual", "aria_bimanual"] + shared_obs_keys: ["front_img_1"] + + shared_stem_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 256 + output_dim: 256 + widths: [256] + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 16 + crossattn_heads: 8 + crossattn_dim_head: 64 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 256 + + stem_specs: + mecka_bimanual: + state_ee_pose: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 12 + output_dim: 256 + widths: [256] + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 16 + crossattn_heads: 8 + crossattn_dim_head: 64 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 256 + + head_specs: + mecka_bimanual: + _target_: egomimic.models.fm_policy.FMPolicy + action_horizon: 100 + num_inference_steps: 50 + pooling: null + time_dist: "beta" + infer_ac_dims: + mecka_bimanual: 12 + model: + _target_: egomimic.models.denoising_nets.CrossTransformer + nblocks: 6 + cond_dim: 256 + hidden_dim: 128 + act_dim: 12 + act_seq: 100 + n_heads: 4 + dropout: 0.1 + mlp_layers: 4 + mlp_ratio: 4 + + encoder_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.ResNet + output_dim: 256 + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.ColorJitter + brightness: 0.1 + contrast: 0.1 + saturation: 0.1 + hue: 0.05 + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 3e-4 + weight_decay: 0.0001 + +scheduler: + _target_: torch.optim.lr_scheduler.CosineAnnealingLR + _partial_: true + T_max: 1400 + eta_min: 1e-5 diff --git a/egomimic/hydra_configs/model/hpt_bc_flow_mecka.yaml b/egomimic/hydra_configs/model/hpt_bc_flow_mecka.yaml index 7f0d4f7c..0516a133 100644 --- a/egomimic/hydra_configs/model/hpt_bc_flow_mecka.yaml +++ b/egomimic/hydra_configs/model/hpt_bc_flow_mecka.yaml @@ -112,11 +112,11 @@ robomimic_model: optimizer: _target_: torch.optim.AdamW _partial_: true - lr: 3e-4 + lr: 1e-4 weight_decay: 0.0001 scheduler: _target_: torch.optim.lr_scheduler.CosineAnnealingLR _partial_: true T_max: 1400 - eta_min: 1e-5 + eta_min: 5e-5 diff --git a/egomimic/hydra_configs/model/hpt_bc_flow_mecka_1B.yaml b/egomimic/hydra_configs/model/hpt_bc_flow_mecka_1B.yaml new file mode 100644 index 00000000..ba11520a --- /dev/null +++ b/egomimic/hydra_configs/model/hpt_bc_flow_mecka_1B.yaml @@ -0,0 +1,122 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.hpt.HPT + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + mecka_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "mecka" # change to base_half if using half res + extrinsics_key: "mecka" + + diffusion: true + 6dof: true + + ac_keys: + mecka_bimanual: "actions_cartesian" + + trunk: + embed_dim: 1428 #94 + num_blocks: 26 + num_heads: 14 + token_postprocessing: "action_token" + observation_horizon: 1 + action_horizon: 64 + no_trunk: false + use_domain_embedding: true + drop_path: 0.1 + weight_init_style: "pytorch" + + multitask: false + pretrained: false + pretrained_checkpoint: "" # TODO + reverse_kl_samples: 8 + + domains: ["mecka_bimanual"] + shared_obs_keys: ["front_img_1"] + + shared_stem_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 1428 + output_dim: 1428 + widths: [1428] + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 22 + crossattn_heads: 16 + crossattn_dim_head: 416 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1428 + + stem_specs: + mecka_bimanual: + state_ee_pose: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 12 + output_dim: 1428 + widths: [1428] + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 22 + crossattn_heads: 16 + crossattn_dim_head: 416 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1428 + + head_specs: + mecka_bimanual: + _target_: egomimic.models.fm_policy.FMPolicy + action_horizon: 100 + num_inference_steps: 50 + pooling: null + time_dist: "beta" + infer_ac_dims: + mecka_bimanual: 12 + model: + _target_: egomimic.models.denoising_nets.CrossTransformer + nblocks: 6 + cond_dim: 1428 + hidden_dim: 408 + act_dim: 12 + act_seq: 100 + n_heads: 6 + dropout: 0.1 + mlp_layers: 7 + mlp_ratio: 7 + + encoder_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.ResNet + output_dim: 1428 + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.ColorJitter + brightness: 0.1 + contrast: 0.1 + saturation: 0.1 + hue: 0.05 + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 3e-4 + weight_decay: 0.0001 + +scheduler: + _target_: torch.optim.lr_scheduler.CosineAnnealingLR + _partial_: true + T_max: 5500 + eta_min: 1e-5 diff --git a/egomimic/hydra_configs/model/hpt_bc_flow_mecka_300M.yaml b/egomimic/hydra_configs/model/hpt_bc_flow_mecka_300M.yaml new file mode 100644 index 00000000..1d63314f --- /dev/null +++ b/egomimic/hydra_configs/model/hpt_bc_flow_mecka_300M.yaml @@ -0,0 +1,123 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.hpt.HPT + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + mecka_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "mecka" # change to base_half if using half res + extrinsics_key: "mecka" + + diffusion: true + 6dof: true + + ac_keys: + mecka_bimanual: "actions_cartesian" + + trunk: + embed_dim: 840 # changed from 256 #84 + num_blocks: 24 # changed from 16 + num_heads: 10 # changed from 8 + token_postprocessing: "action_token" + observation_horizon: 1 + action_horizon: 64 + no_trunk: false + use_domain_embedding: true + drop_path: 0.1 + weight_init_style: "pytorch" + + multitask: false + pretrained: false + pretrained_checkpoint: "" # TODO + reverse_kl_samples: 8 + + domains: ["mecka_bimanual"] + shared_obs_keys: ["front_img_1"] + + shared_stem_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 840 # changed from 512 + output_dim: 840 #changed + widths: [840] # changed from 840 + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 18 + crossattn_heads: 10 + crossattn_dim_head: 140 # changed from 256 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 840 # changed from 840 + + stem_specs: + mecka_bimanual: + state_ee_pose: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 12 + output_dim: 840 # changed from 840 + widths: [840] # changed from 840 + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 18 + crossattn_heads: 10 + crossattn_dim_head: 140 # changed from 256 changed from 1024 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 840 # changed from 840 changed from 1536 + + head_specs: + mecka_bimanual: + _target_: egomimic.models.fm_policy.FMPolicy + action_horizon: 100 + num_inference_steps: 50 + pooling: null + time_dist: "beta" + infer_ac_dims: + mecka_bimanual: 12 + model: + _target_: egomimic.models.denoising_nets.CrossTransformer + nblocks: 6 + cond_dim: 840 # changed from 256 changed from 1536 + hidden_dim: 320 #changed from 128 + act_dim: 12 + act_seq: 100 + n_heads: 5 # changed from 4 changed from 16 + dropout: 0.1 + mlp_layers: 5 # edit num of mlp layers + mlp_ratio: 5 + + encoder_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.ResNet + output_dim: 840 # changed from 512 changed from 1536 + + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.ColorJitter + brightness: 0.1 + contrast: 0.1 + saturation: 0.1 + hue: 0.05 + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 1e-4 + weight_decay: 0.0001 + +scheduler: + _target_: torch.optim.lr_scheduler.CosineAnnealingLR + _partial_: true + T_max: 5500 + eta_min: 1e-5 diff --git a/egomimic/hydra_configs/model/hpt_bc_flow_mecka_600M.yaml b/egomimic/hydra_configs/model/hpt_bc_flow_mecka_600M.yaml new file mode 100644 index 00000000..6f49b662 --- /dev/null +++ b/egomimic/hydra_configs/model/hpt_bc_flow_mecka_600M.yaml @@ -0,0 +1,123 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.hpt.HPT + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + mecka_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "mecka" # change to base_half if using half res + extrinsics_key: "mecka" + + diffusion: true + 6dof: true + + ac_keys: + mecka_bimanual: "actions_cartesian" + + trunk: + embed_dim: 1176 # changed from 256 #96 + num_blocks: 20 # changed from 16 + num_heads: 12 # changed from 8 + token_postprocessing: "action_token" + observation_horizon: 1 + action_horizon: 64 + no_trunk: false + use_domain_embedding: true + drop_path: 0.1 + weight_init_style: "pytorch" + + multitask: false + pretrained: false + pretrained_checkpoint: "" # TODO + reverse_kl_samples: 8 + + domains: ["mecka_bimanual"] + shared_obs_keys: ["front_img_1"] + + shared_stem_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 1176 # changed from 512 + output_dim: 1176 #changed + widths: [1176] # changed from 768 + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 20 + crossattn_heads: 14 + crossattn_dim_head: 308 # changed from 256 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1176 # changed from 768 + + stem_specs: + mecka_bimanual: + state_ee_pose: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 12 + output_dim: 1176 # changed from 768 + widths: [1176] # changed from 768 + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 20 + crossattn_heads: 14 + crossattn_dim_head: 308 # changed from 256 changed from 1176 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1176 # changed from 768 changed from 1536 + + head_specs: + mecka_bimanual: + _target_: egomimic.models.fm_policy.FMPolicy + action_horizon: 100 + num_inference_steps: 50 + pooling: null + time_dist: "beta" + infer_ac_dims: + mecka_bimanual: 12 + model: + _target_: egomimic.models.denoising_nets.CrossTransformer + nblocks: 6 + cond_dim: 1176 # changed from 256 changed from 1536 + hidden_dim: 372 #changed from 128 + act_dim: 12 + act_seq: 100 + n_heads: 6 # changed from 4 changed from 16 + dropout: 0.1 + mlp_layers: 6 # edit num of mlp layers + mlp_ratio: 6 + + encoder_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.ResNet + output_dim: 1176 # changed from 512 changed from 1536 + + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.ColorJitter + brightness: 0.1 + contrast: 0.1 + saturation: 0.1 + hue: 0.05 + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 1e-4 + weight_decay: 0.0001 + +scheduler: + _target_: torch.optim.lr_scheduler.CosineAnnealingLR + _partial_: true + T_max: 5500 + eta_min: 1e-5 diff --git a/egomimic/hydra_configs/model/hpt_bc_flow_scale_1B.yaml b/egomimic/hydra_configs/model/hpt_bc_flow_scale_1B.yaml new file mode 100644 index 00000000..14992731 --- /dev/null +++ b/egomimic/hydra_configs/model/hpt_bc_flow_scale_1B.yaml @@ -0,0 +1,122 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.hpt.HPT + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + scale_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "scale" # change to base_half if using half res + extrinsics_key: "scale" + + diffusion: true + 6dof: true + + ac_keys: + scale_bimanual: "actions_cartesian" + + trunk: + embed_dim: 1428 #94 + num_blocks: 26 + num_heads: 14 + token_postprocessing: "action_token" + observation_horizon: 1 + action_horizon: 64 + no_trunk: false + use_domain_embedding: true + drop_path: 0.1 + weight_init_style: "pytorch" + + multitask: false + pretrained: false + pretrained_checkpoint: "" # TODO + reverse_kl_samples: 8 + + domains: ["scale_bimanual"] + shared_obs_keys: ["front_img_1"] + + shared_stem_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 1428 + output_dim: 1428 + widths: [1428] + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 22 + crossattn_heads: 16 + crossattn_dim_head: 416 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1428 + + stem_specs: + scale_bimanual: + state_ee_pose: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 12 + output_dim: 1428 + widths: [1428] + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 22 + crossattn_heads: 16 + crossattn_dim_head: 416 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1428 + + head_specs: + scale_bimanual: + _target_: egomimic.models.fm_policy.FMPolicy + action_horizon: 100 + num_inference_steps: 50 + pooling: null + time_dist: "beta" + infer_ac_dims: + scale_bimanual: 12 + model: + _target_: egomimic.models.denoising_nets.CrossTransformer + nblocks: 6 + cond_dim: 1428 + hidden_dim: 408 + act_dim: 12 + act_seq: 100 + n_heads: 6 + dropout: 0.1 + mlp_layers: 7 + mlp_ratio: 7 + + encoder_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.ResNet + output_dim: 1428 + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.ColorJitter + brightness: 0.1 + contrast: 0.1 + saturation: 0.1 + hue: 0.05 + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 3e-4 + weight_decay: 0.0001 + +scheduler: + _target_: torch.optim.lr_scheduler.CosineAnnealingLR + _partial_: true + T_max: 5500 + eta_min: 1e-5 diff --git a/egomimic/hydra_configs/model/hpt_bc_flow_scale_600M.yaml b/egomimic/hydra_configs/model/hpt_bc_flow_scale_600M.yaml new file mode 100644 index 00000000..d9bf2c48 --- /dev/null +++ b/egomimic/hydra_configs/model/hpt_bc_flow_scale_600M.yaml @@ -0,0 +1,123 @@ +_target_: egomimic.pl_utils.pl_model.ModelWrapper +robomimic_model: + _target_: egomimic.algo.hpt.HPT + data_schematic: _${data.dataset.data_schematic} + camera_transforms: + scale_bimanual: + _target_: egomimic.utils.egomimicUtils.CameraTransforms + intrinsics_key: "scale" # change to base_half if using half res + extrinsics_key: "scale" + + diffusion: true + 6dof: true + + ac_keys: + scale_bimanual: "actions_cartesian" + + trunk: + embed_dim: 1176 # changed from 256 #96 + num_blocks: 20 # changed from 16 + num_heads: 12 # changed from 8 + token_postprocessing: "action_token" + observation_horizon: 1 + action_horizon: 64 + no_trunk: false + use_domain_embedding: true + drop_path: 0.1 + weight_init_style: "pytorch" + + multitask: false + pretrained: false + pretrained_checkpoint: "" # TODO + reverse_kl_samples: 8 + + domains: ["scale_bimanual"] + shared_obs_keys: ["front_img_1"] + + shared_stem_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 1176 # changed from 512 + output_dim: 1176 #changed + widths: [1176] # changed from 768 + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 20 + crossattn_heads: 14 + crossattn_dim_head: 308 # changed from 256 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1176 # changed from 768 + + stem_specs: + scale_bimanual: + state_ee_pose: + _target_: egomimic.models.hpt_nets.MLPPolicyStem + input_dim: 12 + output_dim: 1176 # changed from 768 + widths: [1176] # changed from 768 + specs: + random_horizon_masking: false + cross_attn: + crossattn_latent: 20 + crossattn_heads: 14 + crossattn_dim_head: 308 # changed from 256 changed from 1176 + crossattn_modality_dropout: 0.1 + modality_embed_dim: 1176 # changed from 768 changed from 1536 + + head_specs: + scale_bimanual: + _target_: egomimic.models.fm_policy.FMPolicy + action_horizon: 100 + num_inference_steps: 50 + pooling: null + time_dist: "beta" + infer_ac_dims: + scale_bimanual: 12 + model: + _target_: egomimic.models.denoising_nets.CrossTransformer + nblocks: 6 + cond_dim: 1176 # changed from 256 changed from 1536 + hidden_dim: 372 #changed from 128 + act_dim: 12 + act_seq: 100 + n_heads: 6 # changed from 4 changed from 16 + dropout: 0.1 + mlp_layers: 6 # edit num of mlp layers + mlp_ratio: 6 + + encoder_specs: + front_img_1: + _target_: egomimic.models.hpt_nets.ResNet + output_dim: 1176 # changed from 512 changed from 1536 + + + train_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.ColorJitter + brightness: 0.1 + contrast: 0.1 + saturation: 0.1 + hue: 0.05 + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + eval_image_augs: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.Normalize + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +optimizer: + _target_: torch.optim.AdamW + _partial_: true + lr: 1e-4 + weight_decay: 0.0001 + +scheduler: + _target_: torch.optim.lr_scheduler.CosineAnnealingLR + _partial_: true + T_max: 5500 + eta_min: 1e-5 diff --git a/egomimic/hydra_configs/model/hpt_cotrain_mecka_flow_shared_head.yaml b/egomimic/hydra_configs/model/hpt_cotrain_mecka_flow_shared_head.yaml index 990c3bd9..f6cbe1a8 100644 --- a/egomimic/hydra_configs/model/hpt_cotrain_mecka_flow_shared_head.yaml +++ b/egomimic/hydra_configs/model/hpt_cotrain_mecka_flow_shared_head.yaml @@ -53,7 +53,7 @@ robomimic_model: _target_: egomimic.models.denoising_nets.CrossTransformer nblocks: 6 cond_dim: 256 - hidden_dim: 128 + hidden_dim: 256 act_dim: 14 act_seq: 100 n_heads: 4 @@ -71,4 +71,4 @@ scheduler: _target_: torch.optim.lr_scheduler.CosineAnnealingLR _partial_: true T_max: 1400 - eta_min: 1e-5 \ No newline at end of file + eta_min: 5e-5 \ No newline at end of file diff --git a/egomimic/hydra_configs/train.yaml b/egomimic/hydra_configs/train.yaml index 828a21b6..931b780c 100644 --- a/egomimic/hydra_configs/train.yaml +++ b/egomimic/hydra_configs/train.yaml @@ -1,102 +1,96 @@ -defaults: - - model: hpt_bc_flow_eva + defaults: + - model: hpt_bc_flow_mecka_1B - paths: default - trainer: ddp - debug: null - logger: wandb - - data: eva_bc_s3 + - data: mecka_all - callbacks: checkpoints - - override hydra/launcher: submitit + - override hydra/launcher: submitit_slurm - _self_ -name: test -description: test -ckpt_path: null -train: true -eval: false + name: test + description: test + ckpt_path: null + train: true + eval: false -eval_class: - _target_ : egomimic.scripts.evaluation.Eve - mode: real - arm: both - eval_path: "./logs/eval/${name}_${now:%Y-%m-%d_%H-%M-%S}" + eval_class: + _target_ : egomimic.scripts.evaluation.Eve + mode: real + arm: both + eval_path: "./logs/eval/${name}_${now:%Y-%m-%d_%H-%M-%S}" -hydra: - run: - # Dir should be experiment_name/description_{timestamp} - dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} - sweep: - dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} + hydra: + run: + # Dir should be experiment_name/description_{timestamp} + dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} + sweep: + dir: ./logs/${name}/${description}_${now:%Y-%m-%d_%H-%M-%S} -launch_params: - gpus_per_node: 1 - nodes: 1 + launch_params: + gpus_per_node: 2 + nodes: 1 -data_schematic: # Dynamically fill in these shapes from the dataset - _target_: egomimic.rldb.utils.DataSchematic - norm_mode: quantile - schematic_dict: - eva_bimanual: - front_img_1: #batch key - key_type: camera_keys # key type - lerobot_key: observations.images.front_img_1 # dataset key - right_wrist_img: - key_type: camera_keys - lerobot_key: observations.images.right_wrist_img - left_wrist_img: - key_type: camera_keys - lerobot_key: observations.images.left_wrist_img - ee_pose: - key_type: proprio_keys - lerobot_key: observations.state.ee_pose - joint_positions: - key_type: proprio_keys - lerobot_key: observations.state.joint_positions - actions_joints: - key_type: action_keys - lerobot_key: actions_joints - actions_cartesian: - key_type: action_keys - lerobot_key: actions_cartesian - embodiment: - key_type: metadata_keys - lerobot_key: metadata.embodiment - aria_bimanual: - front_img_1: - key_type: camera_keys - lerobot_key: observations.images.front_img_1 - ee_pose: - key_type: proprio_keys - lerobot_key: observations.state.ee_pose - actions_cartesian: - key_type: action_keys - lerobot_key: actions_cartesian - embodiment: - key_type: metadata_keys - lerobot_key: metadata.embodiment - mecka_bimanual: - front_img_1: - key_type: camera_keys - lerobot_key: observations.images.front_img_1 - ee_pose: - key_type: proprio_keys - lerobot_key: observations.state.ee_pose_cam - actions_cartesian: - key_type: action_keys - lerobot_key: actions_ee_cartesian_cam - actions_keypoints: - key_type: action_keys - lerobot_key: actions_ee_keypoints_world - actions_head_cartesian: - key_type: action_keys - lerobot_key: actions_head_cartesian_world - embodiment: - key_type: metadata_keys - lerobot_key: metadata.embodiment - viz_img_key: - eva_bimanual: - front_img_1 - aria_bimanual: - front_img_1 - mecka_bimanual: - front_img_1 + data_schematic: # Dynamically fill in these shapes from the dataset + _target_: egomimic.rldb.utils.DataSchematic + norm_mode: quantile + schematic_dict: + eva_bimanual: + front_img_1: #batch key + key_type: camera_keys # key type + lerobot_key: observations.images.front_img_1 # dataset key + right_wrist_img: + key_type: camera_keys + lerobot_key: observations.images.right_wrist_img + left_wrist_img: + key_type: camera_keys + lerobot_key: observations.images.left_wrist_img + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose + joint_positions: + key_type: proprio_keys + lerobot_key: observations.state.joint_positions + actions_joints: + key_type: action_keys + lerobot_key: actions_joints + actions_cartesian: + key_type: action_keys + lerobot_key: actions_cartesian + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + aria_bimanual: + front_img_1: + key_type: camera_keys + lerobot_key: observations.images.front_img_1 + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose + actions_cartesian: + key_type: action_keys + lerobot_key: actions_cartesian + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + mecka_bimanual: + front_img_1: + key_type: camera_keys + lerobot_key: observations.images.front_img_1 + ee_pose: + key_type: proprio_keys + lerobot_key: observations.state.ee_pose_cam + actions_cartesian: + key_type: action_keys + lerobot_key: actions_ee_cartesian_cam + embodiment: + key_type: metadata_keys + lerobot_key: metadata.embodiment + viz_img_key: + eva_bimanual: + front_img_1 + aria_bimanual: + front_img_1 + mecka_bimanual: + front_img_1 diff --git a/egomimic/hydra_configs/train_zarr.yaml b/egomimic/hydra_configs/train_zarr.yaml index 0cc53a23..1683d725 100644 --- a/egomimic/hydra_configs/train_zarr.yaml +++ b/egomimic/hydra_configs/train_zarr.yaml @@ -6,7 +6,7 @@ defaults: - logger: wandb - data: eva - callbacks: checkpoints - - override hydra/launcher: submitit + - override hydra/launcher: submitit_skynet - _self_ name: test diff --git a/egomimic/hydra_configs/trainer/default.yaml b/egomimic/hydra_configs/trainer/default.yaml index a6b47e35..72e3e992 100644 --- a/egomimic/hydra_configs/trainer/default.yaml +++ b/egomimic/hydra_configs/trainer/default.yaml @@ -2,7 +2,7 @@ _target_: lightning.pytorch.trainer.Trainer default_root_dir: ${paths.output_dir} -max_epochs: 2000 +max_epochs: 8000 min_epochs: 2000 accelerator: cpu @@ -11,9 +11,9 @@ devices: 1 # mixed precision for extra speed-up precision: bf16 limit_train_batches: 100 -limit_val_batches: 300 +limit_val_batches: 200 # perform a validation loop every N training epochs -check_val_every_n_epoch: 200 +check_val_every_n_epoch: 500 # set True to to ensure deterministic results # makes training slower but gives more reproducibility than just setting seeds diff --git a/egomimic/pl_utils/pl_model.py b/egomimic/pl_utils/pl_model.py index d44c1270..a7c1dc43 100644 --- a/egomimic/pl_utils/pl_model.py +++ b/egomimic/pl_utils/pl_model.py @@ -142,6 +142,14 @@ def on_validation_start(self): exist_ok=True, ) + + # def train_validation(self): + # """ + # Run a validation step on the training data. + # """ + # self.validation_step(self.train_dataloader(), 0) + # self.on_validation_end() + def validation_step(self, batch, batch_idx, dataloader_idx=0): """ Run a validation step on the batch, and save that batch of images into the val_image_buffer. Once the buffer hits 1000 images, save that as a 30fps video using torchvision.io.write_video. @@ -187,6 +195,7 @@ def validation_step(self, batch, batch_idx, dataloader_idx=0): self.val_counter[key] += 1 self.log_dict(metrics, sync_dist=True) + # self.train_validation() def on_validation_end(self): print(f"[ON_VALIDATION_END] rank={self.global_rank}", flush=True) diff --git a/egomimic/rldb/utils.py b/egomimic/rldb/utils.py index ff9bcee7..50668747 100644 --- a/egomimic/rldb/utils.py +++ b/egomimic/rldb/utils.py @@ -650,6 +650,7 @@ def __init__( logger.info(f"Local Files Only: {local_files_only}") logger.info(f"Percent: {percent}") logger.info(f"Valid Ratio: {valid_ratio}") + logger.info(f"Mode: {mode}") logger.info(f"Debug: {debug}") logger.info(f"kwargs: {kwargs}") @@ -669,11 +670,10 @@ def __init__( valid_collection_names.add(hashes) max_workers = int(os.environ.get("RLDB_LOAD_WORKERS", "10")) - datasets, skipped = self._load_rldb_datasets_parallel( search_path=search_path, embodiment=embodiment, - valid_collection_names=valid_collection_names, + collection_names=selected_collection_names, local_files_only=local_files_only, percent=percent, valid_ratio=valid_ratio, @@ -684,31 +684,6 @@ def __init__( assert datasets, "No valid RLDB datasets found! Check your S3 path and filters." - self.train_collections, self.valid_collections = split_dataset_names( - datasets.keys(), valid_ratio=valid_ratio, seed=SEED - ) - - if mode == "train": - chosen = self.train_collections - elif mode == "valid": - chosen = self.valid_collections - elif mode == "total": - chosen = set(datasets.keys()) - elif mode == "percent": - all_names = sorted(datasets.keys()) - rng = random.Random(SEED) - rng.shuffle(all_names) - - n_keep = int(len(all_names) * percent) - if percent > 0.0: - n_keep = max(1, n_keep) - chosen = set(all_names[:n_keep]) - else: - raise ValueError(f"Unknown mode: {mode}") - - datasets = {rid: ds for rid, ds in datasets.items() if rid in chosen} - assert datasets, "No datasets left after applying mode split." - key_map_per_dataset = ( {repo_id: key_map for repo_id in datasets} if key_map else None ) @@ -776,7 +751,7 @@ def _load_rldb_datasets_parallel( *, search_path: Path, embodiment: str, - valid_collection_names: set[str], + collection_names: set[str], local_files_only: bool, percent: float, valid_ratio: float, @@ -800,7 +775,7 @@ def _load_rldb_datasets_parallel( if debug: logger.info("Debug mode: limiting to 10 datasets.") - valid_collection_names = set(list(valid_collection_names)[:10]) + collection_names = set(list(collection_names)[:10]) def _submit_arg(p: Path): return dict( diff --git a/egomimic/rldb/zarr/zarr_dataset_multi.py b/egomimic/rldb/zarr/zarr_dataset_multi.py index a2971569..9fa641a9 100644 --- a/egomimic/rldb/zarr/zarr_dataset_multi.py +++ b/egomimic/rldb/zarr/zarr_dataset_multi.py @@ -237,9 +237,58 @@ def _get_filtered_paths( logger.info(f"Paths: {paths}") return paths + @classmethod + def episode_already_present_for_progress_bar( + cls, local_dir: Path, episode_hash: str + ) -> bool: + """True if this episode is already on disk. For progress bar polling only.""" + return cls._episode_already_present(local_dir, episode_hash) + + @classmethod + def get_to_sync_paths_for_progress_bar( + cls, + filters: dict, + local_dir: Path, + debug: bool = False, + ) -> tuple[list[tuple[str, str]], list[str]]: + """ + Resolve DB filter and split into to_sync vs already present. + For progress bar only: use this to get total count and list before running sync in a thread. + Returns (to_sync, already) where to_sync is list of (processed_path, episode_hash). + """ + filtered_paths = cls._get_filtered_paths(filters, debug=debug) + to_sync = [] + already = [] + for processed_path, episode_hash in filtered_paths: + if cls._episode_already_present(local_dir, episode_hash): + already.append(episode_hash) + else: + to_sync.append((processed_path, episode_hash)) + return (to_sync, already) + + @classmethod + def run_sync_for_progress_bar( + cls, + bucket_name: str, + to_sync: list[tuple[str, str]], + local_dir: Path, + numworkers: int = 128, + ) -> None: + """Run s5cmd sync for the given to_sync list. For progress bar: call this in a background thread.""" + cls._sync_s3_to_local( + bucket_name=bucket_name, + s3_paths=to_sync, + local_dir=local_dir, + numworkers=numworkers, + ) + @classmethod def _sync_s3_to_local( - cls, bucket_name: str, s3_paths: list[tuple[str, str]], local_dir: Path + cls, + bucket_name: str, + s3_paths: list[tuple[str, str]], + local_dir: Path, + numworkers: int = 128, ): if not s3_paths: return @@ -295,9 +344,34 @@ def _sync_s3_to_local( os.environ["AWS_SECRET_ACCESS_KEY"] = secret_access_key os.environ["AWS_DEFAULT_REGION"] = "auto" os.environ["AWS_REGION"] = "auto" - cmd = ["s5cmd", "--endpoint-url", rl2_endpoint_url, "run", str(batch_path)] - logger.info("Running s5cmd batch (%d lines): %s", len(lines), " ".join(cmd)) - subprocess.run(cmd, check=True) + cmd = [ + "s5cmd", + "--endpoint-url", + rl2_endpoint_url, + "--numworkers", + str(numworkers), + "run", + str(batch_path), + ] + # Uncomment to log downloads + # logger.info( + # "Running s5cmd batch (%d episodes to sync, numworkers=%s): %s", + # len(lines), + # numworkers, + # " ".join(cmd), + # ) + result = subprocess.run( + cmd, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + err = (result.stderr or "").strip() + raise subprocess.CalledProcessError( + result.returncode, cmd, stderr=err or None + ) finally: try: @@ -312,14 +386,17 @@ def sync_from_filters( bucket_name: str, filters: dict, local_dir: Path, + numworkers: int = 128, debug: bool = False, ): """ Public API: - resolves episodes from DB using filters - runs a single aws s3 sync with includes - - downloads into local_dir + - downloads into local_dir (skips episodes already present locally) + Args: + numworkers: Number of parallel workers for s5cmd (default 128). Returns: List[(processed_path, episode_hash)] @@ -336,11 +413,12 @@ def sync_from_filters( f"Syncing S3 datasets with filters {filters} to local directory {local_dir}..." ) - # 3) Sync + # 3) Sync (already-present episodes are skipped inside _sync_s3_to_local) cls._sync_s3_to_local( bucket_name=bucket_name, s3_paths=filtered_paths, local_dir=local_dir, + numworkers=numworkers, ) return filtered_paths diff --git a/external/openpi b/external/openpi index 981483dc..5bff19b0 160000 --- a/external/openpi +++ b/external/openpi @@ -1 +1 @@ -Subproject commit 981483dca0fd9acba698fea00aa6e52d56a66c58 +Subproject commit 5bff19b0c0c447c7a7eaaaccf03f36d50998ec9d diff --git a/sync_s3.py b/sync_s3.py new file mode 100644 index 00000000..59078c33 --- /dev/null +++ b/sync_s3.py @@ -0,0 +1,162 @@ +""" +Sync EgoVerse data from S3/R2 to a local directory. + +Progress (tqdm) and benchmarking live only here; the resolver does plain sync. +Works when s5cmd runs with many workers (parallel downloads): we poll the +filesystem for completed episode dirs and update the bar. +""" + +import argparse +import json +import sys +import threading +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from egomimic.rldb.zarr.zarr_dataset_multi import S3EpisodeResolver +from egomimic.utils.aws.aws_data_utils import load_env + +try: + from tqdm import tqdm +except ImportError: + tqdm = None + + +def _count_present(local_dir: Path, to_sync: list[tuple[str, str]]) -> int: + return sum( + 1 + for _, episode_hash in to_sync + if S3EpisodeResolver.episode_already_present_for_progress_bar(local_dir, episode_hash) + ) + + +def run_resolver_sync( + *, + bucket_name: str, + local_dir: str | Path, + filters: dict, + numworkers: int = 128, + show_progress: bool = True, + debug: bool = False, +): + load_env() + local_dir = Path(local_dir) + + to_sync, already = S3EpisodeResolver.get_to_sync_paths_for_progress_bar( + filters, local_dir, debug=debug + ) + if already: + print(f"Skipping {len(already)} episodes already present locally.") + if not to_sync: + print("Nothing to sync (all episodes already present or none matched).") + return [] + + n = len(to_sync) + sync_error: list[Exception] = [] + + def do_sync() -> None: + try: + S3EpisodeResolver.run_sync_for_progress_bar( + bucket_name=bucket_name, + to_sync=to_sync, + local_dir=local_dir, + numworkers=numworkers, + ) + except Exception as e: + sync_error.append(e) + + t0 = time.perf_counter() + thread = threading.Thread(target=do_sync, daemon=False) + thread.start() + + if show_progress and tqdm is not None: + with tqdm( + total=n, + unit="ep", + desc="sync", + dynamic_ncols=True, + ) as pbar: + while True: + pbar.n = min(n, _count_present(local_dir, to_sync)) + pbar.refresh() + if not thread.is_alive(): + pbar.n = min(n, _count_present(local_dir, to_sync)) + pbar.refresh() + break + time.sleep(0.25) + thread.join() + + if sync_error: + raise sync_error[0] + + elapsed = time.perf_counter() - t0 + print( + f"[sync_s3] Benchmark: {elapsed:.1f} s total for {n} episode(s) " + f"(~{elapsed / n:.2f} s per episode)" + ) + return [(p, h) for p, h in to_sync] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Sync EgoVerse data from S3/R2 to a local directory." + ) + parser.add_argument( + "--bucket-name", + type=str, + required=False, + default="rldb", + help="S3 bucket name (default: rldb).", + ) + parser.add_argument( + "--local-dir", + type=str, + required=False, + default="/coc/flash7/scratch/egoverseS3ZarrDataset", + help="Local directory to sync into.", + ) + parser.add_argument( + "--numworkers", + type=int, + required=False, + default=128, + help="s5cmd parallel workers.", + ) + parser.add_argument( + "--show-progress", + action="store_true", + help="Show tqdm progress bar (default). Kept for backward compatibility.", + ) + parser.add_argument( + "--no-progress", + action="store_true", + help="Disable tqdm progress bar (benchmark still printed).", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Limit to 10 matched episodes (resolver debug mode).", + ) + parser.add_argument( + "--filters", + type=str, + required=False, + default="{}", + help='JSON dict of SQL filters, e.g. \'{"episode_hash": ["h1","h2"], "robot_name": "eva"}\'.', + ) + args = parser.parse_args() + + filters = json.loads(args.filters) + if not isinstance(filters, dict): + raise ValueError("--filters must be a JSON object (dict).") + + run_resolver_sync( + bucket_name=args.bucket_name, + local_dir=args.local_dir, + filters=filters, + numworkers=args.numworkers, + show_progress=args.show_progress or not args.no_progress, + debug=args.debug, + ) diff --git a/sync_s3.sbatch b/sync_s3.sbatch new file mode 100644 index 00000000..c3a5a22b --- /dev/null +++ b/sync_s3.sbatch @@ -0,0 +1,20 @@ +#!/bin/bash +#SBATCH --job-name=sync_s3 +#SBATCH --partition=CPU-3TB-AMD # Keep your lab partition (CPU jobs work here) +#SBATCH --account=gts-dxu345-rl2 +#SBATCH --mem=100G +#SBATCH --ntasks=1 +#SBATCH --nodes=1 # One node only +#SBATCH --output=sync_s3_%j.out +#SBATCH --error=sync_s3_%j.err +#SBATCH --qos=inferno + +# Usage: +# sbatch sync_s3.sbatch -- [sync_s3.py args...] +# +# Everything after `--` is passed directly to `sync_s3.py`. + +set -euo pipefail + +SUBMIT_DIR="${SLURM_SUBMIT_DIR:-$PWD}" +python "$SUBMIT_DIR/sync_s3.py" "$@"