From 7f732bb5c02ccd1f337030fd82aa5eaf903eb7ed Mon Sep 17 00:00:00 2001 From: maple5717 Date: Tue, 23 Apr 2024 00:24:30 +0800 Subject: [PATCH 01/18] added metadata --- fy/base.py | 26 ++++++++++++++++++++------ fy/collision.py | 4 +++- fy/run_tiancheng.py | 21 +++++++++++++++------ fy/solidity.py | 2 ++ fy/support.py | 2 +- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/fy/base.py b/fy/base.py index 6d3f13eb4..82b6c3091 100644 --- a/fy/base.py +++ b/fy/base.py @@ -79,6 +79,9 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.table_name = "table_kb" self.block_name = "block_kb" + self.block_id = None + self.test_obj_id = [] + self.render_data = ("rgba",) self.background_hdri_id = FLAGS.background_hdri_id @@ -88,6 +91,8 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.test_obj = None self.default_camera_pos = [0, 0, 1] self.camera_look_at = [0,0,0] + self.frame_violation_start = -1 + self.violation_type = -1 # load asset sources self.kubasic = kubasic_assets # kb.AssetSource.from_manifest(FLAGS.kubasic_assets) @@ -161,7 +166,7 @@ def _setup_everything(self, shift=[0, 5, 0]): use_indoor = True else: rnd_n = random.random() - use_indoor = rnd_n < 0.5 + use_indoor = 0#rnd_n < 0.5 if use_indoor: self._setup_indoor_scene() @@ -330,7 +335,7 @@ def _random_rotate_scene(self): def prepare_scene(self): """Generate a new random test scene""" - self.i = 0 + # self.i = 0 while True: self.dynamic_objs = [] @@ -340,7 +345,7 @@ def prepare_scene(self): return logging.warning("Current scene is invalid. Regenerating ") # self.renderer.save_state(f"temp_scene/invalid_{self.i}.blend") - self.i += 1 + # self.i += 1 if self.is_move_camera: self.camera_path_sample_stats[self.cur_camera_traj_idx] += 1 @@ -371,7 +376,7 @@ def load_blender_scene(self, blender_scene): """ scene = core.scene.Scene.from_flags(self.flags) simulator = PyBullet(scene, self.scratch_dir) - renderer = Blender(scene, self.scratch_dir,custom_scene=blender_scene) + renderer = Blender(scene, self.scratch_dir,custom_scene=blender_scene, samples_per_pixel=64) self.scene = scene self.simulator = simulator self.renderer = renderer @@ -670,7 +675,13 @@ def write_metadata(self): "metadata": kb.get_scene_metadata(self.scene), "camera": kb.get_camera_info(self.scene.camera), "instances": kb.get_instance_info(self.scene), - "table_id": self.table_id + "has_table": self.add_table, + "table_id": self.table_id, + "test_obj_id": self.test_obj_id, + "block_id": self.block_id, + "fps": self.flags.frame_rate, + "volation_frame_start_at": self.frame_violation_start, + "test_case": self.violation_type }) def change_output_dir(self, new_output_dir): @@ -757,7 +768,7 @@ def add_block_objects(self): aligh_block_objs(self.block_obj) self.block_obj.position = (0, 0, self.ref_h + 0.001 - self.block_obj.aabbox[0][2]) - + self.block_id = block_obj_id # randomly jitter the object ... @@ -883,5 +894,8 @@ def _set_fast_rendering(self): # fast gi approximation bpy.context.scene.cycles.use_fast_gi = True + bpy.context.scene.cycles.debug_use_spatial_splits = False + + \ No newline at end of file diff --git a/fy/collision.py b/fy/collision.py index e8adf8092..32302aca4 100644 --- a/fy/collision.py +++ b/fy/collision.py @@ -79,6 +79,7 @@ def generate_keyframes(self): raise RuntimeError("No collision detected") self.first_collision_frame = first_collision_frame + self.frame_violation_start = self.first_collision_frame logging.debug(f"first_collision_frame: {first_collision_frame}") # make the objects fall straight down after the collision @@ -158,7 +159,8 @@ def add_test_objects(self): name="big_obj") # 2.8 self.test_obj = [obj_1, obj_2] - + self.test_obj_id = [obj_1_id, obj_2_id] + return obj_1, obj_2 def _check_scene(self): diff --git a/fy/run_tiancheng.py b/fy/run_tiancheng.py index dac311f4a..721d281b3 100644 --- a/fy/run_tiancheng.py +++ b/fy/run_tiancheng.py @@ -72,16 +72,16 @@ def main() -> None: num_per_cls = 50 - max_trails = 40 if not(FLAGS.debug) else 10 + max_trails = 20 if not(FLAGS.debug) else 10 test_cls_all = { - "solidity": SolidityTestScene, + # "solidity": SolidityTestScene, # "continuity": ContinuityTestScene, # "Support": SupportTestScene, - # "collision": CollisionTestScene + "collision": CollisionTestScene # "Permanance": PermananceTestScene } for test_name, test_cls in test_cls_all.items(): - n = 0 + n = 0; i = 0 for i in tqdm(range(max_trails)): output_dir = f"{output_dirname}/{test_name}/scene_{i}/" if os.path.isdir(output_dir): @@ -97,8 +97,17 @@ def main() -> None: if test_name in ["Support", "continuity"]: FLAGS.move_camera = False - generate_test_scene(test_cls, FLAGS, output_dir, video_dir, i) - n += 1 + while True: + try: + generate_test_scene(test_cls, FLAGS, output_dir, video_dir, i) + i += 1 + break + except Exception as e: + logging.error(f"Error rendering collision test {n}: {e}\n Skipping to the next one.") + # if debug is on, raise the exception + # if FLAGS.debug: + # raise + continue # if n >= num_per_cls: # break diff --git a/fy/solidity.py b/fy/solidity.py index 0fb679dfe..fc7d6a5d5 100644 --- a/fy/solidity.py +++ b/fy/solidity.py @@ -28,6 +28,7 @@ def __init__(self, FLAGS) -> None: self.collision_z_distance = 0.1 # distance between obj_1 and obj_2 in z direction self.collision_height = 1.2 + self.frame_violation_start = self.violation_time * self.flags.frame_rate self.default_camera_pos = spherical_to_cartesian(r_range=[2.5, 3], theta_range=[89, 91], phi_range=[-5, 5]) # (0, -1, 1.7) self.camera_look_at = [0, 0, self.collision_height] @@ -143,6 +144,7 @@ def add_test_objects(self): name="big_obj") self.test_obj = [obj_1, obj_2] + self.test_obj_id = [obj_1_id, obj_2_id] return obj_1, obj_2 diff --git a/fy/support.py b/fy/support.py index 7cd9606f1..dc2da0b52 100644 --- a/fy/support.py +++ b/fy/support.py @@ -27,7 +27,7 @@ def __init__(self, FLAGS) -> None: self.frame_violation_start = 10 self.initial_dist_to_table = 0 # 1: float in air; 0: pass through table - self.violation_type = np.random.binomial(n=1,p=0.5) + self.violation_type = 1#np.random.binomial(n=1,p=0.5) self.gravity = [0, 0, -1.5] self.is_move_camera = False From bcea1abcb510cd5425026f82ff0493c9f73d0710 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Wed, 24 Apr 2024 15:32:29 +0800 Subject: [PATCH 02/18] merged again --- fy/base.py | 47 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/fy/base.py b/fy/base.py index 87baef8ed..bd385005a 100644 --- a/fy/base.py +++ b/fy/base.py @@ -19,8 +19,8 @@ # --- Some configuration values # the region in which to place objects [(min), (max)] -STATIC_SPAWN_REGION = [(-1, -1, 0), (1, 1, 5)] # for static objects -DYNAMIC_SPAWN_REGION = [(-1, -1, 1), (1, 1, 5)] +STATIC_SPAWN_REGION = [(-4, -1, 0), (4, 7, 10)] # for static objects +DYNAMIC_SPAWN_REGION = [(-5, -5, 1), (5, 5, 5)] VELOCITY_RANGE = [(-4., -4., 0.), (4., 4., 0.)] SCENE_EXCLUDE = ["wobbly_bridge"] @@ -80,9 +80,6 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.table_name = "table_kb" self.block_name = "block_kb" - self.block_id = None - self.test_obj_id = [] - self.render_data = ("rgba",) self.background_hdri_id = FLAGS.background_hdri_id @@ -92,8 +89,6 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.test_obj = None self.default_camera_pos = [0, 0, 1] self.camera_look_at = [0,0,0] - self.frame_violation_start = -1 - self.violation_type = -1 # load asset sources self.kubasic = kubasic_assets # kb.AssetSource.from_manifest(FLAGS.kubasic_assets) @@ -161,7 +156,7 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.object_asset_id_list = self.gso.all_asset_ids - def _setup_everything(self): + def _setup_everything(self, shift=[0, 5, 0]): if self.flags.scene_type == "indoor": use_indoor = True @@ -169,17 +164,14 @@ def _setup_everything(self): use_indoor = False else: rnd_n = random.random() - use_indoor = 0#rnd_n < 0.5 + use_indoor = rnd_n < 0.5 if use_indoor: self._setup_indoor_scene() - # indoor scene should not add dynamic objects, because the scene is not in simulation state - self.is_add_background_dynamic_objects = False else: self._setup_hdri_scene() self.ref_h = 0 - # for hdri scene - self.is_add_background_dynamic_objects = True + self.scene.gravity = self.gravity @@ -345,7 +337,7 @@ def _random_rotate_scene(self): def prepare_scene(self): """Generate a new random test scene""" - # self.i = 0 + self.i = 0 while True: self.dynamic_objs = [] self.static_objs = [] @@ -356,7 +348,7 @@ def prepare_scene(self): return logging.warning("Current scene is invalid. Regenerating ") # self.renderer.save_state(f"temp_scene/invalid_{self.i}.blend") - # self.i += 1 + self.i += 1 if self.is_move_camera: self.camera_path_sample_stats[self.cur_camera_traj_idx] += 1 @@ -387,7 +379,7 @@ def load_blender_scene(self, blender_scene): """ scene = core.scene.Scene.from_flags(self.flags) simulator = PyBullet(scene, self.scratch_dir) - renderer = Blender(scene, self.scratch_dir,custom_scene=blender_scene, samples_per_pixel=64) + renderer = Blender(scene, self.scratch_dir,custom_scene=blender_scene) self.scene = scene self.simulator = simulator self.renderer = renderer @@ -436,9 +428,6 @@ def _setup_hdri_scene(self): print("Setting up the Camera...") self._add_camera(scene) - # default camera lookat for hdri scene - self.camera_look_at = [0, 0, 1.2] - self.default_camera_pos = [0, -4, 1.5] if self.is_add_table: logging.info("Adding table to the scene") table_id = rng.choice(self.shapenet_table_ids) @@ -490,8 +479,8 @@ def _setup_indoor_scene(self, self.load_blender_scene(blender_scene) self._add_camera(self.scene) - # self.scene.camera.position = self.default_camera_pos - # self.scene.camera.look_at(self.camera_look_at) + self.scene.camera.position = self.default_camera_pos + self.scene.camera.look_at(self.camera_look_at) # add floor to the scene logging.info("Adding floor to the scene") @@ -506,8 +495,6 @@ def _setup_indoor_scene(self, bpy.data.objects[self.floor_name].hide_viewport = True - self.default_camera_pos = [0, -1.8, 1] - self.camera_look_at = [0, 0, 0.3] if self.is_add_table: logging.info("Adding table to the scene") table_id = rng.choice(self.shapenet_table_ids) @@ -620,6 +607,8 @@ def add_object(self, kb.move_until_no_overlap(obj, self.simulator, spawn_region=STATIC_SPAWN_REGION, rng=self.rng) + # temporarily set false + # to mitigate error "'XXXTestScene' object has no attribute 'gravity'" if is_dynamic: # reduce the restitution of the object to make it less bouncy # account for the gravity @@ -698,7 +687,12 @@ def write_metadata(self): "metadata": kb.get_scene_metadata(self.scene), "camera": kb.get_camera_info(self.scene.camera), "instances": kb.get_instance_info(self.scene), - "table_id": self.table_id + "test_obj": test_obj_names, + "dynamic_objs": dynamic_obj_names, + "static_objs": static_obj_names, + "block_obj": block_obj_name, + "table": table_asset_id, + }) def change_output_dir(self, new_output_dir): @@ -788,7 +782,7 @@ def add_block_objects(self): aligh_block_objs(self.block_obj) self.block_obj.position = (0, 0, self.ref_h + 0.001 - self.block_obj.aabbox[0][2]) - self.block_id = block_obj_id + # randomly jitter the object ... @@ -914,8 +908,5 @@ def _set_fast_rendering(self): # fast gi approximation bpy.context.scene.cycles.use_fast_gi = True - bpy.context.scene.cycles.debug_use_spatial_splits = False - - \ No newline at end of file From fc3732b4d163078ee57bdf222cc0548fd2df125b Mon Sep 17 00:00:00 2001 From: maple5717 Date: Wed, 24 Apr 2024 16:18:24 +0800 Subject: [PATCH 03/18] merge new --- fy/base.py | 115 +++++++++++++++----------- fy/collision.py | 7 +- fy/collision_free_fall.py | 164 ++++++++++++++++++++++++++++++++++++++ fy/run.py | 25 +++--- fy/solidity.py | 2 +- fy/utils.py | 16 ++-- 6 files changed, 262 insertions(+), 67 deletions(-) create mode 100644 fy/collision_free_fall.py diff --git a/fy/base.py b/fy/base.py index 82b6c3091..ec8d31630 100644 --- a/fy/base.py +++ b/fy/base.py @@ -19,8 +19,8 @@ # --- Some configuration values # the region in which to place objects [(min), (max)] -STATIC_SPAWN_REGION = [(-4, -1, 0), (4, 7, 10)] # for static objects -DYNAMIC_SPAWN_REGION = [(-5, -5, 1), (5, 5, 5)] +STATIC_SPAWN_REGION = [(-1, -1, 0), (1, 1, 5)] # for static objects +DYNAMIC_SPAWN_REGION = [(-1, -1, 1), (1, 1, 5)] VELOCITY_RANGE = [(-4., -4., 0.), (4., 4., 0.)] SCENE_EXCLUDE = ["wobbly_bridge"] @@ -70,6 +70,7 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.gravity = (0, 0, -9.81) self.dynamic_objs = [] + self.static_objs = [] self.block_obj = None self.ref_h = 0 self.table_scale = 2 @@ -153,28 +154,33 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.cur_camera_traj_idx = None self.is_add_block_objects = True self.is_move_camera = FLAGS.move_camera - self.add_table = True + self.is_add_table = True self.table_id = None self.is_add_background_static_objects = True self.is_add_background_dynamic_objects = True self.object_asset_id_list = self.gso.all_asset_ids - def _setup_everything(self, shift=[0, 5, 0]): + def _setup_everything(self): - if self.flags.use_indoor_only: + if self.flags.scene_type == "indoor": use_indoor = True + elif self.flags.scene_type == "hdri": + use_indoor = False else: rnd_n = random.random() use_indoor = 0#rnd_n < 0.5 if use_indoor: self._setup_indoor_scene() + # indoor scene should not add dynamic objects, because the scene is not in simulation state + self.is_add_background_dynamic_objects = False else: self._setup_hdri_scene() self.ref_h = 0 + # for hdri scene + self.is_add_background_dynamic_objects = True - self._random_rotate_scene() self.scene.gravity = self.gravity if self.flags.debug: @@ -191,8 +197,8 @@ def _setup_everything(self, shift=[0, 5, 0]): self.add_test_objects() # self._run_simulate() - - # self.shift_scene(shift) + self._random_rotate_scene() + if self.is_move_camera: traj_idx = random.randint(0, len(self.camera_path_config)-1) self._set_camera_path(self.camera_path_config[traj_idx]) @@ -202,6 +208,10 @@ def _setup_everything(self, shift=[0, 5, 0]): self.scene.camera.position = self.default_camera_pos self.scene.camera.look_at(self.camera_look_at) + if not use_indoor: + # the hdri center texture is wired, shift to avoid + self.shift_scene([0,5,0]) + if self.render_speedup: self._set_fast_rendering() @@ -338,6 +348,7 @@ def prepare_scene(self): # self.i = 0 while True: self.dynamic_objs = [] + self.static_objs = [] self._setup_everything() if self._check_scene(): @@ -425,9 +436,31 @@ def _setup_hdri_scene(self): print("Setting up the Camera...") self._add_camera(scene) - # each scene has a different camera setup, depending on the scene - scene.camera.position = (0, -3, 1.7) # height 1.7, away 2m - scene.camera.look_at((0, 0, 1)) + # default camera lookat for hdri scene + self.camera_look_at = [0, 0, 1.2] + self.default_camera_pos = [0, -4, 1.5] + if self.is_add_table: + logging.info("Adding table to the scene") + table_id = rng.choice(self.shapenet_table_ids) + table = shapenet_assets.create(asset_id=table_id, static=True, name=self.table_name) + # table_obj_id = self.shapenet_table_ids[0]#rng.choice(self.shapenet_table_ids) + # table = self.add_object(assed_id=self.shapenet_table_ids[0], name=self.table_name) + self.table = table + table.metadata["is_dynamic"] = False + table.scale = [self.table_scale] * 3 + table.quaternion = kb.Quaternion(axis=[1, 0, 0], degrees=90) + table.position = table.position - (0, 0, table.aabbox[0][2]) + table_h = table.aabbox[1][2] - table.aabbox[0][2] + scene.add(table) + set_name(self.table_name) + self.ref_h = table_h + self.table_id = table_id + # self.default_camera_pos[2] += table_h + # self.camera_look_at = [0, 0, self.ref_h] + + # # each scene has a different camera setup, depending on the scene + # scene.camera.position = (0, -3, 1.7) # height 1.7, away 2m + # scene.camera.look_at((0, 0, 1)) self.scene = scene self.simulator = simulator @@ -457,8 +490,8 @@ def _setup_indoor_scene(self, self.load_blender_scene(blender_scene) self._add_camera(self.scene) - self.scene.camera.position = self.default_camera_pos - self.scene.camera.look_at(self.camera_look_at) + # self.scene.camera.position = self.default_camera_pos + # self.scene.camera.look_at(self.camera_look_at) # add floor to the scene logging.info("Adding floor to the scene") @@ -473,7 +506,9 @@ def _setup_indoor_scene(self, bpy.data.objects[self.floor_name].hide_viewport = True - if self.add_table: + self.default_camera_pos = [0, -1.8, 1] + self.camera_look_at = [0, 0, 0.3] + if self.is_add_table: logging.info("Adding table to the scene") table_id = rng.choice(self.shapenet_table_ids) table = shapenet_assets.create(asset_id=table_id, static=True, name=self.table_name) @@ -490,7 +525,7 @@ def _setup_indoor_scene(self, self.ref_h = table_h self.default_camera_pos[2] += table_h self.table_id = table_id - print(self.table_id) + # print(self.table_id) self.camera_look_at = [0, 0, self.ref_h] self.rng = rng @@ -534,26 +569,6 @@ def shift_scene(self, shift: np.ndarray): obj.position = np.array(obj.position) + shift self.scene.camera.position = np.array(self.scene.camera.position) + shift - # def add_static_object(self, n_obj:int = 1): - - # logging.info("Randomly placing %d static objects:", n_obj) - # for i in range(n_obj): - # obj = self.gso.create(asset_id=self.rng.choice(self.object_asset_id_list)) - # assert isinstance(obj, kb.FileBasedObject) - # scale = 2 - # obj.scale = scale - # obj.metadata["scale"] = scale - # self.scene += obj - # kb.move_until_no_overlap(obj, self.simulator, spawn_region=STATIC_SPAWN_REGION, - # rng=self.rng) - # obj.friction = 1.0 - # obj.restitution = 0.0 - # obj.metadata["is_dynamic"] = False - # logging.info(" Added %s at %s", obj.asset_id, obj.position) - - # logging.info("Running 100 frames of simulation to let static objects settle ...") - # _, _ = self.simulator.run(frame_start=-100, frame_end=0) - def add_object(self, asset_id=None, position=None, @@ -605,13 +620,12 @@ def add_object(self, kb.move_until_no_overlap(obj, self.simulator, spawn_region=STATIC_SPAWN_REGION, rng=self.rng) - # temporarily set false - # to mitigate error "'XXXTestScene' object has no attribute 'gravity'" if is_dynamic: # reduce the restitution of the object to make it less bouncy # account for the gravity restituion_scale = -self.gravity[2] / 9.8 obj.restitution *= restituion_scale + obj.friction = 1.0 else: # make the object static obj.friction = 1.0 @@ -668,6 +682,15 @@ def render(self, save_to_file=False, **kwargs): return data_stack def write_metadata(self): + + # for gso, object name can be inferred from asset_id + test_obj_names = [obj.asset_id for obj in self.test_obj] + dynamic_obj_names = [obj.asset_id for obj in self.dynamic_objs] + static_obj_names = [obj.asset_id for obj in self.static_objs] + + block_obj_name = self.block_obj.asset_id if self.block_obj is not None else None + table_asset_id = self.table_id if self.table_id is not None else None + logging.info("Collecting and storing metadata for each object.") # return kb.write_json(filename=self.output_dir / "metadata.json", data={ @@ -675,13 +698,12 @@ def write_metadata(self): "metadata": kb.get_scene_metadata(self.scene), "camera": kb.get_camera_info(self.scene.camera), "instances": kb.get_instance_info(self.scene), - "has_table": self.add_table, - "table_id": self.table_id, - "test_obj_id": self.test_obj_id, - "block_id": self.block_id, - "fps": self.flags.frame_rate, - "volation_frame_start_at": self.frame_violation_start, - "test_case": self.violation_type + "test_obj": test_obj_names, + "dynamic_objs": dynamic_obj_names, + "static_objs": static_obj_names, + "block_obj": block_obj_name, + "table": table_asset_id, + }) def change_output_dir(self, new_output_dir): @@ -701,7 +723,10 @@ def add_background_static_objects(self, n_obj:int = 1): """ for _ in range(n_obj): - self.add_object(is_dynamic=False) + # self.add_object(is_dynamic=False) + obj = self.add_object() + self.static_objs.append(obj) + logging.info("Running 100 frames of simulation to let static objects settle ...") _, _ = self.simulator.run(frame_start=-100, frame_end=0) diff --git a/fy/collision.py b/fy/collision.py index 32302aca4..81f1665fc 100644 --- a/fy/collision.py +++ b/fy/collision.py @@ -31,13 +31,14 @@ def __init__(self, FLAGS) -> None: # look at a fixed height # self.scene.camera.position = (0, -5, 1.7) - self.default_camera_pos = spherical_to_cartesian(r_range=[2.5, 3], theta_range=[89, 91], phi_range=[-5, 5]) # (0, -1, 1.7) + # self.default_camera_pos = spherical_to_cartesian(r_range=[2.5, 3], theta_range=[89, 91], phi_range=[-5, 5]) # (0, -1, 1.7) self.camera_look_at = [0, 0, self.collision_height] - self.default_camera_pos[2] += self.collision_height + # self.default_camera_pos[2] += self.collision_height + self.default_camera_pos = [0, -4, 1.7] self.is_move_camera = False self.is_add_block_objects = False - self.add_table = False + self.is_add_table = True def prepare_scene(self): diff --git a/fy/collision_free_fall.py b/fy/collision_free_fall.py new file mode 100644 index 000000000..33ffdf5af --- /dev/null +++ b/fy/collision_free_fall.py @@ -0,0 +1,164 @@ + + +from fy.base import BaseTestScene +import numpy as np +import random +import logging +import abc +from utils import spherical_to_cartesian, getVisibleVertexFraction +import bpy + +class CollisionScene(BaseTestScene): + """Test scene for collision violation. + Start: OBJ1 and OBJ2 collide with each other in the air while falling down. OBJ1's velocity is vertical towards ground, + and OBJ2's velocity is towards OBJ1. + Normal result: the trajectory of OBJ1 and OBJ2 should follow the laws of physics. + Violation: the horizontal velocity of both OBJ disappears, and they fall straight down. + + Args: + BaseTestScene (_type_): _description_ + """ + + def __init__(self, FLAGS) -> None: + + super().__init__(FLAGS) + # collision parameters + self.first_collision_frame = 0 + self.violation_time = 1.0 # second before collision + self.collision_xy_distance = 1.8 # distance between obj_1 and obj_2 in xy plane + self.collision_z_distance = 0.1 # distance between obj_1 and obj_2 in z direction + self.collision_height = 1.2 + self.gravity = [0, 0, -2.8] + + # self.camera_look_at = [0, 0, self.collision_height] + # if self.flags.scene_type == "indoor": + # self.default_camera_pos = spherical_to_cartesian(r_range=[2.5, 3], theta_range=[89, 91], phi_range=[-5, 5]) # (0, -1, 1.7) + # self.default_camera_pos[2] += self.collision_height + # else: + # self.default_camera_pos = [0, -4, 1.7] + + self.is_move_camera = False + self.is_add_block_objects = False + + + def prepare_scene(self): + + # randomly choose to add a table or not + self.is_add_table = random.choice([True, False]) + # if self.flags.scene_type == "indoor": + # self.is_add_table = True + super().prepare_scene() + + def generate_keyframes(self): + """Generate keyframes for the objects, for both violation and non-violation states + """ + + # following the laws of physics + _, collisions = self._run_simulate() + + self.save_non_violation_scene() + + if self.flags.save_states: + fname = "non_violation.blend" + full_path = self.output_dir / fname + logging.info("Saving the renderer state to '%s' ", + full_path) + self.renderer.save_state(full_path) + + if self.flags.generate_violation: + logging.info("Generating scene with violation of laws of physics") + + # find the first collision frame + first_collision_frame = 0 + for i in range(len(collisions)): + instances = collisions[i]['instances'] + if len(instances) == 2: + if instances[0] in self.test_obj and instances[1] in self.test_obj: + first_collision_frame = collisions[i]['frame'] + break + if first_collision_frame == 0: + raise RuntimeError("No collision detected") + + self.first_collision_frame = first_collision_frame + logging.debug(f"first_collision_frame: {first_collision_frame}") + + # make the objects fall straight down after the collision + for obj in self.test_obj: + # linear interpolation + xyz = obj.get_value_at("position", first_collision_frame, interpolation="linear").copy() + + for frame in range(int(first_collision_frame), self.scene.frame_end+1): + # set xy velocity to 0 + vel = obj.keyframes["velocity"][frame].copy() + vel[0] = 0 + vel[1] = 0 + obj.velocity = vel + obj.keyframe_insert("velocity", frame) + + # set xy position to the same as the collision frame + pos = obj.keyframes["position"][frame].copy() + pos[0] = xyz[0] + pos[1] = xyz[1] + obj.position = pos + obj.keyframe_insert("position", frame) + + self.save_violation_scene() + + if self.flags.save_states: + fname = "violation.blend" + full_path = self.output_dir / fname + logging.info("Saving the renderer state to '%s' ", + full_path) + self.renderer.save_state(full_path) + + def add_test_objects(self): + """Add two colliding objects + + Returns: + _type_: _description_ + """ + g = -self.gravity[2] + t = self.violation_time # second + z_dis_before_collision = g * t ** 2 / 2 + obj_dist = self.collision_xy_distance # distance between obj_1 and obj_2 in xy plane + pos_2_delta_z = self.collision_z_distance # distance between obj_1 and obj_2 in z direction + + pos_1 = self.rng.normal(0, 0.2, 3) + pos_1[2] += self.collision_height + z_dis_before_collision + vel_1_xyz = [0, 0, 0] + logging.debug(f"pos_1: {pos_1}, vel_1_xyz: {vel_1_xyz}") + + collision_z = pos_1[2] - z_dis_before_collision + collision_xyz = [pos_1[0], pos_1[1], collision_z] + logging.debug(f"s before collision: {z_dis_before_collision}, t: {t}, collision_xyz: {collision_xyz}") + + pos_2 = self.rng.normal(0, 0.1, 3) + theta = self.rng.uniform(-np.pi/8, np.pi/8) + theta = self.rng.choice([0,1]) * np.pi + theta + pos_2[0] += obj_dist * np.cos(theta) + pos_2[1] += obj_dist * np.sin(theta) + pos_2[2] += pos_1[2] + pos_2_delta_z + logging.debug(f"pos_2: {pos_2}") + vel_2_xy = np.array(pos_1[:2]) - np.array(pos_2[:2]) / t + vel_2_z= -(pos_2_delta_z+0.02) / t + vel_2_xyz = [vel_2_xy[0], vel_2_xy[1], vel_2_z] + + # random select two objects, consider the exclusion list + obj_1_id = self.rng.choice(self.small_object_asset_id_list) + obj_1 = self.add_object(asset_id=obj_1_id, + position=pos_1, + velocity=vel_1_xyz, + scale=1.8, + name="small_obj") # 1.8 + + obj_2_id = self.rng.choice(self.big_object_asset_id_list) + obj_2 = self.add_object(asset_id=obj_2_id, + position=pos_2, + velocity=vel_2_xyz, + scale=2.8, + name="big_obj") # 2.8 + + self.test_obj = [obj_1, obj_2] + + return obj_1, obj_2 + \ No newline at end of file diff --git a/fy/run.py b/fy/run.py index d8df391a6..7087939ba 100644 --- a/fy/run.py +++ b/fy/run.py @@ -12,7 +12,8 @@ from fy.continuity import ContinuityTestScene from fy.support import SupportTestScene import os - +import time +from fy.collision_free_fall import CollisionScene def main() -> None: FLAGS = get_args() @@ -27,14 +28,15 @@ def main() -> None: ] ) - num_per_cls = 100 - max_trails = 150 + num_per_cls = 1000 + max_trails = 5000 test_cls_all = { # "solidity": SolidityTestScene, # "collision": CollisionTestScene, + "collision_free_fall": CollisionScene, # "Permanance": PermananceTestScene , - "Continuity": ContinuityTestScene, - "Support": SupportTestScene, + # "Continuity": ContinuityTestScene, + # "Support": SupportTestScene, } for test_name, test_cls in test_cls_all.items(): @@ -74,11 +76,12 @@ def generate_test_scene(test_class, FLAGS,output_dir) -> None: # render the violation state test_scene.change_output_dir( output_dir + "violation" ) - # igore rendering if debug is on - if FLAGS.render_video: + if FLAGS.render_violate_video: logging.info("Rendering the violation video") + start_time = time.time() test_scene.render(save_to_file=True) - write_video(output_dir + "violation/", output_dir + "violation.mp4") + # write_video(output_dir + "violation/", output_dir + "violation.mp4") + logging.info(f"Rendering the violation video took {time.time() - start_time} seconds") # load the non-violation state and render it logging.info("Loading the non-violation state") @@ -86,10 +89,12 @@ def generate_test_scene(test_class, FLAGS,output_dir) -> None: test_scene.change_output_dir( output_dir + "non_violation" ) # igore rendering if debug is on - if FLAGS.render_video: + if FLAGS.render_non_violate_video: logging.info("Rendering the non-violation video") + start_time = time.time() test_scene.render(save_to_file=True) - write_video(output_dir + "non_violation/", output_dir + "non_violation.mp4") + # write_video(output_dir + "non_violation/", output_dir + "non_violation.mp4") + logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") if __name__ == "__main__": main() diff --git a/fy/solidity.py b/fy/solidity.py index fc7d6a5d5..8bd31677a 100644 --- a/fy/solidity.py +++ b/fy/solidity.py @@ -36,7 +36,7 @@ def __init__(self, FLAGS) -> None: self.is_move_camera = False self.is_add_block_objects = False - self.add_table = False + self.is_add_table = False def prepare_scene(self): diff --git a/fy/utils.py b/fy/utils.py index 681a3960a..ed14d77f8 100644 --- a/fy/utils.py +++ b/fy/utils.py @@ -38,24 +38,24 @@ def get_args(): parser.set_defaults(save_state=False, frame_end=36, frame_rate=12, resolution="512x512") - # parser.add_argument("--debug", type=txt2bool, default=False, action="store_true") - parser.add_argument("--debug", type=txt2bool, default=False) + parser.add_argument("--debug", default=False, action="store_true") - parser.add_argument("--generate_violation", type=txt2bool, default=True) # generate violation results + parser.add_argument("--generate_violation", type=bool, default=False) # generate violation results # parser.add_argument("--render_both_results", type=txt2bool, default=True) # render both violation and non-violation results - parser.add_argument("--move_camera", type=txt2bool, default=True) # move camera - parser.add_argument("--use_indoor_only", type=txt2bool, default=True) + parser.add_argument("--move_camera", type=bool, default=True) # move camera + parser.add_argument("--scene_type", type=str, default="indoor") # scene type indoor | hdri | both - parser.add_argument("--save_states", type=txt2bool, default=False) # save blender states - parser.add_argument("--render_video", type=txt2bool, default=False) # render videos + parser.add_argument("--save_states", action="store_true", default=False) # save blender states + parser.add_argument("--render_non_violate_video", action="store_true", default=False) # render videos + parser.add_argument("--render_violate_video", action="store_true", default=False) # render videos FLAGS = parser.parse_args() if FLAGS.debug: FLAGS.logging_level = logging.DEBUG FLAGS.save_states = True - FLAGS.render_video = True + # FLAGS.render_video = True print("Debug mode is on") else: From 7bf4d29b42d761da46907aeeeeca9b676f665c72 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Wed, 24 Apr 2024 16:22:28 +0800 Subject: [PATCH 04/18] collision --- fy/collision.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fy/collision.py b/fy/collision.py index 81f1665fc..c68da4978 100644 --- a/fy/collision.py +++ b/fy/collision.py @@ -80,7 +80,6 @@ def generate_keyframes(self): raise RuntimeError("No collision detected") self.first_collision_frame = first_collision_frame - self.frame_violation_start = self.first_collision_frame logging.debug(f"first_collision_frame: {first_collision_frame}") # make the objects fall straight down after the collision @@ -160,8 +159,7 @@ def add_test_objects(self): name="big_obj") # 2.8 self.test_obj = [obj_1, obj_2] - self.test_obj_id = [obj_1_id, obj_2_id] - + return obj_1, obj_2 def _check_scene(self): From 2cc1f709b0dadf2fd8d6e750c0eff4bf9b9d4ae2 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Wed, 24 Apr 2024 16:23:03 +0800 Subject: [PATCH 05/18] solidity --- fy/solidity.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fy/solidity.py b/fy/solidity.py index 8bd31677a..7e7328ff3 100644 --- a/fy/solidity.py +++ b/fy/solidity.py @@ -28,7 +28,6 @@ def __init__(self, FLAGS) -> None: self.collision_z_distance = 0.1 # distance between obj_1 and obj_2 in z direction self.collision_height = 1.2 - self.frame_violation_start = self.violation_time * self.flags.frame_rate self.default_camera_pos = spherical_to_cartesian(r_range=[2.5, 3], theta_range=[89, 91], phi_range=[-5, 5]) # (0, -1, 1.7) self.camera_look_at = [0, 0, self.collision_height] @@ -144,7 +143,6 @@ def add_test_objects(self): name="big_obj") self.test_obj = [obj_1, obj_2] - self.test_obj_id = [obj_1_id, obj_2_id] return obj_1, obj_2 From 39efe42106d17ba6a969c1f40d4c2e11e689ccae Mon Sep 17 00:00:00 2001 From: maple5717 Date: Wed, 24 Apr 2024 16:25:04 +0800 Subject: [PATCH 06/18] support --- fy/support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fy/support.py b/fy/support.py index dc2da0b52..7cd9606f1 100644 --- a/fy/support.py +++ b/fy/support.py @@ -27,7 +27,7 @@ def __init__(self, FLAGS) -> None: self.frame_violation_start = 10 self.initial_dist_to_table = 0 # 1: float in air; 0: pass through table - self.violation_type = 1#np.random.binomial(n=1,p=0.5) + self.violation_type = np.random.binomial(n=1,p=0.5) self.gravity = [0, 0, -1.5] self.is_move_camera = False From e718c2e4da429f3e2876385cbfc1e54028c7827f Mon Sep 17 00:00:00 2001 From: maple5717 Date: Wed, 24 Apr 2024 20:20:09 +0800 Subject: [PATCH 07/18] continuity scene rendering --- fy/base.py | 11 +-- fy/continuity.py | 26 ++++--- fy/run_tiancheng.py | 139 +++++++++++++++++-------------------- fy/run_tiancheng_old.py | 148 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 231 insertions(+), 93 deletions(-) create mode 100644 fy/run_tiancheng_old.py diff --git a/fy/base.py b/fy/base.py index 3dd5ae331..38dcb0846 100644 --- a/fy/base.py +++ b/fy/base.py @@ -190,7 +190,7 @@ def _setup_everything(self): self.add_block_objects() self.add_test_objects() - + self.renderer.save_state(f"temp_scene/can_1.blend") # self._run_simulate() self._random_rotate_scene() @@ -202,7 +202,7 @@ def _setup_everything(self): else: self.scene.camera.position = self.default_camera_pos self.scene.camera.look_at(self.camera_look_at) - + # self.renderer.save_state(f"temp_scene/can_2.blend") if not use_indoor: # the hdri center texture is wired, shift to avoid self.shift_scene([0,5,0]) @@ -210,7 +210,7 @@ def _setup_everything(self): if self.render_speedup: self._set_fast_rendering() - + # self.renderer.save_state(f"temp_scene/can_3.blend") # def _check_scene_visible(self): # frame_end = self.flags.frame_end @@ -344,13 +344,16 @@ def prepare_scene(self): while True: self.dynamic_objs = [] self.static_objs = [] + self.test_obj = [] self._setup_everything() + # self.renderer.save_state(f"temp_scene/can_4.blend") if self._check_scene(): self.generate_keyframes() + # self.renderer.save_state(f"temp_scene/can_5.blend") return logging.warning("Current scene is invalid. Regenerating ") - # self.renderer.save_state(f"temp_scene/invalid_{self.i}.blend") + self.renderer.save_state(f"temp_scene/invalid_{self.i}.blend") self.i += 1 if self.is_move_camera: diff --git a/fy/continuity.py b/fy/continuity.py index 0f2d2e234..43222cfc9 100755 --- a/fy/continuity.py +++ b/fy/continuity.py @@ -32,14 +32,17 @@ def __init__(self, FLAGS) -> None: # 1. object disappears # 0. object teleports self.violation_type = np.random.binomial(n=1,p=0.5) + self.is_move_camera = np.random.binomial(n=1,p=0.5) self.gravity = (0, 0, -4.9) if not(self.violation_type): self.default_camera_pos = spherical_to_cartesian(r_range=[2, 3], theta_range=[75, 85]) self.camera_look_at = (0, 0, self.ref_h) - self.flags.move_camera = False + # self.flags.is_move_camera = False else: - self.flags.move_camera = True + # self.flags.is_move_camera = True + pass + def prepare_scene(self): print("preparing scene ...") super().prepare_scene() @@ -148,13 +151,6 @@ def add_test_objects(self): small_obj.velocity = (vx, 0, 0) small_obj.friction = 0.1 - - # align the can object - - # small_obj.position = (-0.8, 0.15, self.ref_h+0.2) - # for _ in range(10): - # print(small_obj.position, self.ref_h, small_obj.aabbox[0][2], self.ref_h - small_obj.aabbox[0][2]) - # print(small_obj.aabbox) self.test_obj = [small_obj] self._run_simulate() self.save_non_violation_scene() @@ -170,7 +166,7 @@ def _check_scene(self): Args: (bool): """ - + less_strict = not(self.flags.render_violate_video) # TODO: farthest point sampling, try reduce num of samples frame_end = self.flags.frame_end @@ -206,18 +202,20 @@ def _check_scene(self): # and visibility_obj[-6] >= 0.15 cond_2 = is_obj_visible[:6].max() and is_obj_visible[-7:].max() cond = [cond_2, - cond_1, + cond_1 or less_strict, in_view[0], in_view[5], - in_view[-12:].max(), + in_view[-12:].max() or less_strict, is_table_visible[6:-6].min(), - obj_on_table.sum() >= 16] + obj_on_table.sum() >= 16 or less_strict + ] else: cond = [visibility_obj[0] >= 0.5, in_view[0], in_view[-5], is_table_visible[6:-6].min(), - obj_on_table.sum() >= 20] + obj_on_table.sum() >= 20 or less_strict + ] is_valid = np.min(cond) if is_valid: diff --git a/fy/run_tiancheng.py b/fy/run_tiancheng.py index 721d281b3..2fd779036 100644 --- a/fy/run_tiancheng.py +++ b/fy/run_tiancheng.py @@ -1,45 +1,24 @@ -from fy.utils import get_args import logging +import colorlog +from fy.utils import get_args import kubric as kb -import imageio from fy.utils import write_video + from fy.solidity import SolidityTestScene from fy.collision import CollisionTestScene from fy.permanance import PermananceTestScene from fy.continuity import ContinuityTestScene from fy.support import SupportTestScene -from tqdm import tqdm -import colorlog import os - -from etils import epath -from random import choice -print(flush=True) -## TODO: set frame num from arg -frame_mid = 18 -frame_end = 36 -output_dirname = "render_output" -path_template = [ - {"euler_xyz": [0,0,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, - {"euler_xyz": [-25,0,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, # ! - {"euler_xyz": [0,-20,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, - {"euler_xyz": [0,-40,0], "key_frame_val": [-15, 20], "key_frame_num": [0, frame_end]}, - {"euler_xyz": [0,-60,0], "key_frame_val": [-10, 15], "key_frame_num": [0, frame_end]}, # ! - {"euler_xyz": [0,20,0], "key_frame_val": [25, -20], "key_frame_num": [0, frame_end]}, - {"euler_xyz": [0,40,0], "key_frame_val": [15, -20], "key_frame_num": [0, frame_end]}, # ! - {"euler_xyz": [0,60,0], "key_frame_val": [10, -15], "key_frame_num": [0, frame_end]}, # ! - {"euler_xyz": [0,0,0], "key_frame_val": [-20, 5, -20], "key_frame_num": [0, frame_mid, frame_end]}, - {"euler_xyz": [0,0,0], "key_frame_val": [20, -5, 20], "key_frame_num": [0, frame_mid, frame_end]}, - {"euler_xyz": [0,-90,0], "key_frame_val": [20, 5, 20], "key_frame_num": [0, frame_mid, frame_end]}, # ? - # {"euler_xyz": [0,0,0], "key_frame_val": [-10, 20, -10], "key_frame_num": [0, frame_mid, frame_end]}, - # {"euler_xyz": [0,0,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, -] +import time +from fy.collision_free_fall import CollisionScene def main() -> None: FLAGS = get_args() + logging.basicConfig(level=FLAGS.logging_level) output_dirname = "render_output" if not(FLAGS.save_states) else "output_temp" @@ -49,7 +28,7 @@ def main() -> None: level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ - logging.FileHandler("output/log"), + logging.FileHandler(f"{output_dirname}/log"), # logging.StreamHandler() ] ) @@ -70,78 +49,88 @@ def main() -> None: logger = logging.getLogger() logger.addHandler(handler) - - num_per_cls = 50 - max_trails = 20 if not(FLAGS.debug) else 10 + num_per_cls = 1000 + max_trails = 5000 test_cls_all = { # "solidity": SolidityTestScene, - # "continuity": ContinuityTestScene, - # "Support": SupportTestScene, - "collision": CollisionTestScene - # "Permanance": PermananceTestScene + # "collision": CollisionTestScene, + # "collision_free_fall": CollisionScene, + # "Permanance": PermananceTestScene , + "Continuity": ContinuityTestScene, + # "Support": SupportTestScene, + } for test_name, test_cls in test_cls_all.items(): - n = 0; i = 0 - for i in tqdm(range(max_trails)): - output_dir = f"{output_dirname}/{test_name}/scene_{i}/" - if os.path.isdir(output_dir): - if any([True for _ in os.listdir(output_dir)]): - logging.warning(f"Directory {output_dir} already exists, skip rendering the current scene") - continue - - print(f"========== Rendering {test_name} test {i} ===========") - video_dir = f"{output_dirname}/{test_name}/videos/" - FLAGS.job_dir = output_dir - # FLAGS.camera_path_config = choice(path_template) - - if test_name in ["Support", "continuity"]: - FLAGS.move_camera = False + # check if exist rendered test in the output folder + # if exist, start from the last one + scene_output_dir = f"{output_dirname}/{test_name}/" + # if os.path.exists(scene_output_dir): + # n = len(os.listdir(scene_output_dir)) + # logging.info(f"Found {n} rendered test in the output folder. Start from the next one.") + # else: + # n = 0 + n = 0 + + for i in range(max_trails): + + logging.info(f"========== Rendering {test_name} test {n} ===========") + output_dir = f"{output_dirname}/{test_name}/scene_{n}/" while True: - try: - generate_test_scene(test_cls, FLAGS, output_dir, video_dir, i) - i += 1 + if os.path.isdir(output_dir): + # if any([True for _ in os.listdir(output_dir)]): + if os.path.isfile(os.path.join(output_dir, "metadata.json")): + logging.warning(f"Directory {output_dir} already exists, skip rendering the current scene") + n += 1 + output_dir = f"{output_dirname}/{test_name}/scene_{n}/" + continue + break + + FLAGS.job_dir = output_dir + try: + generate_test_scene(test_cls, FLAGS, output_dir) + n += 1 + if n >= num_per_cls: break - except Exception as e: - logging.error(f"Error rendering collision test {n}: {e}\n Skipping to the next one.") - # if debug is on, raise the exception - # if FLAGS.debug: - # raise - continue - # if n >= num_per_cls: - # break - + except Exception as e: + logging.error(f"Error rendering {test_name} test {n}: {e}\n Skipping to the next one.") + # if debug is on, raise the exception + if FLAGS.debug: + raise + continue +def generate_test_scene(test_class, FLAGS,output_dir) -> None: -def generate_test_scene(test_class, FLAGS, output_dir, video_dir, i) -> None: - if not(os.path.exists(video_dir)): - os.makedirs(video_dir) with test_class(FLAGS) as test_scene: # first prepare the scene - print("Preparing the scene") + logging.info("Preparing the scene") test_scene.prepare_scene() test_scene.write_metadata() # render the violation state test_scene.change_output_dir( output_dir + "violation" ) - print("Rendering the violation state") - # igore rendering if debug is on - if True and not FLAGS.debug: + if FLAGS.render_violate_video: + logging.info("Rendering the violation video") + start_time = time.time() test_scene.render(save_to_file=True) - write_video(output_dir + "violation/", video_dir + f"violation_{i}.mp4") + # write_video(output_dir + "violation/", output_dir + "violation.mp4") + logging.info(f"Rendering the violation video took {time.time() - start_time} seconds") # load the non-violation state and render it - print("Loading the non-violation state") + logging.info("Loading the non-violation state") test_scene.load_non_violation_scene() test_scene.change_output_dir( output_dir + "non_violation" ) - print("Rendering the non-violation state") # igore rendering if debug is on - if True and not FLAGS.debug: + if FLAGS.render_non_violate_video: + logging.info("Rendering the non-violation video") + start_time = time.time() test_scene.render(save_to_file=True) - write_video(output_dir + "non_violation/", video_dir + f"non_violation_{i}.mp4") - + write_video(output_dir + "non_violation/", output_dir + f"non_violation.mp4") + logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") + if __name__ == "__main__": main() + \ No newline at end of file diff --git a/fy/run_tiancheng_old.py b/fy/run_tiancheng_old.py new file mode 100644 index 000000000..208e4ad7e --- /dev/null +++ b/fy/run_tiancheng_old.py @@ -0,0 +1,148 @@ + + + +from fy.utils import get_args +import logging +import kubric as kb +import imageio +from fy.utils import write_video +from fy.solidity import SolidityTestScene +from fy.collision import CollisionTestScene +from fy.permanance import PermananceTestScene +from fy.continuity import ContinuityTestScene +from fy.support import SupportTestScene +from tqdm import tqdm +import colorlog +import os + +from etils import epath +from random import choice +print(flush=True) +## TODO: set frame num from arg +frame_mid = 18 +frame_end = 36 +output_dirname = "render_output" +path_template = [ + {"euler_xyz": [0,0,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, + {"euler_xyz": [-25,0,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, # ! + {"euler_xyz": [0,-20,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, + {"euler_xyz": [0,-40,0], "key_frame_val": [-15, 20], "key_frame_num": [0, frame_end]}, + {"euler_xyz": [0,-60,0], "key_frame_val": [-10, 15], "key_frame_num": [0, frame_end]}, # ! + {"euler_xyz": [0,20,0], "key_frame_val": [25, -20], "key_frame_num": [0, frame_end]}, + {"euler_xyz": [0,40,0], "key_frame_val": [15, -20], "key_frame_num": [0, frame_end]}, # ! + {"euler_xyz": [0,60,0], "key_frame_val": [10, -15], "key_frame_num": [0, frame_end]}, # ! + {"euler_xyz": [0,0,0], "key_frame_val": [-20, 5, -20], "key_frame_num": [0, frame_mid, frame_end]}, + {"euler_xyz": [0,0,0], "key_frame_val": [20, -5, 20], "key_frame_num": [0, frame_mid, frame_end]}, + {"euler_xyz": [0,-90,0], "key_frame_val": [20, 5, 20], "key_frame_num": [0, frame_mid, frame_end]}, # ? + # {"euler_xyz": [0,0,0], "key_frame_val": [-10, 20, -10], "key_frame_num": [0, frame_mid, frame_end]}, + # {"euler_xyz": [0,0,0], "key_frame_val": [-20, 20], "key_frame_num": [0, frame_end]}, +] + +def main() -> None: + FLAGS = get_args() + logging.basicConfig(level=FLAGS.logging_level) + output_dirname = "render_output" if not(FLAGS.save_states) else "output_temp" + + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler("output/log"), + # logging.StreamHandler() + ] + ) + + + handler = colorlog.StreamHandler() + handler.setFormatter(colorlog.ColoredFormatter( + "%(log_color)s %(asctime)s %(levelname)s:%(message)s", + log_colors={ + 'DEBUG': 'white', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'bold_red', + } + )) + + logger = logging.getLogger() + logger.addHandler(handler) + + + num_per_cls = 50 + max_trails = 20 if not(FLAGS.debug) else 10 + test_cls_all = { + # "solidity": SolidityTestScene, + # "continuity": ContinuityTestScene, + # "Support": SupportTestScene, + "collision": CollisionTestScene + # "Permanance": PermananceTestScene + } + for test_name, test_cls in test_cls_all.items(): + n = 0; i = 0 + for i in tqdm(range(max_trails)): + output_dir = f"{output_dirname}/{test_name}/scene_{i}/" + if os.path.isdir(output_dir): + # if any([True for _ in os.listdir(output_dir)]): + if os.path.isfile(os.path.join(output_dir, "metadata.json")): + logging.warning(f"Directory {output_dir} already exists, skip rendering the current scene") + continue + + print(f"========== Rendering {test_name} test {i} ===========") + video_dir = f"{output_dirname}/{test_name}/videos/" + FLAGS.job_dir = output_dir + # FLAGS.camera_path_config = choice(path_template) + + if test_name in ["Support", "continuity"]: + FLAGS.move_camera = False + + while True: + try: + generate_test_scene(test_cls, FLAGS, output_dir, video_dir, i) + i += 1 + break + except Exception as e: + logging.error(f"Error rendering collision test {n}: {e}\n Skipping to the next one.") + # if debug is on, raise the exception + # if FLAGS.debug: + # raise + continue + # if n >= num_per_cls: + # break + + + +def generate_test_scene(test_class, FLAGS, output_dir, video_dir, i) -> None: + if not(os.path.exists(video_dir)): + os.makedirs(video_dir) + with test_class(FLAGS) as test_scene: + # first prepare the scene + print("Preparing the scene") + test_scene.prepare_scene() + test_scene.write_metadata() + + # render the violation state + test_scene.change_output_dir( output_dir + "violation" ) + + print("Rendering the violation state") + # igore rendering if debug is on + if True and not FLAGS.debug: + test_scene.render(save_to_file=True) + write_video(output_dir + "violation/", video_dir + f"violation_{i}.mp4") + + # load the non-violation state and render it + print("Loading the non-violation state") + test_scene.load_non_violation_scene() + test_scene.change_output_dir( output_dir + "non_violation" ) + print("Rendering the non-violation state") + + # igore rendering if debug is on + if True and not FLAGS.debug: + test_scene.render(save_to_file=True) + write_video(output_dir + "non_violation/", video_dir + f"non_violation_{i}.mp4") + +if __name__ == "__main__": + main() + \ No newline at end of file From 67eea5c1667897bf776e4007223e8d6feec23df4 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 25 Apr 2024 21:10:16 +0800 Subject: [PATCH 08/18] fixed object rotation in continuity scene --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index fb354706a..872ead188 100644 --- a/README.md +++ b/README.md @@ -33,4 +33,13 @@ python fy/run_tiancheng.py --save_states True --debug True ``` python fy/run_tiancheng.py --save_states False --debug False +``` + +``` +python fy/run_tiancheng.py --save_states --debug --render_non_violate_video --test_scene_cls 100 + +``` + +``` +python fy/run_tiancheng.py --render_non_violate_video --test_scene_cls 100 ``` \ No newline at end of file From 09144c8e20f1ef6a55a69ef7e55ec328ceb85f8e Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 25 Apr 2024 21:11:05 +0800 Subject: [PATCH 09/18] fixed object rotation problem in continuty scene --- fy/base.py | 5 +-- fy/continuity.py | 64 ++++++++++++++++++++++-------------- fy/run_tiancheng.py | 10 ++++-- fy/utils.py | 9 ++++- kubric/simulator/pybullet.py | 39 ++++++++++++++++++++++ 5 files changed, 96 insertions(+), 31 deletions(-) diff --git a/fy/base.py b/fy/base.py index 38dcb0846..c446d0d12 100644 --- a/fy/base.py +++ b/fy/base.py @@ -178,6 +178,8 @@ def _setup_everything(self): self.scene.gravity = self.gravity + self._random_rotate_scene() + if self.flags.debug: logging.info("Ignore background objects in debugging mode.") else: @@ -192,7 +194,6 @@ def _setup_everything(self): self.add_test_objects() self.renderer.save_state(f"temp_scene/can_1.blend") # self._run_simulate() - self._random_rotate_scene() if self.is_move_camera: traj_idx = random.randint(0, len(self.camera_path_config)-1) @@ -784,7 +785,7 @@ def add_block_objects(self): position=(0, 0, 0), quaternion=(1,0,0,0), is_dynamic=True, - scale=1.25, + scale=1.25/100, name=self.block_name ) diff --git a/fy/continuity.py b/fy/continuity.py index 43222cfc9..2f066ba4e 100755 --- a/fy/continuity.py +++ b/fy/continuity.py @@ -116,40 +116,43 @@ def add_test_objects(self): _type_: _description_ """ # -- add small object - print("adding the small object") + valid = False + while not valid: # only select valid small object + print("adding the small object") - if self.violation_type: - # relocate the block obj - self.block_obj.position = (0.15, -0.1, self.block_obj.position[2]) - else: - self.block_obj.position = (0, 0, -10) - # initialize the obj. Note the the initial position should be (0,0,0) - # so that the dist between its CoM and the table surface can be calculated - small_obj_id = [name for name, spec in shapenet_assets._assets.items() - if spec["metadata"]["category"] == "can"] - small_obj_id = self.rng.choice(small_obj_id) - small_obj = self.add_object(asset_id=small_obj_id, - position=(0, 0, 0), - velocity=(0, 0, 0), - quaternion=kb.Quaternion(axis=[0, 0, 1], degrees=0), - is_dynamic=True, - scale=0.15, - name="small_obj") - - self.test_obj_z_orn = align_can_objs(small_obj) - + if self.violation_type: + # relocate the block obj + self.block_obj.position = (0.15, -0.1, self.block_obj.position[2]) + else: + self.block_obj.position = (0, 0, -10) + # initialize the obj. Note the the initial position should be (0,0,0) + # so that the dist between its CoM and the table surface can be calculated + small_obj_id = [name for name, spec in shapenet_assets._assets.items() + if spec["metadata"]["category"] == "can"] + small_obj_id = self.rng.choice(small_obj_id) + small_obj = self.add_object(asset_id=small_obj_id, + position=(0, 0, 0), + velocity=(0, 0, 0), + quaternion=kb.Quaternion(axis=[0, 0, 1], degrees=0), + is_dynamic=True, + scale=0.15, + name="small_obj") + + self.test_obj_z_orn, radius, _, valid = align_can_objs(small_obj) + self.radius = radius + # small_obj.quaternion = kb.Quaternion(axis=[0, 0, 1], degrees=180) * small_obj.quaternion # set the position of the can to avoid potential collision # of the block object table_x_range = self.table.aabbox[0][0] block_y_range = self.block_obj.aabbox[1][1] - vx = self.rng.uniform(0.8, 1.0) # initial velocitry + vx = self.rng.uniform(0.5, 0.8) # initial velocitry px = self.rng.uniform(0, 0.05) + small_obj.aabbox[1][2] + table_x_range # initial position py = self.rng.uniform(0.1, 0.15) + block_y_range - pz = self.rng.uniform(0.0, 0.01) + self.ref_h - small_obj.aabbox[0][2] + pz = self.rng.uniform(0.05, 0.1) + self.ref_h + radius small_obj.position = (px, py, pz) small_obj.velocity = (vx, 0, 0) - small_obj.friction = 0.1 + small_obj.friction = 0 self.test_obj = [small_obj] self._run_simulate() @@ -166,6 +169,7 @@ def _check_scene(self): Args: (bool): """ + # self.frame_violation_start = 20; return True less_strict = not(self.flags.render_violate_video) # TODO: farthest point sampling, try reduce num of samples frame_end = self.flags.frame_end @@ -249,13 +253,23 @@ def _run_simulate(self, save_state=False, frame_start=0): # remove the test object's unexpected rotation obj_pos0 = obj.keyframes["position"][frame_start].copy() + for frame in range(frame_start, self.scene.frame_end+1): + pos = obj.keyframes["position"][frame].copy() + + rot_angle = (obj_pos0[0] - pos[0]) / self.radius + quaternion = kb.Quaternion(axis=[0, 1, 0], degrees=-rot_angle * 180 / np.pi) + print(self.radius, (obj_pos0[0] - pos[0]), rot_angle * 180 / np.pi) + obj.quaternion = quaternion + obj.keyframe_insert("quaternion", frame) + return ret for frame in range(frame_start, self.scene.frame_end+1): # set xy velocity to 0 q = obj.keyframes["quaternion"][frame].copy() q = np.array(q) q[1] = 0 q[3] = 0 - obj.quaternion = q + q[2] = -q[2] + obj.quaternion = q / np.linalg.norm(q) obj.keyframe_insert("quaternion", frame) pos = obj.keyframes["position"][frame].copy() diff --git a/fy/run_tiancheng.py b/fy/run_tiancheng.py index 2fd779036..b8bb7241e 100644 --- a/fy/run_tiancheng.py +++ b/fy/run_tiancheng.py @@ -88,7 +88,7 @@ def main() -> None: FLAGS.job_dir = output_dir try: - generate_test_scene(test_cls, FLAGS, output_dir) + generate_test_scene(test_cls, FLAGS, output_dir, n) n += 1 if n >= num_per_cls: break @@ -99,7 +99,11 @@ def main() -> None: raise continue -def generate_test_scene(test_class, FLAGS,output_dir) -> None: +def generate_test_scene(test_class, FLAGS,output_dir, i) -> None: + video_dir = os.path.join(output_dir.rsplit('/', 2)[0], "videos/") + if not(os.path.exists(video_dir)): + os.makedirs(video_dir) + # print(video_dir); return with test_class(FLAGS) as test_scene: # first prepare the scene @@ -127,7 +131,7 @@ def generate_test_scene(test_class, FLAGS,output_dir) -> None: logging.info("Rendering the non-violation video") start_time = time.time() test_scene.render(save_to_file=True) - write_video(output_dir + "non_violation/", output_dir + f"non_violation.mp4") + write_video(output_dir + "non_violation/", video_dir + f"non_violation_{i}.mp4") logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") if __name__ == "__main__": diff --git a/fy/utils.py b/fy/utils.py index e454e5736..a6710483e 100644 --- a/fy/utils.py +++ b/fy/utils.py @@ -389,6 +389,9 @@ def align_can_objs(obj): # otherwise set the longest axis as the principal axis axis = np.argmax(np.array([x_size, y_size, z_size])) + radius = [y_size, z_size, x_size][axis] / 2 + height = [x_size, y_size, z_size][axis] + axis_rot_mapping = { 0: kb.Quaternion(axis=[0, 0, 1], degrees=-90), 1: kb.Quaternion(axis=[1, 0, 0], degrees=0), @@ -398,7 +401,11 @@ def align_can_objs(obj): quaternion_tf = axis_rot_mapping[axis] obj.quaternion = quaternion_tf * obj.quaternion - return axis + x_iccentric = obj.aabbox[1][0] + obj.aabbox[0][0] + y_iccentric = obj.aabbox[1][1] + obj.aabbox[0][1] + valid = np.abs(x_iccentric) <= 0.01 and np.abs(x_iccentric) <= 0.01 + + return axis, radius, height, valid diff --git a/kubric/simulator/pybullet.py b/kubric/simulator/pybullet.py index 0dc586280..e8f8a89d9 100644 --- a/kubric/simulator/pybullet.py +++ b/kubric/simulator/pybullet.py @@ -174,6 +174,45 @@ def _add_object(self, obj: core.FileBasedObject) -> Optional[int]: useFixedBase=obj.static, globalScaling=scale, useMaximalCoordinates=True) + # print(str(path), "02946921" in str(path)) + if "02946921" in str(path) and False: + import numpy as np + bbox = self._physics_client.getAABB(obj_idx) # returns [aabbMin, aabbMax] + bmin, bmax = np.array(bbox[0]), np.array(bbox[1]) + x_size, y_size, z_size = bmax - bmin + axis_compare = np.array([np.isclose(y_size, z_size), + np.isclose(z_size, x_size), + np.isclose(y_size, x_size)]) + + if axis_compare.sum() > 0: + axis = np.argmax(np.array(axis_compare)) + param_dict = { + 0: (x_size, y_size/2, [0,1,0,0]), # (height, radius, quaternion) + 1: (y_size, z_size/2, [1,0,0,0]), + 2: (z_size, x_size/2, [0,0,0,1]), + } + + height, radius, q = param_dict[int(axis)] + + self._physics_client.removeBody(obj_idx) + + col_id = self._physics_client.createCollisionShape( + shapeType=self._physics_client.GEOM_CYLINDER, + radius=radius, + height=height, + collisionFramePosition=[0.0,0,0], + collisionFrameOrientation=q) + + obj_idx = self._physics_client.createMultiBody( + baseMass=0.1, + baseCollisionShapeIndex=col_id, + baseInertialFramePosition=[0, 0, 0], + basePosition=[0, 0, 0], + baseOrientation=[0,0,0,1]) + self._physics_client.changeDynamics(obj_idx, 0, + spinningFriction=1, + rollingFriction=1) + else: raise IOError( "Unsupported format '{path.suffix}' of file '{path}'") From 3e1dfe5ca4ac327b18c2499a1825e0b5b907bff2 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 25 Apr 2024 21:11:35 +0800 Subject: [PATCH 10/18] recovered pybullet file --- kubric/simulator/pybullet.py | 39 ------------------------------------ 1 file changed, 39 deletions(-) diff --git a/kubric/simulator/pybullet.py b/kubric/simulator/pybullet.py index e8f8a89d9..0dc586280 100644 --- a/kubric/simulator/pybullet.py +++ b/kubric/simulator/pybullet.py @@ -174,45 +174,6 @@ def _add_object(self, obj: core.FileBasedObject) -> Optional[int]: useFixedBase=obj.static, globalScaling=scale, useMaximalCoordinates=True) - # print(str(path), "02946921" in str(path)) - if "02946921" in str(path) and False: - import numpy as np - bbox = self._physics_client.getAABB(obj_idx) # returns [aabbMin, aabbMax] - bmin, bmax = np.array(bbox[0]), np.array(bbox[1]) - x_size, y_size, z_size = bmax - bmin - axis_compare = np.array([np.isclose(y_size, z_size), - np.isclose(z_size, x_size), - np.isclose(y_size, x_size)]) - - if axis_compare.sum() > 0: - axis = np.argmax(np.array(axis_compare)) - param_dict = { - 0: (x_size, y_size/2, [0,1,0,0]), # (height, radius, quaternion) - 1: (y_size, z_size/2, [1,0,0,0]), - 2: (z_size, x_size/2, [0,0,0,1]), - } - - height, radius, q = param_dict[int(axis)] - - self._physics_client.removeBody(obj_idx) - - col_id = self._physics_client.createCollisionShape( - shapeType=self._physics_client.GEOM_CYLINDER, - radius=radius, - height=height, - collisionFramePosition=[0.0,0,0], - collisionFrameOrientation=q) - - obj_idx = self._physics_client.createMultiBody( - baseMass=0.1, - baseCollisionShapeIndex=col_id, - baseInertialFramePosition=[0, 0, 0], - basePosition=[0, 0, 0], - baseOrientation=[0,0,0,1]) - self._physics_client.changeDynamics(obj_idx, 0, - spinningFriction=1, - rollingFriction=1) - else: raise IOError( "Unsupported format '{path.suffix}' of file '{path}'") From e38d10e48b317f34eb8dc85b29bb7125ca409c6a Mon Sep 17 00:00:00 2001 From: maple5717 Date: Sat, 27 Apr 2024 00:26:29 +0800 Subject: [PATCH 11/18] added back camera --- fy/base.py | 96 ++++++++++++++++++++++++++++++++++----- fy/continuity.py | 37 +++++++++------ fy/run_tiancheng.py | 51 ++++++++++++--------- fy/run_watch_tiancheng.py | 61 +++++++++++++++++++++++++ fy/utils.py | 14 +++--- 5 files changed, 205 insertions(+), 54 deletions(-) create mode 100644 fy/run_watch_tiancheng.py diff --git a/fy/base.py b/fy/base.py index 37958f6de..8149391cb 100644 --- a/fy/base.py +++ b/fy/base.py @@ -76,9 +76,15 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.table_scale = 2 self.shapenet_table_ids = None + self.camera_front = None + self.camera_back = None + self.active_camera = None + self.floor_name = "floor_kb" self.table_name = "table_kb" self.block_name = "block_kb" + self.camera_name = "camera" + self.back_camera_name = "camera_back" self.render_data = ("rgba",) self.background_hdri_id = FLAGS.background_hdri_id @@ -88,6 +94,7 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.test_obj_states = {"violation": None, "non_violation": None} self.test_obj = None self.default_camera_pos = [0, 0, 1] + self.back_camera_pos = [0, 0, 1] self.camera_look_at = [0,0,0] # load asset sources @@ -151,8 +158,8 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.is_move_camera = FLAGS.move_camera self.is_add_table = True self.table_id = None - self.is_add_background_static_objects = True - self.is_add_background_dynamic_objects = True + self.is_add_background_static_objects = False + self.is_add_background_dynamic_objects = False self.object_asset_id_list = self.gso.all_asset_ids @@ -195,14 +202,19 @@ def _setup_everything(self): # self._run_simulate() # self._random_rotate_scene() + # self._switch_to_back_camera() + # self.scene.camera.position = self.back_camera_pos + # self.scene.camera.look_at(self.camera_look_at) + + self._switch_to_front_camera() if self.is_move_camera: traj_idx = random.randint(0, len(self.camera_path_config)-1) self._set_camera_path(self.camera_path_config[traj_idx]) self.cur_camera_traj_idx = traj_idx self._set_camera_focus_point([0, 0, self.ref_h]) # auto set the height to be the table height if exists else: - self.scene.camera.position = self.default_camera_pos - self.scene.camera.look_at(self.camera_look_at) + self.camera_front.position = self.default_camera_pos + self.camera_front.look_at(self.camera_look_at) # self.renderer.save_state(f"temp_scene/can_2.blend") if not use_indoor: # the hdri center texture is wired, shift to avoid @@ -356,7 +368,7 @@ def prepare_scene(self): self._setup_everything() # self.renderer.save_state(f"temp_scene/can_4.blend") - if self._check_scene(): + if self._check_scene_with_front_back_cameras(): self.generate_keyframes() # self.renderer.save_state(f"temp_scene/can_5.blend") return @@ -368,6 +380,30 @@ def prepare_scene(self): self.camera_path_sample_stats[self.cur_camera_traj_idx] += 1 logging.info(f"Re-sampling... Current stats: {self.camera_path_sample_stats}") + def _check_scene_with_front_back_cameras(self): + """Check if the scene is valid for both the front and back camera. Return Flase if the scene is invalid. + TODO: implement (Override) this function + + Returns: + _type_: _description_ + """ + self._switch_to_front_camera() + if not self._check_scene(): + logging.warning("Check scene failed with Front Camera") + return False + + if not self.flags.add_back_camera: + return True + + self._switch_to_back_camera() + if self._check_scene(): + self._switch_to_front_camera() + return True + else: + logging.warning("Check scene failed with Back Camera") + return False + + def _check_scene(self): """Check if the scene is valid. Return Flase if the scene is invalid. TODO: implement (Override) this function @@ -477,13 +513,35 @@ def _setup_hdri_scene(self): self.scratch_dir = scratch_dir self.background_hdri = background_hdri - def _add_camera(self, scene): - scene.camera = kb.PerspectiveCamera(name="camera") + def _add_camera(self, scene, back_cam=False): + cam_name = self.camera_name if not back_cam else self.back_camera_name + + camera = kb.PerspectiveCamera(name=cam_name) + self.scene += camera + self.scene.camera = camera # avoid name clash - cam_name = "camera" set_name(cam_name) + if not back_cam: + self.camera_front = camera + else: + self.camera_back = camera + # self.scene.camera.position = self.back_camera_pos + # self.scene.camera.look_at(self.camera_look_at) + + def _switch_to_front_camera(self): + self._switch_camera(self.camera_name) + self.scene.camera = self.camera_front + + def _switch_to_back_camera(self): + self._switch_camera(self.back_camera_name) + self.scene.camera = self.camera_back + + def _switch_camera(self, camera_name): + bpy.context.scene.camera = bpy.data.objects[camera_name] + self.active_camera = camera_name + def _setup_indoor_scene(self, ): """Setup the indoor scene for rendering @@ -495,8 +553,8 @@ def _setup_indoor_scene(self, logging.info("Loading blender scene") blender_scene = rng.choice(self.scenes) self.load_blender_scene(blender_scene) - self._add_camera(self.scene) + # self.scene.camera.position = self.default_camera_pos # self.scene.camera.look_at(self.camera_look_at) @@ -513,8 +571,8 @@ def _setup_indoor_scene(self, bpy.data.objects[self.floor_name].hide_viewport = True - self.default_camera_pos = [0, -1.8, 1] - self.camera_look_at = [0, 0, 0.3] + # self.default_camera_pos = [0, -1.8, 1] + # self.camera_look_at = [0, 0, 0.3] if self.is_add_table: logging.info("Adding table to the scene") table_id = rng.choice(self.shapenet_table_ids) @@ -531,6 +589,7 @@ def _setup_indoor_scene(self, set_name(self.table_name) self.ref_h = table_h self.default_camera_pos[2] += table_h + self.back_camera_pos[2] += table_h self.table_id = table_id # print(self.table_id) self.camera_look_at = [0, 0, self.ref_h] @@ -539,6 +598,19 @@ def _setup_indoor_scene(self, self.output_dir = output_dir self.scratch_dir = scratch_dir + if self.flags.add_back_camera: + self._add_camera(self.scene, back_cam=True) + self._switch_to_back_camera() + self.camera_back.position = self.back_camera_pos + self.camera_back.look_at(self.camera_look_at) + self.renderer.save_state(f"temp_scene/bgcam0.blend") + + self._add_camera(self.scene) + self._switch_to_front_camera() + self.camera_front.position = self.default_camera_pos + self.camera_front.look_at(self.camera_look_at) + self.renderer.save_state(f"temp_scene/bgcam1.blend") + ################################ # add random directional lighting # the light is placed at some random position sampled from a sphere, with min height @@ -566,7 +638,7 @@ def _setup_indoor_scene(self, direc_light = bpy.data.objects["direc_light"] direc_light.data.shadow_soft_size = self.rng.uniform(*shadow_soft_size) - + def shift_scene(self, shift: np.ndarray): """Shift the scene by a given vector. diff --git a/fy/continuity.py b/fy/continuity.py index 2f066ba4e..6ea1afff0 100755 --- a/fy/continuity.py +++ b/fy/continuity.py @@ -14,7 +14,7 @@ # TODO: test if obj is on the table at most of time; why velocity changes after teletransportation -class ContinuityTestScene(PermananceTestScene): +class ContinuityTestScene(BaseTestScene): """Test scene for permanance violation. Start: Object is visible Normal result: ... @@ -25,7 +25,6 @@ class ContinuityTestScene(PermananceTestScene): """ def __init__(self, FLAGS) -> None: - super().__init__(FLAGS) self.frame_violation_start = -1 # two cases: @@ -35,13 +34,19 @@ def __init__(self, FLAGS) -> None: self.is_move_camera = np.random.binomial(n=1,p=0.5) self.gravity = (0, 0, -4.9) - if not(self.violation_type): - self.default_camera_pos = spherical_to_cartesian(r_range=[2, 3], theta_range=[75, 85]) - self.camera_look_at = (0, 0, self.ref_h) - # self.flags.is_move_camera = False - else: - # self.flags.is_move_camera = True - pass + self.back_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[70, 80], phi_range=[-45+180, 45+180]) + self.default_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[75, 85]) + self.camera_look_at = (0, 0, self.ref_h) + + # if not(self.violation_type): + # self.default_camera_pos = spherical_to_cartesian(r_range=[2, 3], theta_range=[75, 85]) + # self.camera_look_at = (0, 0, self.ref_h) + # # self.flags.is_move_camera = False + # else: + # # self.flags.is_move_camera = True + # pass + + def prepare_scene(self): print("preparing scene ...") @@ -51,7 +56,6 @@ def prepare_scene(self): # remove block object self.block_obj.position = (0, 0, -10) - def generate_keyframes(self): """Generate keyframes for the test objects, for both violation and non-violation states """ @@ -117,8 +121,11 @@ def add_test_objects(self): """ # -- add small object valid = False + small_obj = None while not valid: # only select valid small object print("adding the small object") + if small_obj is not None: + self.scene.remove(small_obj) if self.violation_type: # relocate the block obj @@ -146,7 +153,7 @@ def add_test_objects(self): # of the block object table_x_range = self.table.aabbox[0][0] block_y_range = self.block_obj.aabbox[1][1] - vx = self.rng.uniform(0.5, 0.8) # initial velocitry + vx = self.rng.uniform(0.35, 0.8) # initial velocitry px = self.rng.uniform(0, 0.05) + small_obj.aabbox[1][2] + table_x_range # initial position py = self.rng.uniform(0.1, 0.15) + block_y_range pz = self.rng.uniform(0.05, 0.1) + self.ref_h + radius @@ -185,8 +192,8 @@ def _check_scene(self): obj = self.test_obj[0] # work for only one test obj for i, frame in enumerate(tqdm(range(self.flags.frame_start, self.flags.frame_end))): bpy.context.scene.frame_set(frame) - vis_obj = getVisibleVertexFraction("small_obj", self.rng) - vis_table = isPointVisible([0, 0, self.ref_h], [self.table_name, self.block_name, "small_obj"]) + vis_obj = getVisibleVertexFraction("small_obj", self.rng, self.active_camera) + vis_table = isPointVisible([0, 0, self.ref_h], [self.table_name, self.block_name, "small_obj"], self.active_camera) visibility_obj[i] = vis_obj visibility_table[i] = vis_table obj_on_table[i] = obj.keyframes["position"][frame][2] > self.ref_h-0.05 @@ -216,7 +223,7 @@ def _check_scene(self): else: cond = [visibility_obj[0] >= 0.5, in_view[0], - in_view[-5], + in_view[-5] or less_strict, is_table_visible[6:-6].min(), obj_on_table.sum() >= 20 or less_strict ] @@ -258,7 +265,7 @@ def _run_simulate(self, save_state=False, frame_start=0): rot_angle = (obj_pos0[0] - pos[0]) / self.radius quaternion = kb.Quaternion(axis=[0, 1, 0], degrees=-rot_angle * 180 / np.pi) - print(self.radius, (obj_pos0[0] - pos[0]), rot_angle * 180 / np.pi) + # print(self.radius, (obj_pos0[0] - pos[0]), rot_angle * 180 / np.pi) obj.quaternion = quaternion obj.keyframe_insert("quaternion", frame) return ret diff --git a/fy/run_tiancheng.py b/fy/run_tiancheng.py index b8bb7241e..85a46492f 100644 --- a/fy/run_tiancheng.py +++ b/fy/run_tiancheng.py @@ -112,27 +112,36 @@ def generate_test_scene(test_class, FLAGS,output_dir, i) -> None: test_scene.write_metadata() # render the violation state - test_scene.change_output_dir( output_dir + "violation" ) - - if FLAGS.render_violate_video: - logging.info("Rendering the violation video") - start_time = time.time() - test_scene.render(save_to_file=True) - # write_video(output_dir + "violation/", output_dir + "violation.mp4") - logging.info(f"Rendering the violation video took {time.time() - start_time} seconds") - - # load the non-violation state and render it - logging.info("Loading the non-violation state") - test_scene.load_non_violation_scene() - test_scene.change_output_dir( output_dir + "non_violation" ) - - # igore rendering if debug is on - if FLAGS.render_non_violate_video: - logging.info("Rendering the non-violation video") - start_time = time.time() - test_scene.render(save_to_file=True) - write_video(output_dir + "non_violation/", video_dir + f"non_violation_{i}.mp4") - logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") + + + cameras = [test_scene.camera_name] + if FLAGS.add_back_camera: + cameras += [test_scene.back_camera_name] + camera_names = ["front", "back"] + for j, camera in enumerate(cameras): + test_scene._switch_camera(camera) + camera_name = camera_names[j] + + test_scene.change_output_dir( output_dir + f"violation_{camera_name}" ) + if FLAGS.render_violate_video: + logging.info("Rendering the violation video") + start_time = time.time() + test_scene.render(save_to_file=True) + write_video(output_dir + f"violation_{camera_name}/", output_dir + f"violation_{i}_{camera_name}.mp4") + logging.info(f"Rendering the violation video took {time.time() - start_time} seconds") + + # load the non-violation state and render it + logging.info("Loading the non-violation state") + test_scene.load_non_violation_scene() + test_scene.change_output_dir( output_dir + f"non_violation_{camera_name}" ) + + # igore rendering if debug is on + if FLAGS.render_non_violate_video: + logging.info("Rendering the non-violation video") + start_time = time.time() + test_scene.render(save_to_file=True) + write_video(output_dir + f"non_violation_{camera_name}/", video_dir + f"non_violation_{i}_{camera_name}.mp4") + logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") if __name__ == "__main__": main() diff --git a/fy/run_watch_tiancheng.py b/fy/run_watch_tiancheng.py new file mode 100644 index 000000000..4373dbab0 --- /dev/null +++ b/fy/run_watch_tiancheng.py @@ -0,0 +1,61 @@ +""" Run the script as subprocess, restart it if it is killed by the system. """ +import os +import subprocess +import argparse +import time +def check_if_job_finished(num_per_cls: int, test_scene_cls) -> bool: + # check if output/{test_scene} has {num_per_cls} folders + for test_scene in test_scene_cls: + scene_output_dir = f"output/{test_scene}/" + if os.path.exists(scene_output_dir): + n = len(os.listdir(scene_output_dir)) + if n >= num_per_cls: + print(f"Found {n} rendered test in the {scene_output_dir} folder. Job finished for {test_scene}.") + continue + print(f"Job not finished. Found {n} rendered test in the {scene_output_dir} folder.") + return False + return True + +def run_watch(): + parser = argparse.ArgumentParser() + parser.add_argument("--num_per_cls", type=int, required=False) + parser.add_argument("--test_scene_cls",nargs='+', required=False) # test scenes + # rest of the arguments as list + + args, other_args = parser.parse_known_args() + print(args) + print(other_args) + + while True: + # check if the job is finished + + # if check_if_job_finished(args.num_per_cls, args.test_scene_cls): + # print("Job finished.") + # break + + all_args = ["/bin/python", "fy/run_tiancheng.py", "--render_non_violate_video", "--test_scene_cls", "1000"] + other_args + print("=============================================================") + print("Restarting the job with the following arguments:") + print(" ".join(all_args)) + print("=============================================================") + proc = subprocess.Popen(all_args,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + + while True: + # first check if the job is killed + if proc.poll() is not None: + print("Job is killed. Restarting the job.") + break + # read all the output since the last read + output = proc.stdout.read(200) + + if output == b'' and proc.poll() is not None: + break + if output: + print(output.strip().decode()) + # print("Still runing...") + # wait for 1 second + time.sleep(1) + + +if __name__ == "__main__": + run_watch() \ No newline at end of file diff --git a/fy/utils.py b/fy/utils.py index a6710483e..61b310d73 100644 --- a/fy/utils.py +++ b/fy/utils.py @@ -43,6 +43,7 @@ def get_args(): parser.add_argument("--generate_violation", type=bool, default=False) # generate violation results # parser.add_argument("--render_both_results", type=txt2bool, default=True) # render both violation and non-violation results + parser.add_argument("--add_back_camera", type=bool, default=True) parser.add_argument("--move_camera", type=bool, default=True) # move camera parser.add_argument("--scene_type", type=str, default="indoor") # scene type indoor | hdri | both @@ -227,7 +228,7 @@ def set_name(name): obj_bpy.name = name break -def getVisibleVertexFraction(obj_name, rng, sample_num=1000): +def getVisibleVertexFraction(obj_name, rng, camera_name="camera", sample_num=1000): """ Calculates and returns the fraction of vertices of a given object that are visible from a given camera position. @@ -243,7 +244,7 @@ def getVisibleVertexFraction(obj_name, rng, sample_num=1000): Returns: - float: The fraction of vertices of the object that are visible from the camera position. """ - camera = bpy.data.objects['camera'] + camera = bpy.data.objects[camera_name] obj = bpy.data.objects[obj_name] camera_loc = [camera.matrix_world[0][3], camera.matrix_world[1][3], camera.matrix_world[2][3]] @@ -254,6 +255,7 @@ def getVisibleVertexFraction(obj_name, rng, sample_num=1000): num_vert_in_fov = 0 # number of vertices in the camera's field of view vert_list = list(obj.data.vertices) + sampled_verts = sample(vert_list, sample_num) if sample_num < len(vert_list) else vert_list sample_num = len(sampled_verts) for vert in sampled_verts: @@ -268,7 +270,7 @@ def getVisibleVertexFraction(obj_name, rng, sample_num=1000): return num_vert_in_fov / sample_num -def isPointVisible(pt, obj_names): +def isPointVisible(pt, obj_names, camera_name="camera"): """ Calculates and returns the fraction of vertices of a given object that are visible from a given camera position. @@ -284,7 +286,7 @@ def isPointVisible(pt, obj_names): Returns: - float: The fraction of vertices of the object that are visible from the camera position. """ - camera = bpy.data.objects['camera'] + camera = bpy.data.objects[camera_name] camera_loc = [camera.matrix_world[0][3], camera.matrix_world[1][3], camera.matrix_world[2][3]] dist = np.array(pt) - np.array(camera_loc) @@ -429,9 +431,9 @@ def align_can_objs(obj): def spherical_to_cartesian(r_range=[3, 4], theta_range=[60, 80], phi_range=[-30, 30]): r = np.random.uniform(r_range[0], r_range[1]) theta = np.random.uniform(theta_range[0], theta_range[1]) * np.pi/180 - phi = np.random.uniform(np.random.uniform(phi_range[0], phi_range[1])) * np.pi/180 - phi -= np.pi / 2 + phi = np.random.uniform(phi_range[0], phi_range[1]) * np.pi/180 + phi -= np.pi / 2 x = r * np.sin(theta) * np.cos(phi) y = r * np.sin(theta) * np.sin(phi) z = r * np.cos(theta) From 8ad2baea92570807c52f95f3034185d749cc5141 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Sat, 27 Apr 2024 00:26:39 +0800 Subject: [PATCH 12/18] readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 872ead188..312c74c95 100644 --- a/README.md +++ b/README.md @@ -42,4 +42,12 @@ python fy/run_tiancheng.py --save_states --debug --render_non_violate_video --t ``` python fy/run_tiancheng.py --render_non_violate_video --test_scene_cls 100 +``` + +``` +python fy/run_tiancheng.py --debug --save_states --test_scene_cls 100 +``` +run with watch +``` +python fy/run_watch_tiancheng.py ``` \ No newline at end of file From 62d9860499cfa03fb5fd871d427651496a6c4620 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Sat, 27 Apr 2024 00:48:52 +0800 Subject: [PATCH 13/18] added watch file --- fy/continuity.py | 2 +- fy/run_watch_tiancheng.py | 2 +- fy/utils.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fy/continuity.py b/fy/continuity.py index 6ea1afff0..4f8ab595d 100755 --- a/fy/continuity.py +++ b/fy/continuity.py @@ -190,7 +190,7 @@ def _check_scene(self): # Check visibility of the test obj at each frame print("Checking scene...") obj = self.test_obj[0] # work for only one test obj - for i, frame in enumerate(tqdm(range(self.flags.frame_start, self.flags.frame_end))): + for i, frame in enumerate((range(self.flags.frame_start, self.flags.frame_end))): bpy.context.scene.frame_set(frame) vis_obj = getVisibleVertexFraction("small_obj", self.rng, self.active_camera) vis_table = isPointVisible([0, 0, self.ref_h], [self.table_name, self.block_name, "small_obj"], self.active_camera) diff --git a/fy/run_watch_tiancheng.py b/fy/run_watch_tiancheng.py index 4373dbab0..842ccfba5 100644 --- a/fy/run_watch_tiancheng.py +++ b/fy/run_watch_tiancheng.py @@ -51,7 +51,7 @@ def run_watch(): if output == b'' and proc.poll() is not None: break if output: - print(output.strip().decode()) + print(output.strip().decode("utf-8", 'ignore')) # print("Still runing...") # wait for 1 second time.sleep(1) diff --git a/fy/utils.py b/fy/utils.py index 61b310d73..e1dffda68 100644 --- a/fy/utils.py +++ b/fy/utils.py @@ -300,7 +300,8 @@ def isPointVisible(pt, obj_names, camera_name="camera"): outcome = (target.name in obj_names) if not(outcome): - logging.warning(f"The object {obj_names} is blocked in at least one frame, {target.name} detected instead") + pass + # logging.warning(f"The object {obj_names} is blocked in at least one frame, {target.name} detected instead") return outcome From af30417216b4ee9849289f0d65fa4d5dbdc55231 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 2 May 2024 20:32:12 +0800 Subject: [PATCH 14/18] deleted some weird tables --- fy/configs/tables.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fy/configs/tables.txt b/fy/configs/tables.txt index 14d81b7f2..f07ba6b95 100644 --- a/fy/configs/tables.txt +++ b/fy/configs/tables.txt @@ -139,7 +139,6 @@ 04379243/bbae4abbff206a2a14038d588fd1342f 04379243/c4388c59f863de596edd3f7982f0bf26 04379243/c7358b3aed4160cb21bc3cf138f79e -04379243/6ab7ebf9b94176456f1e07a56c129dfc 04379243/6f019fe6ab60a3e37b11ae648ea92233 04379243/fe0e7198411fc340c057222d6d091c56 04379243/f0cee441d88de6dafebad4f49b26ec52 @@ -162,9 +161,7 @@ 04379243/87ca3e8e37367054dcabaa2ad147fa73 04379243/9d71f9424fc659e17a50afc9c93f8a50 04379243/97bda10740c4a74036b0f2a1430e993a -04379243/b14c4d5783a339609fd4171283f33ca8 04379243/a4dbf0f4fef1c36cf199233c9f2ce2ce -04379243/726164afa497b154b075b4c36d25279a 04379243/a99a74777f6aacf2489e5619471f9f53 04379243/f64617385056e0b1beedb4c8fd29e2d1 04379243/fe99a1127734f7852b70eac6546e93fd @@ -851,7 +848,6 @@ 04379243/8839cf79a5338a568ce66f12ba927a2b 04379243/be5501adc4564d9edf30786b8faddb78 04379243/f80cce35d167ff9b855931d119219022 -04379243/9a71b92445cd3f023a9bc242c86fb7a0 04379243/ae5ac5b2b027fcf9118ddfdb81cc6068 04379243/6f576d151a46bdefd5cb6d178687b980 04379243/8768002c872788b8e513931a191fd77c From b9c4d32da45455fe94c253971be679303258257a Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 2 May 2024 20:32:39 +0800 Subject: [PATCH 15/18] check scene with both cameras --- fy/base.py | 171 ++++++++++++++++++++++------------------------------- 1 file changed, 72 insertions(+), 99 deletions(-) diff --git a/fy/base.py b/fy/base.py index 804b52e67..56f04263b 100644 --- a/fy/base.py +++ b/fy/base.py @@ -76,15 +76,9 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.table_scale = 2 self.shapenet_table_ids = None - self.camera_front = None - self.camera_back = None - self.active_camera = None - self.floor_name = "floor_kb" self.table_name = "table_kb" self.block_name = "block_kb" - self.camera_name = "camera" - self.back_camera_name = "camera_back" self.render_data = ("rgba",) self.background_hdri_id = FLAGS.background_hdri_id @@ -94,7 +88,6 @@ def __init__(self, FLAGS, camera_path_config=None) -> None: self.test_obj_states = {"violation": None, "non_violation": None} self.test_obj = None self.default_camera_pos = [0, 0, 1] - self.back_camera_pos = [0, 0, 1] self.camera_look_at = [0,0,0] # load asset sources @@ -187,8 +180,7 @@ def _setup_everything(self): self.is_add_background_dynamic_objects = True self.scene.gravity = self.gravity - self._random_rotate_scene() - + if self.flags.debug: logging.info("Ignore background objects in debugging mode.") else: @@ -201,24 +193,19 @@ def _setup_everything(self): self.add_block_objects() self.add_test_objects() - self.renderer.save_state(f"temp_scene/can_1.blend") + # self._run_simulate() # self._random_rotate_scene() - # self._switch_to_back_camera() - # self.scene.camera.position = self.back_camera_pos - # self.scene.camera.look_at(self.camera_look_at) - - self._switch_to_front_camera() if self.is_move_camera: traj_idx = random.randint(0, len(self.camera_path_config)-1) self._set_camera_path(self.camera_path_config[traj_idx]) self.cur_camera_traj_idx = traj_idx self._set_camera_focus_point([0, 0, self.ref_h]) # auto set the height to be the table height if exists else: - self.camera_front.position = self.default_camera_pos - self.camera_front.look_at(self.camera_look_at) - # self.renderer.save_state(f"temp_scene/can_2.blend") + self.scene.camera.position = self.default_camera_pos + self.scene.camera.look_at(self.camera_look_at) + if not use_indoor: # the hdri center texture is wired, shift to avoid self.shift_scene([0,5,0]) @@ -226,7 +213,7 @@ def _setup_everything(self): if self.render_speedup: self._set_fast_rendering() - # self.renderer.save_state(f"temp_scene/can_3.blend") + # def _check_scene_visible(self): # frame_end = self.flags.frame_end @@ -262,6 +249,8 @@ def _set_camera_path(self, path_config): key_frame_val = path_config["key_frame_val"] camera = bpy.data.objects['camera'] + + bpy.ops.curve.primitive_bezier_circle_add(enter_editmode=False, align='WORLD', location=center, @@ -306,6 +295,7 @@ def _set_camera_path(self, path_config): keyframe.interpolation = 'CUBIC' keyframe.easing='EASE_IN_OUT' + camera.location = [0,0,0] return path_con def _set_camera_focus_point(self, look_at=[0,0,0]): @@ -367,13 +357,10 @@ def prepare_scene(self): while True: self.dynamic_objs = [] self.static_objs = [] - self.test_obj = [] self._setup_everything() - # self.renderer.save_state(f"temp_scene/can_4.blend") - if self._check_scene_with_front_back_cameras(): + if self._check_scene_with_all_cameras(): self.generate_keyframes() - # self.renderer.save_state(f"temp_scene/can_5.blend") return logging.warning("Current scene is invalid. Regenerating ") self.renderer.save_state(f"temp_scene/invalid_{self.i}.blend") @@ -383,40 +370,60 @@ def prepare_scene(self): self.camera_path_sample_stats[self.cur_camera_traj_idx] += 1 logging.info(f"Re-sampling... Current stats: {self.camera_path_sample_stats}") - def _check_scene_with_front_back_cameras(self): - """Check if the scene is valid for both the front and back camera. Return Flase if the scene is invalid. - TODO: implement (Override) this function + def _check_scene_with_all_cameras(self): + """Check if the scene is valid for all the cameras. Return Flase if the scene is invalid. + TODO: implement (Override) this function - Returns: - _type_: _description_ - """ - self._switch_to_front_camera() - if not self._check_scene(): - logging.warning("Check scene failed with Front Camera") - return False - - if not self.flags.add_back_camera: - return True - - self._switch_to_back_camera() - if self._check_scene(): - self._switch_to_front_camera() - return True - else: - logging.warning("Check scene failed with Back Camera") - return False + Returns: + _type_: _description_ + """ + # self.scene.camera.position = self.default_camera_pos + # self.scene.camera.look_at(self.camera_look_at) + print("checking camera view 1") + self._switch_to_view1() + if not (self._check_scene()): + logging.warning("Check scene failed with Camera View 1") + return False + + if not self.flags.render_multiview: + return True + + print("checking camera view 2") + self._switch_to_view2() + if self._check_scene(): + # change back to the front camera + self._switch_to_view1() + return True + else: + logging.warning("Check scene failed with Camera View 2") + return False + + def _switch_to_view1(self): + cam = bpy.data.objects["camera"] + self.scene.camera.position = [0,0,0] + self.scene.camera.rotation_quaternion = [1,0,0,0] + + for c in cam.constraints: + c.mute = False + + + def _switch_to_view2(self): + cam = bpy.data.objects["camera"] + for c in cam.constraints: + c.mute = True + self.scene.camera.position = self.alternative_camera_pos + self.scene.camera.look_at(self.alternative_camera_look_at) def _check_scene(self): - """Check if the scene is valid. Return Flase if the scene is invalid. - TODO: implement (Override) this function + """Check if the scene is valid Return Flase if the scene is invalid. + TODO: implement (Override) this function - Returns: - _type_: _description_ - """ - return True - # return self._check_scene_visible() - + Returns: + _type_: _description_ + """ + + return True def __enter__(self): return self @@ -526,35 +533,13 @@ def _setup_hdri_scene(self): self.scratch_dir = scratch_dir self.background_hdri = background_hdri - def _add_camera(self, scene, back_cam=False): - cam_name = self.camera_name if not back_cam else self.back_camera_name - - camera = kb.PerspectiveCamera(name=cam_name) - self.scene += camera - self.scene.camera = camera + def _add_camera(self, scene): + scene.camera = kb.PerspectiveCamera(name="camera") # avoid name clash + cam_name = "camera" set_name(cam_name) - if not back_cam: - self.camera_front = camera - else: - self.camera_back = camera - # self.scene.camera.position = self.back_camera_pos - # self.scene.camera.look_at(self.camera_look_at) - - def _switch_to_front_camera(self): - self._switch_camera(self.camera_name) - self.scene.camera = self.camera_front - - def _switch_to_back_camera(self): - self._switch_camera(self.back_camera_name) - self.scene.camera = self.camera_back - - def _switch_camera(self, camera_name): - bpy.context.scene.camera = bpy.data.objects[camera_name] - self.active_camera = camera_name - def _setup_indoor_scene(self, ): """Setup the indoor scene for rendering @@ -566,8 +551,8 @@ def _setup_indoor_scene(self, logging.info("Loading blender scene") blender_scene = rng.choice(self.scenes) self.load_blender_scene(blender_scene) + self._add_camera(self.scene) - # self.scene.camera.position = self.default_camera_pos # self.scene.camera.look_at(self.camera_look_at) @@ -584,8 +569,8 @@ def _setup_indoor_scene(self, bpy.data.objects[self.floor_name].hide_viewport = True - # self.default_camera_pos = [0, -1.8, 1] - # self.camera_look_at = [0, 0, 0.3] + self.default_camera_pos = [0, -1.8, 1] + self.camera_look_at = [0, 0, 0.3] if self.is_add_table: logging.info("Adding table to the scene") table_id = rng.choice(self.shapenet_table_ids) @@ -602,28 +587,16 @@ def _setup_indoor_scene(self, set_name(self.table_name) self.ref_h = table_h self.default_camera_pos[2] += table_h - self.back_camera_pos[2] += table_h + self.alternative_camera_pos[2] += table_h self.table_id = table_id # print(self.table_id) self.camera_look_at = [0, 0, self.ref_h] + self.alternative_camera_look_at = [0, 0, self.ref_h] self.rng = rng self.output_dir = output_dir self.scratch_dir = scratch_dir - if self.flags.add_back_camera: - self._add_camera(self.scene, back_cam=True) - self._switch_to_back_camera() - self.camera_back.position = self.back_camera_pos - self.camera_back.look_at(self.camera_look_at) - self.renderer.save_state(f"temp_scene/bgcam0.blend") - - self._add_camera(self.scene) - self._switch_to_front_camera() - self.camera_front.position = self.default_camera_pos - self.camera_front.look_at(self.camera_look_at) - self.renderer.save_state(f"temp_scene/bgcam1.blend") - ################################ # add random directional lighting # the light is placed at some random position sampled from a sphere, with min height @@ -651,7 +624,7 @@ def _setup_indoor_scene(self, direc_light = bpy.data.objects["direc_light"] direc_light.data.shadow_soft_size = self.rng.uniform(*shadow_soft_size) - + def shift_scene(self, shift: np.ndarray): """Shift the scene by a given vector. @@ -671,7 +644,6 @@ def add_object(self, **kwargs): """Add objects to the scene. - """ # delete the old object with the same name to avoid clashing kwargs["scale"] = scale @@ -879,7 +851,7 @@ def add_block_objects(self): position=(0, 0, 0), quaternion=(1,0,0,0), is_dynamic=True, - scale=1.25/100, + scale=1.25, name=self.block_name ) @@ -1016,8 +988,9 @@ def render_alternative_view(self, save_to_file=False, **kwargs): _type_: _description_ """ self.load_non_violation_scene() # only used for non-violation scene - self.scene.camera.position = self.alternative_camera_pos - self.scene.camera.look_at(self.alternative_camera_look_at) + # self.scene.camera.position = self.alternative_camera_pos + # self.scene.camera.look_at(self.alternative_camera_look_at) + self._switch_to_view2() data_stack = self.renderer.render(return_layers=self.render_data) if save_to_file: kb.write_image_dict(data_stack, self.output_dir, **kwargs) From d1b5ccb60a071b2aa480d23d3862ce6ddccf6669 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 2 May 2024 20:33:22 +0800 Subject: [PATCH 16/18] supported rendering with two cameras --- fy/continuity.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/fy/continuity.py b/fy/continuity.py index 4f8ab595d..e8e406e4d 100755 --- a/fy/continuity.py +++ b/fy/continuity.py @@ -12,7 +12,14 @@ from utils import align_can_objs, spherical_to_cartesian # TODO: test if obj is on the table at most of time; why velocity changes after teletransportation - +invalid_small_objs = ["02946921/f8fd565d00a7a6f9fef1ca8c5a3d2e08", + "02946921/343287cd508a798d38df439574e01b2", + "02946921/91a524cc9c9be4872999f92861cdea7a", + "02946921/4d4fc73864844dad1ceb7b8cc3792fd", + "02946921/19fa6044dd31aa8e9487fa707cec1558", + "02946921/129880fda38f3f2ba1ab68e159bfb347", + "02946921/7b643c8136a720d9db4a36333be9155", + ] class ContinuityTestScene(BaseTestScene): """Test scene for permanance violation. @@ -26,17 +33,18 @@ class ContinuityTestScene(BaseTestScene): def __init__(self, FLAGS) -> None: super().__init__(FLAGS) + self.is_add_background_static_objects = False self.frame_violation_start = -1 # two cases: # 1. object disappears # 0. object teleports - self.violation_type = np.random.binomial(n=1,p=0.5) + self.violation_type = 1#np.random.binomial(n=1,p=0.5) self.is_move_camera = np.random.binomial(n=1,p=0.5) self.gravity = (0, 0, -4.9) self.back_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[70, 80], phi_range=[-45+180, 45+180]) - self.default_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[75, 85]) - self.camera_look_at = (0, 0, self.ref_h) + self.alternative_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[75, 85]) + # if not(self.violation_type): # self.default_camera_pos = spherical_to_cartesian(r_range=[2, 3], theta_range=[75, 85]) @@ -51,6 +59,10 @@ def __init__(self, FLAGS) -> None: def prepare_scene(self): print("preparing scene ...") super().prepare_scene() + self.alternative_camera_pos[2] += self.ref_h + self.default_camera_pos[2] += self.ref_h + self.camera_look_at = (0, 0, self.ref_h) + self.alternative_camera_look_at = (0, 0, self.ref_h) if not(self.violation_type): # remove block object @@ -134,8 +146,9 @@ def add_test_objects(self): self.block_obj.position = (0, 0, -10) # initialize the obj. Note the the initial position should be (0,0,0) # so that the dist between its CoM and the table surface can be calculated - small_obj_id = [name for name, spec in shapenet_assets._assets.items() + small_obj_id_all = [name for name, spec in shapenet_assets._assets.items() if spec["metadata"]["category"] == "can"] + small_obj_id = [id for id in small_obj_id_all if id not in invalid_small_objs] small_obj_id = self.rng.choice(small_obj_id) small_obj = self.add_object(asset_id=small_obj_id, position=(0, 0, 0), @@ -192,8 +205,8 @@ def _check_scene(self): obj = self.test_obj[0] # work for only one test obj for i, frame in enumerate((range(self.flags.frame_start, self.flags.frame_end))): bpy.context.scene.frame_set(frame) - vis_obj = getVisibleVertexFraction("small_obj", self.rng, self.active_camera) - vis_table = isPointVisible([0, 0, self.ref_h], [self.table_name, self.block_name, "small_obj"], self.active_camera) + vis_obj = getVisibleVertexFraction("small_obj", self.rng, "camera") + vis_table = isPointVisible([0, 0, self.ref_h], [self.table_name, self.block_name, "small_obj"], "camera") visibility_obj[i] = vis_obj visibility_table[i] = vis_table obj_on_table[i] = obj.keyframes["position"][frame][2] > self.ref_h-0.05 @@ -211,7 +224,11 @@ def _check_scene(self): # cond_2 = visibility_obj[0] >= 0.15 \ # and visibility_obj[5] >= 0.15 \ # and visibility_obj[-6] >= 0.15 - cond_2 = is_obj_visible[:6].max() and is_obj_visible[-7:].max() + if less_strict: + cond_2 = is_obj_visible[:10].max() and is_obj_visible[-10:].max() + else: + cond_2 = is_obj_visible[:6].max() and is_obj_visible[-7:].max() + cond = [cond_2, cond_1 or less_strict, in_view[0], @@ -220,6 +237,7 @@ def _check_scene(self): is_table_visible[6:-6].min(), obj_on_table.sum() >= 16 or less_strict ] + else: cond = [visibility_obj[0] >= 0.5, in_view[0], @@ -229,6 +247,10 @@ def _check_scene(self): ] is_valid = np.min(cond) + logging.error(cond) + # logging.error(np.min(cond)) + # logging.error(is_obj_visible[:10]) + # logging.error( is_obj_visible[-10:]) if is_valid: # set when the test object is set disappeared if self.violation_type: From 3e22ef9c92154ffa39f960eca46bb1625fecd011 Mon Sep 17 00:00:00 2001 From: maple5717 Date: Thu, 2 May 2024 20:33:35 +0800 Subject: [PATCH 17/18] run --- fy/run_tiancheng.py | 51 +++++++++++++++++++-------------------- fy/run_watch_tiancheng.py | 2 +- run_tiancheng.sh | 1 + 3 files changed, 27 insertions(+), 27 deletions(-) create mode 100755 run_tiancheng.sh diff --git a/fy/run_tiancheng.py b/fy/run_tiancheng.py index 85a46492f..5fa0390ec 100644 --- a/fy/run_tiancheng.py +++ b/fy/run_tiancheng.py @@ -105,6 +105,7 @@ def generate_test_scene(test_class, FLAGS,output_dir, i) -> None: os.makedirs(video_dir) # print(video_dir); return + with test_class(FLAGS) as test_scene: # first prepare the scene logging.info("Preparing the scene") @@ -113,35 +114,33 @@ def generate_test_scene(test_class, FLAGS,output_dir, i) -> None: # render the violation state - - cameras = [test_scene.camera_name] - if FLAGS.add_back_camera: - cameras += [test_scene.back_camera_name] - camera_names = ["front", "back"] - for j, camera in enumerate(cameras): - test_scene._switch_camera(camera) - camera_name = camera_names[j] - - test_scene.change_output_dir( output_dir + f"violation_{camera_name}" ) - if FLAGS.render_violate_video: - logging.info("Rendering the violation video") - start_time = time.time() - test_scene.render(save_to_file=True) - write_video(output_dir + f"violation_{camera_name}/", output_dir + f"violation_{i}_{camera_name}.mp4") - logging.info(f"Rendering the violation video took {time.time() - start_time} seconds") - - # load the non-violation state and render it - logging.info("Loading the non-violation state") + if FLAGS.render_non_violate_video: + # test_scene.load_non_violation_scene() + test_scene.change_output_dir( output_dir + "non_violation_view_1" ) + logging.info("Rendering the non-violation video") + start_time = time.time() test_scene.load_non_violation_scene() - test_scene.change_output_dir( output_dir + f"non_violation_{camera_name}" ) + test_scene.render(save_to_file=True) + # write_video(output_dir + "non_violation/", output_dir + "non_violation.mp4") + logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") - # igore rendering if debug is on - if FLAGS.render_non_violate_video: - logging.info("Rendering the non-violation video") + if FLAGS.render_multiview and test_scene.alternative_camera_pos: + logging.info("Rendering the non-violation video with alternative camera positions") + test_scene.change_output_dir( output_dir + "non_violation_view_2" ) start_time = time.time() - test_scene.render(save_to_file=True) - write_video(output_dir + f"non_violation_{camera_name}/", video_dir + f"non_violation_{i}_{camera_name}.mp4") - logging.info(f"Rendering the non-violation video took {time.time() - start_time} seconds") + test_scene.render_alternative_view(save_to_file=True) + # write_video(output_dir + "non_violation_multiview/", output_dir + "non_violation_multiview.mp4") + logging.info(f"Rendering the non-violation video with alternative camera positions took {time.time() - start_time} seconds") + + if FLAGS.render_violate_video: + # render the violation state + test_scene.change_output_dir( output_dir + "violation" ) + test_scene.load_violation_scene() + logging.info("Rendering the violation video") + start_time = time.time() + test_scene.render(save_to_file=True) + # write_video(output_dir + "violation/", output_dir + "violation.mp4") + logging.info(f"Rendering the violation video took {time.time() - start_time} seconds") if __name__ == "__main__": main() diff --git a/fy/run_watch_tiancheng.py b/fy/run_watch_tiancheng.py index 842ccfba5..6f3f3e6dc 100644 --- a/fy/run_watch_tiancheng.py +++ b/fy/run_watch_tiancheng.py @@ -33,7 +33,7 @@ def run_watch(): # print("Job finished.") # break - all_args = ["/bin/python", "fy/run_tiancheng.py", "--render_non_violate_video", "--test_scene_cls", "1000"] + other_args + all_args = ["/bin/python", "fy/run_tiancheng.py", "--render_non_violate_video", "--render_multiview", "--test_scene_cls", "1000"] + other_args print("=============================================================") print("Restarting the job with the following arguments:") print(" ".join(all_args)) diff --git a/run_tiancheng.sh b/run_tiancheng.sh new file mode 100755 index 000000000..cf0485573 --- /dev/null +++ b/run_tiancheng.sh @@ -0,0 +1 @@ +python fy/run_tiancheng.py --render_non_violate_video --render_multiview --test_scene_cls 100 From 9a50c0536d332ed5a5ab7c85324763e3127ce92b Mon Sep 17 00:00:00 2001 From: maple5717 Date: Fri, 3 May 2024 13:10:45 +0800 Subject: [PATCH 18/18] fixed the view problem of the alternative camera --- fy/base.py | 10 ++++++++-- fy/continuity.py | 4 ++-- fy/run_tiancheng.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/fy/base.py b/fy/base.py index 56f04263b..422e659bb 100644 --- a/fy/base.py +++ b/fy/base.py @@ -400,8 +400,14 @@ def _check_scene_with_all_cameras(self): def _switch_to_view1(self): cam = bpy.data.objects["camera"] - self.scene.camera.position = [0,0,0] - self.scene.camera.rotation_quaternion = [1,0,0,0] + + if self.is_move_camera: + self.scene.camera.position = [0,0,0] + self.scene.camera.rotation_quaternion = [1,0,0,0] + else: + self.scene.camera.position = self.default_camera_pos + self.scene.camera.look_at(self.camera_look_at) + for c in cam.constraints: c.mute = False diff --git a/fy/continuity.py b/fy/continuity.py index e8e406e4d..014d77943 100755 --- a/fy/continuity.py +++ b/fy/continuity.py @@ -42,8 +42,8 @@ def __init__(self, FLAGS) -> None: self.is_move_camera = np.random.binomial(n=1,p=0.5) self.gravity = (0, 0, -4.9) - self.back_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[70, 80], phi_range=[-45+180, 45+180]) - self.alternative_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[75, 85]) + # self.back_camera_pos = spherical_to_cartesian(r_range=[2.25, 2.75], theta_range=[70, 80], phi_range=[-45+180, 45+180]) + self.alternative_camera_pos = spherical_to_cartesian(r_range=[1.5, 2.25], theta_range=[70, 80], phi_range=[-45+180, 45+180]) # if not(self.violation_type): diff --git a/fy/run_tiancheng.py b/fy/run_tiancheng.py index 5fa0390ec..e1b1fcba4 100644 --- a/fy/run_tiancheng.py +++ b/fy/run_tiancheng.py @@ -95,7 +95,7 @@ def main() -> None: except Exception as e: logging.error(f"Error rendering {test_name} test {n}: {e}\n Skipping to the next one.") # if debug is on, raise the exception - if FLAGS.debug: + if FLAGS.debug or True: raise continue