Humanoid

.gitignore

@@ -81,3 +81,7 @@ ipython_config.py
 venv/
 env.bak/
 venv.bak/
+
+# Smooth exploration
+gym/smooth_exploration/data*
+gym/smooth_exploration/figures*

gym/envs/__init__.py

@@ -19,28 +19,40 @@
     "Anymal": ".anymal_c.anymal",
     "A1": ".a1.a1",
     "HumanoidRunning": ".mit_humanoid.humanoid_running",
+    "Pendulum": ".pendulum.pendulum",
+    "Lander": ".mit_humanoid.lander",
 }
 
 config_dict = {
     "CartpoleCfg": ".cartpole.cartpole_config",
     "MiniCheetahCfg": ".mini_cheetah.mini_cheetah_config",
     "MiniCheetahRefCfg": ".mini_cheetah.mini_cheetah_ref_config",
     "MiniCheetahOscCfg": ".mini_cheetah.mini_cheetah_osc_config",
+    "MiniCheetahSACCfg": ".mini_cheetah.mini_cheetah_SAC_config",
     "MITHumanoidCfg": ".mit_humanoid.mit_humanoid_config",
     "A1Cfg": ".a1.a1_config",
     "AnymalCFlatCfg": ".anymal_c.flat.anymal_c_flat_config",
     "HumanoidRunningCfg": ".mit_humanoid.humanoid_running_config",
+    "PendulumCfg": ".pendulum.pendulum_config",
+    "PendulumSACCfg": ".pendulum.pendulum_SAC_config",
+    "LanderCfg": ".mit_humanoid.lander_config",
+    "PendulumPSDCfg": ".pendulum.pendulum_PSD_config",
 }
 
 runner_config_dict = {
     "CartpoleRunnerCfg": ".cartpole.cartpole_config",
     "MiniCheetahRunnerCfg": ".mini_cheetah.mini_cheetah_config",
     "MiniCheetahRefRunnerCfg": ".mini_cheetah.mini_cheetah_ref_config",
     "MiniCheetahOscRunnerCfg": ".mini_cheetah.mini_cheetah_osc_config",
+    "MiniCheetahSACRunnerCfg": ".mini_cheetah.mini_cheetah_SAC_config",
     "MITHumanoidRunnerCfg": ".mit_humanoid.mit_humanoid_config",
     "A1RunnerCfg": ".a1.a1_config",
     "AnymalCFlatRunnerCfg": ".anymal_c.flat.anymal_c_flat_config",
     "HumanoidRunningRunnerCfg": ".mit_humanoid.humanoid_running_config",
+    "PendulumRunnerCfg": ".pendulum.pendulum_config",
+    "PendulumSACRunnerCfg": ".pendulum.pendulum_SAC_config",
+    "LanderRunnerCfg": ".mit_humanoid.lander_config",
+    "PendulumPSDRunnerCfg": ".pendulum.pendulum_PSD_config",
 }
 
 task_dict = {
@@ -56,14 +68,22 @@
         "MiniCheetahOscCfg",
         "MiniCheetahOscRunnerCfg",
     ],
+    "sac_mini_cheetah": [
+        "MiniCheetahRef",
+        "MiniCheetahSACCfg",
+        "MiniCheetahSACRunnerCfg"
+    ],
     "humanoid": ["MIT_Humanoid", "MITHumanoidCfg", "MITHumanoidRunnerCfg"],
     "humanoid_running": [
         "HumanoidRunning",
         "HumanoidRunningCfg",
         "HumanoidRunningRunnerCfg",
     ],
-    "a1": ["A1", "A1Cfg", "A1RunnerCfg"],
     "flat_anymal_c": ["Anymal", "AnymalCFlatCfg", "AnymalCFlatRunnerCfg"],
+    "pendulum": ["Pendulum", "PendulumCfg", "PendulumRunnerCfg"],
+    "sac_pendulum": ["Pendulum", "PendulumSACCfg", "PendulumSACRunnerCfg"],
+    "lander": ["Lander", "LanderCfg", "LanderRunnerCfg"],
+    "psd_pendulum": ["Pendulum", "PendulumPSDCfg", "PendulumPSDRunnerCfg"],
 }
 
 for class_name, class_location in class_dict.items():

gym/envs/a1/a1_config.py

@@ -137,13 +137,13 @@ class scaling(LeggedRobotCfg.scaling):
 
 class A1RunnerCfg(LeggedRobotRunnerCfg):
     seed = -1
+    runner_class_name = "OldPolicyRunner"
 
-    class policy(LeggedRobotRunnerCfg.policy):
-        actor_hidden_dims = [256, 256, 256]
-        critic_hidden_dims = [256, 256, 256]
+    class actor(LeggedRobotRunnerCfg.actor):
+        hidden_dims = [256, 256, 256]
         # activation can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
-        actor_obs = [
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -154,7 +154,19 @@ class policy(LeggedRobotRunnerCfg.policy):
             "commands",
         ]
 
-        critic_obs = [
+        actions = ["dof_pos_target"]
+
+        class noise:
+            dof_pos_obs = 0.005
+            dof_vel = 0.005
+            base_ang_vel = 0.05
+            projected_gravity = 0.02
+
+    class critic:
+        hidden_dims = [256, 256, 256]
+        # activation can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
+        activation = "elu"
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -165,16 +177,8 @@ class policy(LeggedRobotRunnerCfg.policy):
             "commands",
         ]
 
-        actions = ["dof_pos_target"]
-
-        class noise:
-            dof_pos_obs = 0.005  # can be made very low
-            dof_vel = 0.005
-            base_ang_vel = 0.05
-            projected_gravity = 0.02
-
-        class reward(LeggedRobotRunnerCfg.policy.reward):
-            class weights(LeggedRobotRunnerCfg.policy.reward.weights):
+        class reward:
+            class weights:
                 tracking_lin_vel = 1.0
                 tracking_ang_vel = 1.0
                 lin_vel_z = 0.0

gym/envs/anymal_c/flat/anymal_c_flat_config.py

@@ -85,6 +85,7 @@ class push_robots:
         toggle = True
         interval_s = 1
         max_push_vel_xy = 0.5
+        push_box_dims = [0.2, 0.2, 0.2]
 
     class domain_rand(LeggedRobotCfg.domain_rand):
         randomize_base_mass = True
@@ -120,14 +121,14 @@ class scaling(LeggedRobotCfg.scaling):
 
 class AnymalCFlatRunnerCfg(LeggedRobotRunnerCfg):
     seed = -1
+    runner_class_name = "OldPolicyRunner"
 
-    class policy(LeggedRobotRunnerCfg.policy):
-        actor_hidden_dims = [256, 256, 256]
-        critic_hidden_dims = [256, 256, 256]
+    class actor(LeggedRobotRunnerCfg.actor):
+        hidden_dims = [256, 256, 256]
         # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
 
-        actor_obs = [
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -138,7 +139,19 @@ class policy(LeggedRobotRunnerCfg.policy):
             "dof_pos_history",
         ]
 
-        critic_obs = [
+        actions = ["dof_pos_target"]
+
+        class noise:
+            dof_pos_obs = 0.005
+            dof_vel = 0.005
+            base_ang_vel = 0.05  # 0.027, 0.14, 0.37
+            projected_gravity = 0.02
+
+    class critic(LeggedRobotRunnerCfg.critic):
+        hidden_dims = [256, 256, 256]
+        # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
+        activation = "elu"
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -149,16 +162,8 @@ class policy(LeggedRobotRunnerCfg.policy):
             "dof_pos_history",
         ]
 
-        actions = ["dof_pos_target"]
-
-        class noise:
-            dof_pos_obs = 0.005  # can be made very low
-            dof_vel = 0.005
-            base_ang_vel = 0.05  # 0.027, 0.14, 0.37
-            projected_gravity = 0.02
-
-        class reward(LeggedRobotRunnerCfg.policy.reward):
-            class weights(LeggedRobotRunnerCfg.policy.reward.weights):
+        class reward:
+            class weights:
                 tracking_lin_vel = 3.0
                 tracking_ang_vel = 1.0
                 lin_vel_z = 0.0

gym/envs/anymal_c/mixed_terrains/anymal_c_rough_config.py

@@ -56,12 +56,15 @@ class domain_rand(AnymalCFlatCfg.domain_rand):
 
 
 class AnymalCRoughCCfgPPO(AnymalCFlatCfgPPO):
-    class policy(AnymalCFlatCfgPPO.policy):
-        actor_hidden_dims = [128, 64, 32]
+    class actor(AnymalCFlatCfgPPO.actor):
+        hidden_dims = [128, 64, 32]
         critic_hidden_dims = [128, 64, 32]
         # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
 
+    class critic(AnymalCFlatCfgPPO.critic):
+        pass
+
     class algorithm(AnymalCFlatCfgPPO.algorithm):
         entropy_coef = 0.01
 

gym/envs/base/base_task.py

@@ -18,33 +18,20 @@ def __init__(self, gym, sim, cfg, sim_params, sim_device, headless):
         # * env device is GPU only if sim is on GPU and use_gpu_pipeline=True,
         # * otherwise returned tensors are copied to CPU by physX.
         if sim_device_type == "cuda" and sim_params.use_gpu_pipeline:
-            self.device = self.sim_device
+            device = self.sim_device
         else:
-            self.device = "cpu"
+            device = "cpu"
 
         # * graphics device for rendering, -1 for no rendering
         self.graphics_device_id = self.sim_device_id
 
-        self.num_envs = cfg.env.num_envs
         self.num_actuators = cfg.env.num_actuators
 
         # * optimization flags for pytorch JIT
         torch._C._jit_set_profiling_mode(False)
         torch._C._jit_set_profiling_executor(False)
 
-        # allocate buffers
-        self.to_be_reset = torch.ones(
-            self.num_envs, device=self.device, dtype=torch.bool
-        )
-        self.terminated = torch.ones(
-            self.num_envs, device=self.device, dtype=torch.bool
-        )
-        self.episode_length_buf = torch.zeros(
-            self.num_envs, device=self.device, dtype=torch.long
-        )
-        self.timed_out = torch.zeros(
-            self.num_envs, device=self.device, dtype=torch.bool
-        )
+        super().__init__(num_envs=cfg.env.num_envs, device=device)
 
         # todo: read from config
         self.enable_viewer_sync = True