js/lqc

gym/envs/__init__.py

@@ -19,8 +19,8 @@
     "Anymal": ".anymal_c.anymal",
     "A1": ".a1.a1",
     "HumanoidRunning": ".mit_humanoid.humanoid_running",
-    "HumanoidBouncing": ".mit_humanoid.humanoid_bouncing",
     "Pendulum": ".pendulum.pendulum",
+    "LQRPendulum": ".pendulum.lqr_pendulum",
 }
 
 config_dict = {
@@ -32,8 +32,8 @@
     "A1Cfg": ".a1.a1_config",
     "AnymalCFlatCfg": ".anymal_c.flat.anymal_c_flat_config",
     "HumanoidRunningCfg": ".mit_humanoid.humanoid_running_config",
-    "HumanoidBouncingCfg": ".mit_humanoid.humanoid_bouncing_config",
     "PendulumCfg": ".pendulum.pendulum_config",
+    "LQRPendulumCfg": ".pendulum.lqr_pendulum_config",
 }
 
 runner_config_dict = {
@@ -45,8 +45,8 @@
     "A1RunnerCfg": ".a1.a1_config",
     "AnymalCFlatRunnerCfg": ".anymal_c.flat.anymal_c_flat_config",
     "HumanoidRunningRunnerCfg": ".mit_humanoid.humanoid_running_config",
-    "HumanoidBouncingRunnerCfg": ".mit_humanoid.humanoid_bouncing_config",
     "PendulumRunnerCfg": ".pendulum.pendulum_config",
+    "LQRPendulumRunnerCfg": ".pendulum.lqr_pendulum_config",
 }
 
 task_dict = {
@@ -68,12 +68,9 @@
         "HumanoidRunningCfg",
         "HumanoidRunningRunnerCfg",
     ],
-    "humanoid_bouncing": ["HumanoidBouncing",
-                          "HumanoidBouncingCfg",
-                          "HumanoidBouncingRunnerCfg"],
-    "a1": ["A1", "A1Cfg", "A1RunnerCfg"],
     "flat_anymal_c": ["Anymal", "AnymalCFlatCfg", "AnymalCFlatRunnerCfg"],
-    "pendulum": ["Pendulum", "PendulumCfg", "PendulumRunnerCfg"]
+    "pendulum": ["Pendulum", "PendulumCfg", "PendulumRunnerCfg"],
+    "lqr_pendulum": ["LQRPendulum", "LQRPendulumCfg", "LQRPendulumRunnerCfg"],
 }
 
 for class_name, class_location in class_dict.items():

gym/envs/a1/a1_config.py

@@ -137,13 +137,13 @@ class scaling(LeggedRobotCfg.scaling):
 
 class A1RunnerCfg(LeggedRobotRunnerCfg):
     seed = -1
+    runner_class_name = "OldPolicyRunner"
 
-    class policy(LeggedRobotRunnerCfg.policy):
-        actor_hidden_dims = [256, 256, 256]
-        critic_hidden_dims = [256, 256, 256]
+    class actor(LeggedRobotRunnerCfg.actor):
+        hidden_dims = [256, 256, 256]
         # activation can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
-        actor_obs = [
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -154,7 +154,19 @@ class policy(LeggedRobotRunnerCfg.policy):
             "commands",
         ]
 
-        critic_obs = [
+        actions = ["dof_pos_target"]
+
+        class noise:
+            dof_pos_obs = 0.005
+            dof_vel = 0.005
+            base_ang_vel = 0.05
+            projected_gravity = 0.02
+
+    class critic:
+        hidden_dims = [256, 256, 256]
+        # activation can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
+        activation = "elu"
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -165,16 +177,8 @@ class policy(LeggedRobotRunnerCfg.policy):
             "commands",
         ]
 
-        actions = ["dof_pos_target"]
-
-        class noise:
-            dof_pos_obs = 0.005  # can be made very low
-            dof_vel = 0.005
-            base_ang_vel = 0.05
-            projected_gravity = 0.02
-
-        class reward(LeggedRobotRunnerCfg.policy.reward):
-            class weights(LeggedRobotRunnerCfg.policy.reward.weights):
+        class reward:
+            class weights:
                 tracking_lin_vel = 1.0
                 tracking_ang_vel = 1.0
                 lin_vel_z = 0.0

gym/envs/anymal_c/flat/anymal_c_flat_config.py

@@ -85,6 +85,7 @@ class push_robots:
         toggle = True
         interval_s = 1
         max_push_vel_xy = 0.5
+        push_box_dims = [0.2, 0.2, 0.2]
 
     class domain_rand(LeggedRobotCfg.domain_rand):
         randomize_base_mass = True
@@ -120,14 +121,14 @@ class scaling(LeggedRobotCfg.scaling):
 
 class AnymalCFlatRunnerCfg(LeggedRobotRunnerCfg):
     seed = -1
+    runner_class_name = "OldPolicyRunner"
 
-    class policy(LeggedRobotRunnerCfg.policy):
-        actor_hidden_dims = [256, 256, 256]
-        critic_hidden_dims = [256, 256, 256]
+    class actor(LeggedRobotRunnerCfg.actor):
+        hidden_dims = [256, 256, 256]
         # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
 
-        actor_obs = [
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -138,7 +139,19 @@ class policy(LeggedRobotRunnerCfg.policy):
             "dof_pos_history",
         ]
 
-        critic_obs = [
+        actions = ["dof_pos_target"]
+
+        class noise:
+            dof_pos_obs = 0.005
+            dof_vel = 0.005
+            base_ang_vel = 0.05  # 0.027, 0.14, 0.37
+            projected_gravity = 0.02
+
+    class critic(LeggedRobotRunnerCfg.critic):
+        hidden_dims = [256, 256, 256]
+        # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
+        activation = "elu"
+        obs = [
             "base_height",
             "base_lin_vel",
             "base_ang_vel",
@@ -149,16 +162,8 @@ class policy(LeggedRobotRunnerCfg.policy):
             "dof_pos_history",
         ]
 
-        actions = ["dof_pos_target"]
-
-        class noise:
-            dof_pos_obs = 0.005  # can be made very low
-            dof_vel = 0.005
-            base_ang_vel = 0.05  # 0.027, 0.14, 0.37
-            projected_gravity = 0.02
-
-        class reward(LeggedRobotRunnerCfg.policy.reward):
-            class weights(LeggedRobotRunnerCfg.policy.reward.weights):
+        class reward:
+            class weights:
                 tracking_lin_vel = 3.0
                 tracking_ang_vel = 1.0
                 lin_vel_z = 0.0

gym/envs/anymal_c/mixed_terrains/anymal_c_rough_config.py

@@ -56,12 +56,15 @@ class domain_rand(AnymalCFlatCfg.domain_rand):
 
 
 class AnymalCRoughCCfgPPO(AnymalCFlatCfgPPO):
-    class policy(AnymalCFlatCfgPPO.policy):
-        actor_hidden_dims = [128, 64, 32]
+    class actor(AnymalCFlatCfgPPO.actor):
+        hidden_dims = [128, 64, 32]
         critic_hidden_dims = [128, 64, 32]
         # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
 
+    class critic(AnymalCFlatCfgPPO.critic):
+        pass
+
     class algorithm(AnymalCFlatCfgPPO.algorithm):
         entropy_coef = 0.01
 

gym/envs/base/fixed_robot.py

@@ -417,8 +417,8 @@ def _init_buffers(self):
         self.act_idx = to_torch(actuated_idx, dtype=torch.long, device=self.device)
         # * check that init range highs and lows are consistent
         # * and repopulate to match
-        if self.cfg.init_state.reset_mode == "reset_to_range":
-            self.initialize_ranges_for_initial_conditions()
+        # if self.cfg.init_state.reset_mode == "reset_to_range":
+        self.initialize_ranges_for_initial_conditions()
 
     def initialize_ranges_for_initial_conditions(self):
         self.dof_pos_range = torch.zeros(

gym/envs/base/fixed_robot_config.py

@@ -123,34 +123,33 @@ class FixedRobotCfgPPO(BaseConfig):
     class logging:
         enable_local_saving = True
 
-    class policy:
+    class actor:
         init_noise_std = 1.0
-        actor_hidden_dims = [512, 256, 128]
-        critic_hidden_dims = [512, 256, 128]
+        hidden_dims = [512, 256, 128]
         # * can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
-        # only for 'ActorCriticRecurrent':
-        # rnn_type = 'lstm'
-        # rnn_hidden_size = 512
-        # rnn_num_layers = 1
-
-        actor_obs = [
+        obs = [
             "observation_a",
             "observation_b",
             "these_need_to_be_atributes_(states)_of_the_robot_env",
         ]
-
-        critic_obs = [
-            "observation_x",
-            "observation_y",
-            "critic_obs_can_be_the_same_or_different_than_actor_obs",
-        ]
+        normalize_obs = True
 
         actions = ["tau_ff"]
         disable_actions = False
 
         class noise:
-            noise = 0.1  # implement as needed, also in your robot class
+            observation_a = 0.1  # implement as needed, also in your robot class
+
+    class critic:
+        hidden_dims = [512, 256, 128]
+        activation = "elu"
+        normalize_obs = True
+        obs = [
+            "observation_x",
+            "observation_y",
+            "critic_obs_can_be_the_same_or_different_than_actor_obs",
+        ]
 
         class rewards:
             class weights:
@@ -182,7 +181,7 @@ class algorithm:
 
     class runner:
         policy_class_name = "ActorCritic"
-        algorithm_class_name = "PPO"
+        algorithm_class_name = "PPO2"
         num_steps_per_env = 24  # per iteration
         max_iterations = 500  # number of policy updates
 

gym/envs/base/legged_robot_config.py

@@ -233,10 +233,9 @@ class LeggedRobotRunnerCfg(BaseConfig):
     class logging:
         enable_local_saving = True
 
-    class policy:
+    class actor:
         init_noise_std = 1.0
-        actor_hidden_dims = [512, 256, 128]
-        critic_hidden_dims = [512, 256, 128]
+        hidden_dims = [512, 256, 128]
         # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
         activation = "elu"
         normalize_obs = True
@@ -263,6 +262,17 @@ class noise:
             projected_gravity = 0.05
             height_measurements = 0.1
 
+    class critic:
+        hidden_dims = [512, 256, 128]
+        # can be elu, relu, selu, crelu, lrelu, tanh, sigmoid
+        activation = "elu"
+        normalize_obs = True
+        obs = [
+            "observation_x",
+            "observation_y",
+            "critic_obs_can_be_the_same_or_different_than_actor_obs",
+        ]
+
         class reward:
             class weights:
                 tracking_lin_vel = 0.0
@@ -283,25 +293,30 @@ class termination_weight:
                 termination = 0.01
 
     class algorithm:
-        # * training params
-        value_loss_coef = 1.0
-        use_clipped_value_loss = True
+        # both
+        gamma = 0.99
+        lam = 0.95
+        # shared
+        batch_size = 2**15
+        max_grad_steps = 10
+        # new
+        storage_size = 2**17  # new
+        mini_batch_size = 2**15  #  new
+
         clip_param = 0.2
-        entropy_coef = 0.01
-        num_learning_epochs = 5
-        # * mini batch size = num_envs*nsteps / nminibatches
-        num_mini_batches = 4
         learning_rate = 1.0e-3
+        max_grad_norm = 1.0
+        # Critic
+        use_clipped_value_loss = True
+        # Actor
+        entropy_coef = 0.01
         schedule = "adaptive"  # could be adaptive, fixed
-        gamma = 0.99
-        lam = 0.95
         desired_kl = 0.01
-        max_grad_norm = 1.0
 
     class runner:
         policy_class_name = "ActorCritic"
-        algorithm_class_name = "PPO"
-        num_steps_per_env = 24
+        algorithm_class_name = "PPO2"
+        num_steps_per_env = 24  # deprecate
         max_iterations = 1500
         save_interval = 50
         run_name = ""

gym/envs/base/task_skeleton.py

@@ -45,6 +45,7 @@ def reset(self):
         """Reset all robots"""
         self._reset_idx(torch.arange(self.num_envs, device=self.device))
         self.step()
+        self.episode_length_buf[:] = 0
 
     def _reset_buffers(self):
         self.to_be_reset[:] = False

gym/envs/cartpole/cartpole_config.py

@@ -67,24 +67,21 @@ class CartpoleRunnerCfg(FixedRobotCfgPPO):
     seed = -1
     runner_class_name = "OnPolicyRunner"
 
-    class policy(FixedRobotCfgPPO.policy):
+    class actor(FixedRobotCfgPPO.actor):
         init_noise_std = 1.0
         num_layers = 2
         num_units = 32
-        actor_hidden_dims = [num_units] * num_layers
-        critic_hidden_dims = [num_units] * num_layers
+        hidden_dims = [num_units] * num_layers
         activation = "elu"
 
-        actor_obs = [
+        obs = [
             "cart_obs",
             "pole_trig_obs",
             "dof_vel",
             "cart_vel_square",
             "pole_vel_square",
         ]
 
-        critic_obs = actor_obs
-
         actions = ["tau_ff"]
 
         class noise:
@@ -94,6 +91,20 @@ class noise:
             pole_vel = 0.010
             actuation = 0.00
 
+    class critic:
+        num_layers = 2
+        num_units = 32
+        hidden_dims = [num_units] * num_layers
+        activation = "elu"
+
+        obs = [
+            "cart_obs",
+            "pole_trig_obs",
+            "dof_vel",
+            "cart_vel_square",
+            "pole_vel_square",
+        ]
+
         class reward:
             class weights:
                 pole_pos = 5
@@ -125,7 +136,7 @@ class algorithm(FixedRobotCfgPPO.algorithm):
 
     class runner(FixedRobotCfgPPO.runner):
         policy_class_name = "ActorCritic"
-        algorithm_class_name = "PPO"
+        algorithm_class_name = "PPO2"
         num_steps_per_env = 96  # per iteration
         max_iterations = 500  # number of policy updates