a1

.gitignore

@@ -0,0 +1,5 @@
+*.pyc
+
+.vs/
+__pycache__/
+output/

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Xue Bin Peng
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,42 @@
+# RL Assignments
+
+Implementation of rl algorithms and environments.
+
+## Getting Started
+
+Install requirements:
+
+`pip install -r requirements.txt`
+
+and it should be good to go.
+
+
+## Train Models
+
+To train a policy, run the following command:
+
+``python run.py --env_config data/envs/dm_cheetah.yaml --agent_config a2/dm_cheetah_cem_agent.yaml --mode train --log_file output/log.txt --out_model_file output/model.pt --visualize``
+
+- `--env_config` specifies the configuration file for the environment.
+- `--agent_config` specifies configuration file for the agent.
+- `--visualize` enables visualization. Rendering should be disabled for faster training.
+- `--log_file` specifies the output log file, which will keep track of statistics during training.
+- `--out_model_file` specifies the output model file, which contains the model parameters.
+
+## Test Models
+
+To load a trained model, run the following command:
+
+``python run.py --env_config data/envs/dm_cheetah_env.yaml --agent_config a2/dm_cheetah_cem_agent.yaml --mode test --model_file data/models/dm_cheetah_ppo_model.pt --visualize``
+
+- `--model_file` specifies the `.pt` file that contains parameters for the trained model. Pretrained models are available in `data/models/`.
+
+
+## Visualizing Training Logs
+
+During training, a tensorboard `events` file will be saved the same output directory as the log file. The log can be viewed with:
+
+``tensorboard --logdir=output/ --port=6006 --bind_all``
+
+
+The output log `.txt` file can also be plotted using the plotting script in `tools/plot_log/plot_log.py`.

a1/bc_agent.py

@@ -0,0 +1,138 @@
+import numpy as np
+import torch
+
+import learning.agent_builder as agent_builder
+import learning.base_agent as base_agent
+import learning.bc_model as bc_model
+import util.torch_util as torch_util
+
+class BCAgent(base_agent.BaseAgent):
+    NAME = "BC"
+
+    def __init__(self, config, env, device):
+        super().__init__(config, env, device)
+        return
+
+    def _load_params(self, config):
+        super()._load_params(config)
+
+        buffer_size = config["exp_buffer_size"]
+        self._exp_buffer_length = max(buffer_size, self._steps_per_iter)
+
+        self._batch_size = config["batch_size"]
+        self._update_epochs = config["update_epochs"]
+        return
+
+    def _build_model(self, config):
+        model_config = config["model"]
+        self._model = bc_model.BCModel(model_config, self._env)
+
+        self._build_expert(config)
+        self._sync_normalizers()
+        return    
+
+    def _build_expert(self, config):
+        expert_config = config["expert_config"]
+        expert = agent_builder.build_agent(expert_config, self._env, self._device)
+
+        expert_model_file = config["expert_model_file"]
+        assert(expert_model_file is not None)
+        expert.load(expert_model_file)
+        expert.set_mode(base_agent.AgentMode.TEST)
+
+        # putting the expert in a list makes the expert's parameters invisible to the pytorch module
+        self._experts = [expert]
+        return
+
+    def _sync_normalizers(self):
+        expert = self._experts[0]
+        self._obs_norm.load_state_dict(expert._obs_norm.state_dict())
+        self._a_norm.load_state_dict(expert._a_norm.state_dict())
+        return
+
+    def _get_exp_buffer_length(self):
+        return self._exp_buffer_length
+
+    def _need_normalizer_update(self):
+        return False
+
+    def _build_exp_buffer(self, config):
+        super()._build_exp_buffer(config)
+
+        expert_a_buffer = torch.zeros_like(self._exp_buffer.get_data("action"))
+        self._exp_buffer.add_buffer("expert_a", expert_a_buffer)
+        return
+
+    def _record_data_pre_step(self, obs, info, action, action_info):
+        super()._record_data_pre_step(obs, info, action, action_info)
+        self._exp_buffer.record("expert_a", action_info["expert_a"])
+        return
+
+    def _update_model(self):
+        self.train()
+
+        num_samples = self._exp_buffer.get_sample_count()
+        batch_size = self._batch_size 
+        num_batches = int(np.ceil(float(num_samples) / batch_size))
+        train_info = dict()
+
+        for i in range(self._update_epochs):
+            for b in range(num_batches):
+                batch = self._exp_buffer.sample(batch_size)
+                loss_info = self._compute_loss(batch)
+
+                self._optimizer.zero_grad()
+                loss = loss_info["loss"]
+                loss.backward()
+                self._optimizer.step()
+
+                torch_util.add_torch_dict(loss_info, train_info)
+
+        num_steps = self._update_epochs * num_batches
+        torch_util.scale_torch_dict(1.0 / num_steps, train_info)
+
+        return train_info        
+
+    def _compute_loss(self, batch):
+        norm_obs = self._obs_norm.normalize(batch["obs"])
+        norm_expert_a = self._a_norm.normalize(batch["expert_a"])
+
+        actor_loss = self._compute_actor_loss(norm_obs, norm_expert_a)
+
+        info = {
+            "loss": actor_loss
+        }
+        return info
+
+    def _eval_expert(self, obs):
+        info = None
+        expert = self._experts[0]
+        expert_a, _ = expert._decide_action(obs, info)
+        return expert_a
+
+    def _decide_action(self, obs, info):
+        '''
+        TODO 1.1: Implement code for sampling from the policy and
+        querying the expert policy for the expert actions.
+        '''
+
+        ## a) sample an action from the policy
+        # placeholder
+        a = torch.zeros([self._env.compute_act_shape()], device=self._device)
+
+        ## b) query the expert for an action
+        # placeholder
+        expert_a = torch.zeros([self._env.compute_act_shape()], device=self._device)
+
+        a_info = {
+            "expert_a": expert_a
+        }
+        return a, a_info
+
+    def _compute_actor_loss(self, norm_obs, norm_expert_a):
+        '''
+        TODO 1.2: Implement code to calculate the loss for training the policy.
+        '''
+        # placeholder
+        loss = torch.zeros(1)
+        return loss

a1/dm_cheetah_bc_agent.yaml

@@ -0,0 +1,21 @@
+agent_name: "BC"
+
+model:
+  actor_net: "fc_2layers_64units"
+  actor_init_output_scale: 0.01
+  actor_std_type: "FIXED"
+  action_std: 0.2
+
+expert_config: "data/agents/bc_expert_agent.yaml"
+expert_model_file: "data/models/dm_cheetah_expert_model.pt"
+
+discount: 0.99
+steps_per_iter: 100
+iters_per_output: 10
+test_episodes: 10
+
+learning_rate: 5e-4
+update_epochs: 20
+
+exp_buffer_size: 1000000
+batch_size: 256

a1/dm_walker_bc_agent.yaml

@@ -0,0 +1,21 @@
+agent_name: "BC"
+
+model:
+  actor_net: "fc_2layers_64units"
+  actor_init_output_scale: 0.01
+  actor_std_type: "FIXED"
+  action_std: 0.2
+
+expert_config: "data/agents/bc_expert_agent.yaml"
+expert_model_file: "data/models/dm_walker_run_expert_model.pt"
+
+discount: 0.99
+steps_per_iter: 100
+iters_per_output: 10
+test_episodes: 10
+
+learning_rate: 5e-4
+update_epochs: 20
+
+exp_buffer_size: 1000000
+batch_size: 256

data/agents/bc_expert_agent.yaml

@@ -0,0 +1,18 @@
+agent_name: "Expert"
+
+model:
+  actor_net: "fc_2layers_128units"
+  actor_init_output_scale: 0.01
+  actor_std_type: "FIXED"
+  action_std: 0.2
+
+  critic_net: "fc_2layers_128units"
+
+
+discount: 0.99
+steps_per_iter: 4096
+iters_per_output: 10
+test_episodes: 32
+
+# optimizer parameters
+learning_rate: 1e-3

data/envs/dm_cartpole.yaml

@@ -0,0 +1,4 @@
+env_name: "dm_cartpole"
+task: "swingup"
+
+time_limit: 10 # s

data/envs/dm_cheetah.yaml

@@ -0,0 +1,4 @@
+env_name: "dm_cheetah"
+task: "run"
+
+time_limit: 10 # s

data/envs/dm_hopper.yaml

@@ -0,0 +1,5 @@
+env_name: "dm_hopper"
+task: "hop"
+
+time_limit: 10
+

data/envs/dm_humanoid.yaml

@@ -0,0 +1,5 @@
+env_name: "dm_humanoid"
+task: "run"
+
+time_limit: 10
+

data/envs/dm_pointmass.yaml

@@ -0,0 +1,4 @@
+env_name: "dm_cheetah"
+task: "run"
+
+time_limit: 10 # s

data/envs/dm_reacher.yaml

@@ -0,0 +1,4 @@
+env_name: "dm_reacher"
+task: "easy"
+
+time_limit: 10 # s

data/envs/dm_walker.yaml

@@ -0,0 +1,5 @@
+env_name: "dm_walker"
+task: "run"
+
+time_limit: 10
+

envs/base_env.py

@@ -0,0 +1,57 @@
+import abc
+import enum
+import numpy as np
+
+class EnvMode(enum.Enum):
+    TRAIN = 0
+    TEST = 1
+
+class DoneFlags(enum.Enum):
+    NULL = 0
+    FAIL = 1
+    SUCC = 2
+    TIME = 3
+
+class BaseEnv(abc.ABC):
+    NAME = "base"
+
+    def __init__(self, visualize):
+        self._mode = EnvMode.TRAIN
+        self._visualize = visualize
+        self._action_space = None
+        return
+
+    @abc.abstractmethod
+    def reset(self, env_ids=None):
+        return
+
+    @abc.abstractmethod
+    def step(self, action):
+        return
+
+    def compute_obs_shape(self):
+        obs, obs_dict = self.reset()
+        obs_shape = list(obs.shape)
+        return obs_shape
+
+    def get_action_space(self):
+        return self._action_space
+
+    def set_mode(self, mode):
+        self._mode = mode
+        return
+
+    def get_num_envs(self):
+        return int(1)
+
+    def get_reward_bounds(self):
+        return (-np.inf, np.inf)
+
+    def get_reward_fail(self):
+        return 0.0
+
+    def get_reward_succ(self):
+        return 0.0
+
+    def get_visualize(self):
+        return self._visualize