cop_robbers_RL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8d8df1d0-78d5-4f92-82c4-0c764c22d214",
   "metadata": {},
   "source": [
    "# Cops and Robbers: Multi-Agent RL Environment\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e92841bf-c81c-489f-a3b9-c9ed117b8b8d",
   "metadata": {},
   "source": [
    "### 1) Setup and Imports\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51f1876d-7110-4da9-869c-8a79e2a05074",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gymnasium as gym\n",
    "from gymnasium import spaces\n",
    "from pettingzoo.utils.env import ParallelEnv\n",
    "import functools\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.ppo import MlpPolicy\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.env_util import make_vec_env"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc403706-44b8-443f-b3d0-2ef7b1070f70",
   "metadata": {},
   "source": [
    "### 2) Environment Implementation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6c573da0-de6b-4ae2-9d4c-e5d76a038367",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CopsAndRobbers(ParallelEnv):\n",
    "    metadata = {\"render_modes\": [\"human\"], \"name\": \"cops_and_robbers_v0\"}\n",
    "\n",
    "    def __init__(self):\n",
    "        self.possible_agents = [\"cop\", \"robber\"]\n",
    "        self.agents = self.possible_agents.copy()\n",
    "        self.grid_size = 10\n",
    "        self._action_spaces = {agent: spaces.Discrete(5) for agent in self.possible_agents}\n",
    "        self._observation_spaces = {agent: spaces.Box(low=0, high=self.grid_size - 1, shape=(4,), dtype=np.float32) for agent in self.possible_agents}\n",
    "\n",
    "    def reset(self, seed=None, options=None):\n",
    "        self.agents = self.possible_agents.copy()\n",
    "        if seed is not None:\n",
    "            np.random.seed(seed)\n",
    "        self.state = {agent: np.random.randint(self.grid_size, size=2) for agent in self.agents}\n",
    "        observations = self._get_observations()\n",
    "        return observations, {agent: {} for agent in self.agents}\n",
    "\n",
    "    def step(self, actions):\n",
    "        for agent in self.agents:\n",
    "            action = actions.get(agent, 0)  # Default to 'stay' if no action is provided\n",
    "            if action == 1: self.state[agent][1] = min(self.state[agent][1] + 1, self.grid_size - 1)\n",
    "            elif action == 2: self.state[agent][1] = max(self.state[agent][1] - 1, 0)\n",
    "            elif action == 3: self.state[agent][0] = max(self.state[agent][0] - 1, 0)\n",
    "            elif action == 4: self.state[agent][0] = min(self.state[agent][0] + 1, self.grid_size - 1)\n",
    "\n",
    "        caught = np.array_equal(self.state[\"cop\"], self.state[\"robber\"])\n",
    "        rewards = {agent: 10 if agent == \"cop\" and caught else -10 if agent == \"robber\" and caught else -1 if agent == \"cop\" else 1 for agent in self.agents}\n",
    "        terminations = {agent: caught for agent in self.agents}\n",
    "        truncations = {agent: False for agent in self.agents}\n",
    "        observations = self._get_observations()\n",
    "        return observations, rewards, terminations, truncations, {agent: {} for agent in self.agents}\n",
    "\n",
    "    def _get_observations(self):\n",
    "        return {\n",
    "            agent: np.concatenate([self.state[agent], self.state[\"robber\" if agent == \"cop\" else \"cop\"]]).astype(np.float32)\n",
    "            for agent in self.agents\n",
    "        }\n",
    "\n",
    "    def observation_space(self, agent): \n",
    "        return self._observation_spaces[agent]\n",
    "\n",
    "    def action_space(self, agent): \n",
    "        return self._action_spaces[agent]\n",
    "\n",
    "    def render(self):\n",
    "        grid = np.full((self.grid_size, self.grid_size), '.', dtype=str)\n",
    "        for agent, position in self.state.items():\n",
    "            grid[position[1], position[0]] = agent[0].upper()\n",
    "        print(\"\\n\" + \"=\" * (self.grid_size * 2 + 3))\n",
    "        for row in grid:\n",
    "            print(\"| \" + \" \".join(row) + \" |\")\n",
    "        print(\"=\" * (self.grid_size * 2 + 3))\n",
    "        print(\"\\nAgent Positions:\")\n",
    "        for agent, position in self.state.items():\n",
    "            print(f\"{agent.capitalize()}: ({position[0]}, {position[1]})\")\n",
    "        print()\n",
    "\n",
    "    def close(self): pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cb777af-51d2-4412-a568-b948091f154e",
   "metadata": {},
   "source": [
    "### 3) Wrapper for Single-Agent Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69066363-f3e4-419c-a8f4-386cc5f920b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SingleAgentWrapper(gym.Env):\n",
    "    def __init__(self, env, agent_name):\n",
    "        self.env = env\n",
    "        self.agent_name = agent_name\n",
    "        self.action_space = env.action_space(agent_name)\n",
    "        self.observation_space = env.observation_space(agent_name)\n",
    "\n",
    "    def reset(self, seed=None, options=None):\n",
    "        obs, _ = self.env.reset(seed=seed, options=options)\n",
    "        return obs[self.agent_name], {}\n",
    "\n",
    "    def step(self, action):\n",
    "        actions = {agent: self.env.action_space(agent).sample() for agent in self.env.possible_agents}\n",
    "        actions[self.agent_name] = action\n",
    "        obs, rewards, terminations, truncations, infos = self.env.step(actions)\n",
    "        return obs[self.agent_name], rewards[self.agent_name], terminations[self.agent_name], truncations[self.agent_name], infos[self.agent_name]\n",
    "\n",
    "    def render(self):\n",
    "        self.env.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "330edf0b-da7f-4ec1-9e22-06a0d3aa13d2",
   "metadata": {},
   "source": [
    "### 4) Training Function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2f87e2b3-6b30-4400-b68f-c6a095dc68f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda device\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 167      |\n",
      "|    ep_rew_mean     | -156     |\n",
      "| time/              |          |\n",
      "|    fps             | 1132     |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 7        |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 133         |\n",
      "|    ep_rew_mean          | -122        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 502         |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 32          |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010264745 |\n",
      "|    clip_fraction        | 0.118       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.6        |\n",
      "|    explained_variance   | -0.00879    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 8.89        |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.00681    |\n",
      "|    value_loss           | 41.7        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 73.5        |\n",
      "|    ep_rew_mean          | -62.5       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 429         |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 57          |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011253471 |\n",
      "|    clip_fraction        | 0.136       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.58       |\n",
      "|    explained_variance   | 0.0675      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 48.9        |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0118     |\n",
      "|    value_loss           | 81.4        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 42.4        |\n",
      "|    ep_rew_mean          | -31.4       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 440         |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 74          |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011349242 |\n",
      "|    clip_fraction        | 0.151       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.55       |\n",
      "|    explained_variance   | 0.107       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 61.9        |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0143     |\n",
      "|    value_loss           | 132         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 29.4        |\n",
      "|    ep_rew_mean          | -18.4       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 449         |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 91          |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011529678 |\n",
      "|    clip_fraction        | 0.13        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.5        |\n",
      "|    explained_variance   | 0.182       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 93.9        |\n",
      "|    n_updates            | 40          |\n",
      "|    policy_gradient_loss | -0.0136     |\n",
      "|    value_loss           | 164         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 26.7        |\n",
      "|    ep_rew_mean          | -15.7       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 431         |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 113         |\n",
      "|    total_timesteps      | 49152       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014845435 |\n",
      "|    clip_fraction        | 0.197       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.43       |\n",
      "|    explained_variance   | 0.191       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 72.9        |\n",
      "|    n_updates            | 50          |\n",
      "|    policy_gradient_loss | -0.0211     |\n",
      "|    value_loss           | 146         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 20          |\n",
      "|    ep_rew_mean          | -9.04       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 449         |\n",
      "|    iterations           | 7           |\n",
      "|    time_elapsed         | 127         |\n",
      "|    total_timesteps      | 57344       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014763801 |\n",
      "|    clip_fraction        | 0.193       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.34       |\n",
      "|    explained_variance   | 0.202       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 57.4        |\n",
      "|    n_updates            | 60          |\n",
      "|    policy_gradient_loss | -0.0185     |\n",
      "|    value_loss           | 111         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 18.8        |\n",
      "|    ep_rew_mean          | -7.82       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 464         |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 141         |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017037515 |\n",
      "|    clip_fraction        | 0.214       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.23       |\n",
      "|    explained_variance   | 0.17        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 35.1        |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.0206     |\n",
      "|    value_loss           | 83.7        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 15.5        |\n",
      "|    ep_rew_mean          | -4.5        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 475         |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 155         |\n",
      "|    total_timesteps      | 73728       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016642157 |\n",
      "|    clip_fraction        | 0.202       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.12       |\n",
      "|    explained_variance   | 0.18        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 27.1        |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | -0.0188     |\n",
      "|    value_loss           | 59.7        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 13.7        |\n",
      "|    ep_rew_mean          | -2.69       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 486         |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 168         |\n",
      "|    total_timesteps      | 81920       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.013643531 |\n",
      "|    clip_fraction        | 0.148       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1          |\n",
      "|    explained_variance   | 0.147       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 22.4        |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | -0.0128     |\n",
      "|    value_loss           | 43.3        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 10.5       |\n",
      "|    ep_rew_mean          | 0.48       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 497        |\n",
      "|    iterations           | 11         |\n",
      "|    time_elapsed         | 181        |\n",
      "|    total_timesteps      | 90112      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01033132 |\n",
      "|    clip_fraction        | 0.129      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.904     |\n",
      "|    explained_variance   | 0.21       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 21         |\n",
      "|    n_updates            | 100        |\n",
      "|    policy_gradient_loss | -0.0117    |\n",
      "|    value_loss           | 30.3       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 11.6        |\n",
      "|    ep_rew_mean          | -0.61       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 505         |\n",
      "|    iterations           | 12          |\n",
      "|    time_elapsed         | 194         |\n",
      "|    total_timesteps      | 98304       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010255857 |\n",
      "|    clip_fraction        | 0.125       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.797      |\n",
      "|    explained_variance   | 0.212       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 9.21        |\n",
      "|    n_updates            | 110         |\n",
      "|    policy_gradient_loss | -0.0116     |\n",
      "|    value_loss           | 26.1        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 11          |\n",
      "|    ep_rew_mean          | -0.04       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 512         |\n",
      "|    iterations           | 13          |\n",
      "|    time_elapsed         | 207         |\n",
      "|    total_timesteps      | 106496      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010243945 |\n",
      "|    clip_fraction        | 0.106       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.697      |\n",
      "|    explained_variance   | 0.298       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 8.23        |\n",
      "|    n_updates            | 120         |\n",
      "|    policy_gradient_loss | -0.00788    |\n",
      "|    value_loss           | 17.8        |\n",
      "-----------------------------------------\n",
      "Using cuda device\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 225      |\n",
      "|    ep_rew_mean     | 214      |\n",
      "| time/              |          |\n",
      "|    fps             | 1970     |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 4        |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 258         |\n",
      "|    ep_rew_mean          | 247         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 940         |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 17          |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009026187 |\n",
      "|    clip_fraction        | 0.091       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.6        |\n",
      "|    explained_variance   | -0.0155     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 11.5        |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.005      |\n",
      "|    value_loss           | 39.9        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 298         |\n",
      "|    ep_rew_mean          | 287         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 800         |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 30          |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010421996 |\n",
      "|    clip_fraction        | 0.0893      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.59       |\n",
      "|    explained_variance   | 0.122       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 10          |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.00754    |\n",
      "|    value_loss           | 47.8        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 384         |\n",
      "|    ep_rew_mean          | 373         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 742         |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 44          |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011521816 |\n",
      "|    clip_fraction        | 0.135       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.55       |\n",
      "|    explained_variance   | 0.143       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 1.72        |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.00555    |\n",
      "|    value_loss           | 22.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 427         |\n",
      "|    ep_rew_mean          | 416         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 718         |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 57          |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010228936 |\n",
      "|    clip_fraction        | 0.0962      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.51       |\n",
      "|    explained_variance   | 0.0725      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 19.6        |\n",
      "|    n_updates            | 40          |\n",
      "|    policy_gradient_loss | -0.0043     |\n",
      "|    value_loss           | 28.5        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 546         |\n",
      "|    ep_rew_mean          | 535         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 701         |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 70          |\n",
      "|    total_timesteps      | 49152       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008330656 |\n",
      "|    clip_fraction        | 0.0922      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.46       |\n",
      "|    explained_variance   | 0.0462      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 23.8        |\n",
      "|    n_updates            | 50          |\n",
      "|    policy_gradient_loss | -0.00238    |\n",
      "|    value_loss           | 17.6        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 631          |\n",
      "|    ep_rew_mean          | 620          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 692          |\n",
      "|    iterations           | 7            |\n",
      "|    time_elapsed         | 82           |\n",
      "|    total_timesteps      | 57344        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0085017355 |\n",
      "|    clip_fraction        | 0.0629       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.46        |\n",
      "|    explained_variance   | 0.0515       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 15.1         |\n",
      "|    n_updates            | 60           |\n",
      "|    policy_gradient_loss | -0.00317     |\n",
      "|    value_loss           | 34.4         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 675         |\n",
      "|    ep_rew_mean          | 664         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 683         |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 95          |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010236687 |\n",
      "|    clip_fraction        | 0.0653      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.42       |\n",
      "|    explained_variance   | 0.0607      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 17.7        |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.00273    |\n",
      "|    value_loss           | 36.5        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 696         |\n",
      "|    ep_rew_mean          | 685         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 676         |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 109         |\n",
      "|    total_timesteps      | 73728       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010420367 |\n",
      "|    clip_fraction        | 0.111       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.38       |\n",
      "|    explained_variance   | 0.0574      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.47        |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | -0.0033     |\n",
      "|    value_loss           | 14.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 769         |\n",
      "|    ep_rew_mean          | 758         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 668         |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 122         |\n",
      "|    total_timesteps      | 81920       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010394318 |\n",
      "|    clip_fraction        | 0.125       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.35       |\n",
      "|    explained_variance   | 0.00458     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 2.52        |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | -0.00317    |\n",
      "|    value_loss           | 24.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 804         |\n",
      "|    ep_rew_mean          | 793         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 662         |\n",
      "|    iterations           | 11          |\n",
      "|    time_elapsed         | 135         |\n",
      "|    total_timesteps      | 90112       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.010260481 |\n",
      "|    clip_fraction        | 0.11        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.38       |\n",
      "|    explained_variance   | 0.00911     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.862       |\n",
      "|    n_updates            | 100         |\n",
      "|    policy_gradient_loss | -0.000926   |\n",
      "|    value_loss           | 32.8        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 815         |\n",
      "|    ep_rew_mean          | 804         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 658         |\n",
      "|    iterations           | 12          |\n",
      "|    time_elapsed         | 149         |\n",
      "|    total_timesteps      | 98304       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007675552 |\n",
      "|    clip_fraction        | 0.0794      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.41       |\n",
      "|    explained_variance   | 0.0235      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.476       |\n",
      "|    n_updates            | 110         |\n",
      "|    policy_gradient_loss | -0.00175    |\n",
      "|    value_loss           | 36.5        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 887        |\n",
      "|    ep_rew_mean          | 876        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 655        |\n",
      "|    iterations           | 13         |\n",
      "|    time_elapsed         | 162        |\n",
      "|    total_timesteps      | 106496     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00752923 |\n",
      "|    clip_fraction        | 0.106      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.43      |\n",
      "|    explained_variance   | 0.0432     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 34.5       |\n",
      "|    n_updates            | 120        |\n",
      "|    policy_gradient_loss | -0.00309   |\n",
      "|    value_loss           | 29.5       |\n",
      "----------------------------------------\n"
     ]
    }
   ],
   "source": [
    "def make_env(agent_name):\n",
    "    def _init():\n",
    "        env = CopsAndRobbers()\n",
    "        return SingleAgentWrapper(env, agent_name)\n",
    "    return _init\n",
    "\n",
    "def train_agent(agent_name, total_timesteps=100000):\n",
    "    vec_env = make_vec_env(make_env(agent_name), n_envs=4)\n",
    "\n",
    "    model = PPO(MlpPolicy, vec_env, verbose=1)\n",
    "    model.learn(total_timesteps=total_timesteps)\n",
    "    \n",
    "    return model\n",
    "\n",
    "# Train the cop and robber agents\n",
    "cop_model = train_agent(\"cop\")\n",
    "robber_model = train_agent(\"robber\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf476959-79d6-4e0f-a495-4765b046a129",
   "metadata": {},
   "source": [
    "### 5) Testing and Simulation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a05ec678-4287-47bb-ad2c-23e3050ecaf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Environment test started.\n",
      "\n",
      "Step 1\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 2)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 1, 'robber': 1}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 2\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 3)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 1, 'robber': 4}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 3\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . C . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (5, 3)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 4, 'robber': 0}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 4\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . C . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (5, 2)\n",
      "Robber: (9, 3)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 2}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 5\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 2)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 3, 'robber': 1}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 6\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 2)\n",
      "Robber: (8, 4)\n",
      "\n",
      "Actions: {'cop': 0, 'robber': 3}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 7\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 2)\n",
      "Robber: (8, 4)\n",
      "\n",
      "Actions: {'cop': 3, 'robber': 0}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 8\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 1)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 4}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 9\n",
      "\n",
      "=======================\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 0)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 4}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 10\n",
      "\n",
      "=======================\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 0)\n",
      "Robber: (9, 4)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 4}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 11\n",
      "\n",
      "=======================\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 0)\n",
      "Robber: (8, 4)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 3}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 12\n",
      "\n",
      "=======================\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 0)\n",
      "Robber: (8, 3)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 2}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 13\n",
      "\n",
      "=======================\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 0)\n",
      "Robber: (8, 3)\n",
      "\n",
      "Actions: {'cop': 4, 'robber': 0}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 14\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 1)\n",
      "Robber: (8, 4)\n",
      "\n",
      "Actions: {'cop': 1, 'robber': 1}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 15\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 1)\n",
      "Robber: (8, 4)\n",
      "\n",
      "Actions: {'cop': 3, 'robber': 0}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 16\n",
      "\n",
      "=======================\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 0)\n",
      "Robber: (8, 5)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 1}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 17\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . C . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (3, 1)\n",
      "Robber: (9, 5)\n",
      "\n",
      "Actions: {'cop': 1, 'robber': 4}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 18\n",
      "\n",
      "=======================\n",
      "| . . . . . . . . . . |\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . R |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 1)\n",
      "Robber: (9, 6)\n",
      "\n",
      "Actions: {'cop': 4, 'robber': 1}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 19\n",
      "\n",
      "=======================\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 0)\n",
      "Robber: (8, 6)\n",
      "\n",
      "Actions: {'cop': 2, 'robber': 3}, Rewards: {'cop': -1, 'robber': 1}\n",
      "\n",
      "Step 20\n",
      "\n",
      "=======================\n",
      "| . . . . C . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . R . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "| . . . . . . . . . . |\n",
      "=======================\n",
      "\n",
      "Agent Positions:\n",
      "Cop: (4, 0)\n",
      "Robber: (8, 5)\n",
      "\n",
      "Actions: {'cop': 0, 'robber': 2}, Rewards: {'cop': -1, 'robber': 1}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def test_environment():\n",
    "    env = CopsAndRobbers()\n",
    "    \n",
    "    print(\"Environment test started.\")\n",
    "\n",
    "    obs = env.reset()\n",
    "    for step in range(20):\n",
    "        print(f\"\\nStep {step + 1}\")\n",
    "        actions = {agent: np.random.choice(5) for agent in env.agents}\n",
    "        obs, rewards, terminations, _, _ = env.step(actions)\n",
    "        env.render()\n",
    "        print(f\"Actions: {actions}, Rewards: {rewards}\")\n",
    "        if all(terminations.values()):\n",
    "            print(\"Game over!\")\n",
    "            break\n",
    "    env.close()\n",
    "\n",
    "test_environment()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eadad79c-b320-4bb1-b63b-b01c406119b9",
   "metadata": {},
   "source": [
    "# After spending hours debugging, I finally managed to get the code running. I believe I've done a fair job on this project."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc64c8f0-3200-43b7-855a-74e62e1a0a8b",
   "metadata": {},
   "source": [
    "### Name- Chirag Sindhwani\n",
    "### Dept of Electrical Engineering\n",
    "### Roll no. - 23085130"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}