Last notebook changes

Author: ansantam

img/markov.png

@@ -138,6 +138,36 @@
     "```"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "slideshow": {
+     "slide_type": "slide"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Importing the required packages\n",
+    "from time import sleep\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "import names\n",
+    "import numpy as np\n",
+    "from gymnasium.wrappers import RescaleAction\n",
+    "from IPython.display import clear_output, display\n",
+    "from stable_baselines3 import PPO\n",
+    "\n",
+    "from utils.helpers import (\n",
+    "    evaluate_ares_ea_agent,\n",
+    "    plot_ares_ea_training_history,\n",
+    "    show_video,\n",
+    ")\n",
+    "from utils.train import ARESEACheetah, make_env, read_from_yaml\n",
+    "from utils.train import train as train_ares_ea\n",
+    "from utils.utils import NotVecNormalize"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -159,17 +189,7 @@
     }
    },
    "source": [
-    "<h2 style=\"color: #b51f2a\">Formulating the RL problem</h2>\n",
-    "<h3>Refresher from the lecture</h3>\n",
-    "\n",
-    "We need to define:\n",
-    "\n",
-    "- Actions\n",
-    "- Observations\n",
-    "- Reward\n",
-    "- Environment\n",
-    "- Agent\n",
-    "  <img src=\"img/rl_problem_2.png\" style=\"width:70%; margin:auto;\"/>\n"
+    "<img src=\"img/the_rl_problem.png\" style=\"width:70%; margin:auto;\"/>\n"
    ]
   },
   {
@@ -181,7 +201,6 @@
    },
    "source": [
     "<h2 style=\"color: #b51f2a\">Formulating the RL problem</h2>\n",
-    "<h3>Refresher from the lecture</h3>\n",
     "\n",
     "We need to define:\n",
     "\n",
@@ -201,17 +220,7 @@
     }
    },
    "source": [
-    "<h2 style=\"color: #b51f2a\">Formulating the RL problem</h2>\n",
-    "<h3>Refresher from the lecture</h3>\n",
-    "\n",
-    "We need to define:\n",
-    "\n",
-    "- Actions\n",
-    "- Observations\n",
-    "- Reward\n",
-    "- Environment\n",
-    "- Agent\n",
-    "  <img src=\"img/rl_problem_2.png\" style=\"width:70%; margin:auto;\"/>\n"
+    "<img src=\"img/cumulate_reward.png\" style=\"width:70%; margin:auto;\"/>\n"
    ]
   },
   {
@@ -222,17 +231,18 @@
     }
    },
    "source": [
-    "<h2 style=\"color: #b51f2a\">Formulating the RL problem</h2>\n",
-    "<h3>Refresher from the lecture</h3>\n",
-    "\n",
-    "We need to define:\n",
-    "\n",
-    "- Actions\n",
-    "- Observations\n",
-    "- Reward\n",
-    "- Environment\n",
-    "- Agent\n",
-    "  <img src=\"img/rl_problem_2.png\" style=\"width:70%; margin:auto;\"/>\n"
+    "<img src=\"img/the_agent.png\" style=\"width:70%; margin:auto;\"/>\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "slide"
+    }
+   },
+   "source": [
+    "<img src=\"img/markov.png\" style=\"width:70%; margin:auto;\"/>\n"
    ]
   },
   {
@@ -284,7 +294,7 @@
     "\n",
     "<h3 style=\"color:#038aa1;\">Discussion</h3>\n",
     "<p style=\"color:#038aa1;\"> $\\implies$  Is the action space continuous or discrete? </p>\n",
-    "<p style=\"color:#038aa1;\"> $\\implies$  Is the problem deterministic or stochastic?</p>\n"
+    "<p style=\"color:#038aa1;\"> $\\implies$  Is the problem fully observable or partially observable?</p>\n"
    ]
   },
   {
@@ -578,36 +588,6 @@
     "You can find more information in the [paper](https://arxiv.org/abs/2401.05815) and the [code repository](https://github.com/desy-ml/cheetah).\n"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "slideshow": {
-     "slide_type": "slide"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# Importing the required packages\n",
-    "from time import sleep\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "import names\n",
-    "import numpy as np\n",
-    "from gymnasium.wrappers import RescaleAction\n",
-    "from IPython.display import clear_output, display\n",
-    "from stable_baselines3 import PPO\n",
-    "\n",
-    "from utils.helpers import (\n",
-    "    evaluate_ares_ea_agent,\n",
-    "    plot_ares_ea_training_history,\n",
-    "    show_video,\n",
-    ")\n",
-    "from utils.train import ARESEACheetah, make_env, read_from_yaml\n",
-    "from utils.train import train as train_ares_ea\n",
-    "from utils.utils import NotVecNormalize"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {