Update

.ipynb_checkpoints/RL_Network_Routing_agent-checkpoint.ipynb

@@ -0,0 +1,171 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "53516aaa",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'Network_Routing_Env1'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-4-4f75e88769a3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpdb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mNetwork_Routing_Env1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'Network_Routing_Env1'"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "# import keras.backend.tensorflow_backend as backend\n",
+    "from keras.models import Sequential\n",
+    "from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Activation, Flatten\n",
+    "from keras.callbacks import TensorBoard\n",
+    "from tensorflow.keras.optimizers import Adam\n",
+    "from collections import deque\n",
+    "import tensorflow as tf\n",
+    "from collections import deque\n",
+    "import time\n",
+    "import random\n",
+    "from tqdm import tqdm\n",
+    "import os\n",
+    "import sys\n",
+    "# import PIL import Image\n",
+    "import cv2\n",
+    "import pdb\n",
+    "\n",
+    "from Network_Routing_Env1 import new_state\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f1bd0bff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ca359cdc",
+   "metadata": {},
+   "source": [
+    "### Createating class Agent for the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "e3608684",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class agent_learner:\n",
+    "    \n",
+    "    def __init__(self, df, action, gamma=0.99, stddev=0.2, learning_rate=0.0001):\n",
+    "        #initializing Q nets\n",
+    "        # Q networks\n",
+    "        Q_initializer = tf.keras.initializers.GlorotNormal()\n",
+    "        optimizer_Q = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
+    "        identity_Q = tf.keras.initializers.Identity()\n",
+    "\n",
+    "        input_Q = tf.keras.layers.Input(shape=(df), name=\"state\")\n",
+    "        x_Q = layers.Dense(64, activation=\"relu\", kernel_initializer=initializer_Q, name=\"relu_dense_Q_1\",)(input_Q)\n",
+    "        x_Q = layers.Dense(64, activation=\"relu\", kernel_initializer=initializer_Q, name=\"relu_dense_Q_2\",)(x_Q)\n",
+    "        x_Q = layers.Dense(32, activation=\"relu\", kernel_initializer=initializer_Q, name=\"relu_dense_Q_3\",)(x_Q)\n",
+    "        x_Q = layers.Dense(16, activation=\"relu\", kernel_initializer=initializer_Q, name=\"relu_dense_Q_4\",)(x_Q)\n",
+    "\n",
+    "        output_Q = layers.Dense(action, activation=\"relu\", kernel_initializer=initializer_Q, name=\"Q_value\",)(x_Q)\n",
+    "        self.Q = keras.Model(inputs=input_Q, outputs = output_Q)\n",
+    "        self.Q.compile(optimizer=optimizer_Q, loss=[\"mse\"])\n",
+    "\n",
+    "        #the target Q-network (the prime one)\n",
+    "\n",
+    "        initializer_Q_t = tf.keras.initializers.GlorotNormal()\n",
+    "        optimizer_Q_t = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
+    "        identity_Q_t = tf.keras.initializers.Identity()\n",
+    "\n",
+    "        input_Q_t = tf.keras.layers.Input(shape=(df), name=\"state\")\n",
+    "        x_Q_t = layers.Dense(64, activation=\"relu\", kernel_initializer=initializer_Q_t, name=\"relu_dense_Qt_1\",)(input_Q_t)\n",
+    "        x_Q_t = layers.Dense(64, activation=\"relu\", kernel_initializer=initializer_Q_t, name=\"relu_dense_Qt_2\",)(x_Q_t)\n",
+    "        x_Q_t = layers.Dense(32, activation=\"relu\", kernel_initializer=initializer_Q_t, name=\"relu_dense_Qt_3\",)(x_Q_t)\n",
+    "        x_Q_t = layers.Dense(16, activation=\"relu\", kernel_initializer=initializer_Q_t, name=\"relu_dense_Qt_4\",)(x_Q_t)\n",
+    "\n",
+    "        output_Q_t = layers.Dense(action, activation=\"relu\", kernel_initializer=initializer_Q, name=\"Q_value\",)(x_Q_t)\n",
+    "        self.Q_t = keras.Model(inputs=input_Q_t, outputs = output_Q_t)\n",
+    "        self.Q_t.compile(optimizer=optimizer_Q_t, loss=[\"mse\"])\n",
+    "    \n",
+    "        self.gamma = gamma\n",
+    "    \n",
+    "    \n",
+    "    \n",
+    "    \n",
+    "    def prepare_learning_materials(self, action, Env):\n",
+    "        '''\n",
+    "        Creating the y vector for learning.\n",
+    "        The y vector should be (but it is not)\n",
+    "        y(s,a) := r(s,a) + (1 - done) * gamma * Q_t(s', argmax_a'(Q(s',a'))\n",
+    "        with  Q_t -- the target Q-function\n",
+    "        \n",
+    "        Instead the output is Q(s) where Q is actually a network which return values of all actions:\n",
+    "        Q(s) returns a vector of size action\n",
+    "        Therefor the returned object is the Q(s) where only its a-th element is modified to\n",
+    "        r(s,a) + (1 - done) * gamma * Q_t(s', argmax_a'(Q(s',a'))\n",
+    "        Keyword arguments:\n",
+    "        action -- a list of action\n",
+    "        Env -- the environment\n",
+    "        returns:\n",
+    "        y vector\n",
+    "        '''\n",
+    "\n",
+    "        debug = False\n",
+    "\n",
+    "        if debug:\n",
+    "            import pdb; pdb.set_trace()\n",
+    "\n",
+    "        nr_samples = len(action)\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dbe8c06e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}