Experiment/nowcast

notebooks/draft/31_tl_vci_nowcast_experiment.ipynb

@@ -0,0 +1,395 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Nowcast Experiment (NO VCI in X DATA)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/Users/tommylees/github/ml_drought\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import os\n",
+    "import warnings\n",
+    "\n",
+    "%load_ext autoreload\n",
+    "%autoreload\n",
+    "\n",
+    "# ignore warnings for now ...\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "if Path('.').absolute().parents[1].name == 'ml_drought':\n",
+    "    os.chdir(Path('.').absolute().parents[1])\n",
+    "\n",
+    "!pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import xarray as xr\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import pickle\n",
+    "\n",
+    "data_dir = Path('data/')\n",
+    "# data_dir = Path('/Volumes/Lees_Extend/data/zip_data')\n",
+    "data_dir = Path('/Volumes/Lees_Extend/data/ecmwf_sowc/data/')\n",
+    "\n",
+    "assert data_dir.exists()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import seaborn as sns\n",
+    "from src.utils import drop_nans_and_flatten\n",
+    "\n",
+    "from src.analysis import read_train_data, read_test_data, read_pred_data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Read in the data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, y_train = read_train_data(data_dir)\n",
+    "X_test, y_test = read_test_data(data_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = xr.merge([y_train, y_test]).sortby('time').sortby('lat')\n",
+    "d_ = xr.merge([X_train, X_test]).sortby('time').sortby('lat')\n",
+    "ds = xr.merge([ds, d_])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.utils import get_ds_mask\n",
+    "mask = get_ds_mask(X_train.VCI)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "['ealstm', 'linear_network', 'linear_regression', 'previous_month', 'rnn']"
+      ],
+      "text/plain": [
+       "['ealstm', 'linear_network', 'linear_regression', 'previous_month', 'rnn']"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "[d.name for d in (data_dir / 'models' / '2020_01_13:151544_nowcast_tommy').iterdir()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Standard Experiments\n",
+    "bline_pred = read_pred_data('previous_month', data_dir)[-1].where(~mask)\n",
+    "orig_ln_pred = read_pred_data('linear_network', data_dir,)[-1].where(~mask)\n",
+    "orig_rnn_pred = read_pred_data('rnn', data_dir,)[-1].where(~mask)\n",
+    "orig_ealstm_pred = read_pred_data('ealstm', data_dir)[-1].where(~mask)\n",
+    "\n",
+    "# Predict Delta Experiments \n",
+    "ln_pred = read_pred_data('linear_network', data_dir, experiment='one_month_forecast_predict_delta')[-1].where(~mask)\n",
+    "ealstm_pred = read_pred_data('ealstm', data_dir, experiment='one_month_forecast_predict_delta')[-1].where(~mask)\n",
+    "\n",
+    "# ealstm_pred = read_pred_data('ealstm', data_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Explore how perform in each month"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.analysis import annual_scores"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "[]"
+      ],
+      "text/plain": [
+       "[]"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print((data_dir / 'models/one_month_forecast_predict_delta/rnn/').exists())\n",
+    "[d.name for d in (data_dir / 'models/one_month_forecast_predict_delta/rnn/').iterdir()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ealstm</th>\n",
+       "      <th>linear_network</th>\n",
+       "      <th>month</th>\n",
+       "      <th>year</th>\n",
+       "      <th>metric</th>\n",
+       "      <th>time</th>\n",
+       "      <th>previous_month</th>\n",
+       "      <th>orig_linear_network</th>\n",
+       "      <th>orig_ealstm</th>\n",
+       "      <th>orig_rnn</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>9.518235</td>\n",
+       "      <td>20.316086</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>2011.0</td>\n",
+       "      <td>rmse</td>\n",
+       "      <td>2011-01-01</td>\n",
+       "      <td>9.79577</td>\n",
+       "      <td>22.490418</td>\n",
+       "      <td>13.402106</td>\n",
+       "      <td>11.853916</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>9.518235</td>\n",
+       "      <td>20.316086</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>2011.0</td>\n",
+       "      <td>rmse</td>\n",
+       "      <td>2011-01-01</td>\n",
+       "      <td>9.79577</td>\n",
+       "      <td>-0.354995</td>\n",
+       "      <td>0.518841</td>\n",
+       "      <td>0.623585</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>9.518235</td>\n",
+       "      <td>20.316086</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>2011.0</td>\n",
+       "      <td>rmse</td>\n",
+       "      <td>2011-01-01</td>\n",
+       "      <td>9.79577</td>\n",
+       "      <td>22.490418</td>\n",
+       "      <td>13.402106</td>\n",
+       "      <td>11.853916</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>9.518235</td>\n",
+       "      <td>20.316086</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>2011.0</td>\n",
+       "      <td>rmse</td>\n",
+       "      <td>2011-01-01</td>\n",
+       "      <td>9.79577</td>\n",
+       "      <td>-0.354995</td>\n",
+       "      <td>0.518841</td>\n",
+       "      <td>0.623585</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>9.518235</td>\n",
+       "      <td>20.316086</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>2011.0</td>\n",
+       "      <td>rmse</td>\n",
+       "      <td>2011-01-01</td>\n",
+       "      <td>9.79577</td>\n",
+       "      <td>22.490418</td>\n",
+       "      <td>13.402106</td>\n",
+       "      <td>11.853916</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     ealstm  linear_network  month    year metric       time  previous_month  \\\n",
+       "0  9.518235       20.316086    1.0  2011.0   rmse 2011-01-01         9.79577   \n",
+       "1  9.518235       20.316086    1.0  2011.0   rmse 2011-01-01         9.79577   \n",
+       "2  9.518235       20.316086    1.0  2011.0   rmse 2011-01-01         9.79577   \n",
+       "3  9.518235       20.316086    1.0  2011.0   rmse 2011-01-01         9.79577   \n",
+       "4  9.518235       20.316086    1.0  2011.0   rmse 2011-01-01         9.79577   \n",
+       "\n",
+       "   orig_linear_network  orig_ealstm   orig_rnn  \n",
+       "0            22.490418    13.402106  11.853916  \n",
+       "1            -0.354995     0.518841   0.623585  \n",
+       "2            22.490418    13.402106  11.853916  \n",
+       "3            -0.354995     0.518841   0.623585  \n",
+       "4            22.490418    13.402106  11.853916  "
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# load the monthly scores dictionary\n",
+    "experiment = 'one_month_forecast_predict_delta'\n",
+    "monthly_scores = annual_scores(\n",
+    "    data_path=data_dir,\n",
+    "    models=['linear_network', 'ealstm'],\n",
+    "    metrics=['rmse', 'r2'],\n",
+    "    pred_years=[y for y in range(2011, 2019)],\n",
+    "    experiment=experiment,\n",
+    "    true_data_experiment='one_month_forecast',\n",
+    "    target_var='VCI',\n",
+    "    verbose=False,\n",
+    "    to_dataframe=True\n",
+    ")\n",
+    "monthly_scores['time'] = monthly_scores.apply(lambda row: pd.to_datetime(f\"{int(row.month)}-{int(row.year)}\"), axis=1)\n",
+    "\n",
+    "monthly_scores2 = annual_scores(\n",
+    "    data_path=data_dir,\n",
+    "    models=['linear_network', 'ealstm', 'rnn', 'previous_month'],\n",
+    "    metrics=['rmse', 'r2'],\n",
+    "    pred_years=[y for y in range(2011, 2019)],\n",
+    "    experiment='one_month_forecast',\n",
+    "    true_data_experiment='one_month_forecast',\n",
+    "    target_var='VCI',\n",
+    "    verbose=False,\n",
+    "    to_dataframe=True\n",
+    ")\n",
+    "monthly_scores2['time'] = monthly_scores.apply(lambda row: pd.to_datetime(f\"{int(row.month)}-{int(row.year)}\"), axis=1)\n",
+    "\n",
+    "# rename columns and to merge dataframes\n",
+    "monthly_scores2 = monthly_scores2.join(\n",
+    "    monthly_scores2[['linear_network', 'ealstm', 'rnn']].add_prefix('orig_')\n",
+    ").drop(columns=['linear_network', 'ealstm', 'rnn'])\n",
+    "\n",
+    "# merge\n",
+    "monthly_scores = pd.merge(monthly_scores, monthly_scores2)\n",
+    "monthly_scores.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}