Updates on views-evaluation

examples/quickstart.ipynb

@@ -73,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -93,14 +93,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Actual data\n",
     "df_actual = pd.DataFrame(\n",
     "    {\n",
-    "        \"target\": [0, 1, 1, 2, 2, 3, 3, 4],\n",
+    "        \"lr_target\": [0, 1, 1, 2, 2, 3, 3, 4],\n",
     "        \"covariate_1\": [3, 2, 4, 5, 2, 6, 8, 5],\n",
     "    },\n",
     "    index=index,\n",
@@ -109,21 +109,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Point predictions\n",
-    "df1_point = pd.DataFrame({\"pred_target\": [1, 3, 5, 7]}, index=index_0)\n",
-    "df2_point = pd.DataFrame({\"pred_target\": [2, 4, 6, 8]}, index=index_1)\n",
+    "df1_point = pd.DataFrame({\"pred_lr_target\": [1, 3, 5, 7]}, index=index_0)\n",
+    "df2_point = pd.DataFrame({\"pred_lr_target\": [2, 4, 6, 8]}, index=index_1)\n",
     "dfs_point = [df1_point, df2_point]\n",
     "\n",
     "# Uncertainty\n",
     "df1_uncertainty = pd.DataFrame(\n",
-    "    {\"pred_target\": [[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]]}, index=index_0\n",
+    "    {\"pred_lr_target\": [[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]]}, index=index_0\n",
     ")\n",
     "df2_uncertainty = pd.DataFrame(\n",
-    "    {\"pred_target\": [[4, 6, 8], [5, 7, 9], [6, 8, 10], [7, 9, 11]]}, index=index_1\n",
+    "    {\"pred_lr_target\": [[4, 6, 8], [5, 7, 9], [6, 8, 10], [7, 9, 11]]}, index=index_1\n",
     ")\n",
     "dfs_uncertainty = [df1_uncertainty, df2_uncertainty]"
    ]
@@ -149,7 +149,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "metrics_list = ['RMSLE', 'CRPS'] # Add other metrics as needed\n",
+    "metrics_list = ['RMSLE', 'CRPS', 'MIS'] # Add other metrics as needed\n",
     "evaluation_manager = EvaluationManager(metrics_list)"
    ]
   },
@@ -162,17 +162,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Metric MIS is not a default metric, skipping...\n",
+      "Metric MIS is not a default metric, skipping...\n",
+      "Metric MIS is not a default metric, skipping...\n"
+     ]
+    }
+   ],
    "source": [
     "steps = [1, 2]\n",
-    "point_evaluation_results = evaluation_manager.evaluate(df_actual, dfs_point, target='target', steps=steps)"
+    "point_evaluation_results = evaluation_manager.evaluate(df_actual, dfs_point, target='lr_target', steps=steps)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -190,7 +200,7 @@
        " ts01  0.420849   2.0)"
       ]
      },
-     "execution_count": 36,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -208,21 +218,56 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Metric RMSLE is not a default metric, skipping...\n",
+      "Metric RMSLE is not a default metric, skipping...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
       "Metric RMSLE is not a default metric, skipping...\n",
       "Metric RMSLE is not a default metric, skipping...\n"
      ]
     }
    ],
    "source": [
-    "uncertainty_evaluation_results = evaluation_manager.evaluate(df_actual, dfs_uncertainty, target='target', steps=steps)"
+    "uncertainty_evaluation_results = evaluation_manager.evaluate(df_actual, dfs_uncertainty, target='lr_target', steps=steps)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(              CRPS     MIS\n",
+       " month100  0.555556    3.90\n",
+       " month101  2.333333   65.85\n",
+       " month102  4.111111  127.80,\n",
+       "             CRPS    MIS\n",
+       " step01  1.833333  45.85\n",
+       " step02  2.833333  85.85,\n",
+       "           CRPS    MIS\n",
+       " ts00  1.055556   23.9\n",
+       " ts01  3.611111  107.8)"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "uncertainty_evaluation_results['month'][1], uncertainty_evaluation_results['step'][1], uncertainty_evaluation_results['time_series'][1]"
    ]
   },
   {
@@ -234,18 +279,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Get the evaluation type, i.e., uncertainty or point\n",
-    "is_uncertainty = EvaluationManager.get_evaluation_type(dfs_point)\n",
-    "month_point_evaluation_results = evaluation_manager.month_wise_evaluation(df_actual, dfs_point, target='target', is_uncertainty=is_uncertainty)"
+    "actual = EvaluationManager.transform_data(\n",
+    "            EvaluationManager.convert_to_arrays(df_actual), 'lr_target'\n",
+    "        )\n",
+    "predictions = [\n",
+    "    EvaluationManager.transform_data(\n",
+    "        EvaluationManager.convert_to_arrays(pred), f\"pred_lr_target\"\n",
+    "    )\n",
+    "    for pred in dfs_point\n",
+    "]\n",
+    "is_uncertainty = EvaluationManager.get_evaluation_type(predictions)\n",
+    "month_point_evaluation_results = evaluation_manager.month_wise_evaluation(actual, predictions, target='lr_target', is_uncertainty=is_uncertainty)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -262,6 +316,13 @@
    "source": [
     "print(month_point_evaluation_results[1])"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "views_evaluation"
-version = "0.2.0"
+version = "0.4.0"
 description = ""
 authors = [
     "Xiaolong Sun <xiaolong.sun@pcr.uu.se>",