diff --git a/IBM_AIF360_Module (1).ipynb b/IBM_AIF360_Module (1).ipynb new file mode 100644 index 00000000..99fcd939 --- /dev/null +++ b/IBM_AIF360_Module (1).ipynb @@ -0,0 +1,1223 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Understanding Fairness Through Analyzing IBM's AIF360" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Welcome to a introductory module to IBM's AIF360. AIF360 is a AI Fairness package that allows users to evaluate their models for bias during preprocessing, during processing and post-processing. Furthermore, the package allows interference with the model in the three stages mentioned afore, and also serves as a tool to create fairness in your own AI or ML model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Importing Packages and Data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting aif360\n", + " Using cached https://files.pythonhosted.org/packages/54/a7/de16a858cbd70d9d7b9c79c06286d79bcc6ca58507f919c656e8c324286c/aif360-0.2.3-py3-none-any.whl\n", + "Requirement already satisfied: pandas>=0.23.3 in /opt/conda/lib/python3.7/site-packages (from aif360) (0.25.3)\n", + "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from aif360) (0.21.3)\n", + "Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from aif360) (1.3.2)\n", + "Requirement already satisfied: numpy>=1.16 in /opt/conda/lib/python3.7/site-packages (from aif360) (1.17.3)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23.3->aif360) (2.8.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23.3->aif360) (2019.3)\n", + "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->aif360) (0.14.1)\n", + "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas>=0.23.3->aif360) (1.13.0)\n", + "Installing collected packages: aif360\n", + "Successfully installed aif360-0.2.3\n" + ] + } + ], + "source": [ + "%%bash\n", + "pip install aif360\n", + "pip install eli5" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.metrics import accuracy_score\n", + "# setting random seed\n", + "np.random.seed(218)\n", + "from aif360.datasets import AdultDataset\n", + "from aif360.metrics import BinaryLabelDatasetMetric\n", + "from aif360.metrics import DatasetMetric\n", + "from aif360.algorithms.preprocessing import Reweighing\n", + "from IPython.display import Markdown, display\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### We will be using the AdultDataSet which serves as a CENSUS income dataset for adults with demographic information. The sensitive variable, or in other words, the variable that creates the biased \"priviledge\" is male." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['age', 'education-num', 'sex']\n", + "['income-per-year']\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.7/site-packages/aif360/datasets/standard_dataset.py:121: FutureWarning: outer method for ufunc is not implemented on pandas objects. Returning an ndarray, but in the future this will raise a 'NotImplementedError'. Consider explicitly converting the Series to an array with '.array' first.\n", + " priv = np.logical_or.reduce(np.equal.outer(vals, df[attr]))\n", + "/opt/conda/lib/python3.7/site-packages/aif360/datasets/standard_dataset.py:142: FutureWarning: outer method for ufunc is not implemented on pandas objects. Returning an ndarray, but in the future this will raise a 'NotImplementedError'. Consider explicitly converting the Series to an array with '.array' first.\n", + " df[label_name]))\n" + ] + } + ], + "source": [ + "single_protected = ['sex'] \n", + "single_privileged = [['Male']]\n", + "ad = AdultDataset(protected_attribute_names=single_protected,\n", + " privileged_classes = single_privileged, \n", + " categorical_features=[], \n", + " # even if the priviledged features are not specified, they are kept\n", + " features_to_keep=['age', 'education-num'])\n", + "print(ad.feature_names)\n", + "print(ad.label_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_orig_train, dataset_orig_test = ad.split([0.7], shuffle=True)\n", + "privileged_groups = [{'sex': 1}]\n", + "unprivileged_groups = [{'sex': 0}]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Explore the Data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Training Dataset shape" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(34189, 3)\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Protected attribute names" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['sex']\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Privileged and unprivileged protected attribute values" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[array([1.])] [array([0.])]\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Dataset feature names" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['age', 'education-num', 'sex']\n" + ] + } + ], + "source": [ + "display(Markdown(\"#### Training Dataset shape\"))\n", + "print(dataset_orig_train.features.shape)\n", + "display(Markdown(\"#### Protected attribute names\"))\n", + "print(dataset_orig_train.protected_attribute_names)\n", + "display(Markdown(\"#### Privileged and unprivileged protected attribute values\"))\n", + "print(dataset_orig_train.privileged_protected_attributes, \n", + " dataset_orig_train.unprivileged_protected_attributes)\n", + "display(Markdown(\"#### Dataset feature names\"))\n", + "print(dataset_orig_train.feature_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### After we assign the priviledged and under-priviledged groups, we calculate the baseline fairness." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Original training dataset" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference in mean outcomes between unprivileged and privileged groups = -0.192812\n" + ] + } + ], + "source": [ + "metric_orig_train = BinaryLabelDatasetMetric(dataset_orig_train, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + "display(Markdown(\"#### Original training dataset\"))\n", + "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % \n", + " metric_orig_train.mean_difference())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This value shows that there is a mean difference between priviledged and underpriviledged groups, and the biad towards the privildeged group." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction of Fainess Algorithm: Inprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### I will be introducing, executing and walking you through one of the inprocessing techniques that IBM AI360 has created to introduce fairness to the model. The interesting aspect of inprocess algorithm is the intervention the algorithm makes in the model, not post or pre analysis. \n", + "\n", + "#### Meta Fair Classifier takes the fairness metric, BinaryLabelDatasetMetric, as an input and optimizes the algorithm for it. It is essentially passing the fairness metric as a loss function, and trianing the model, while sacrificing from some of the accuracy, to be more fair. The example code for this introductory modelu is inspired by https://github.com/IBM/AIF360/blob/master/aif360/algorithms/inprocessing/meta_fair_classifier.py.\n", + "\n", + "#### Below we create the MetaFairClassifier class, and then create the fit and predict functions for the model. Here, we train the biased model, where algorithm has a tau of 0, motivating it to not be fair:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Import error: No module named 'tensorflow'\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "import copy\n", + "from aif360.algorithms import Transformer\n", + "from aif360.algorithms.inprocessing.celisMeta.FalseDiscovery import FalseDiscovery\n", + "from aif360.algorithms.inprocessing.celisMeta.StatisticalRate import StatisticalRate\n", + "from aif360.algorithms.inprocessing.meta_fair_classifier import MetaFairClassifier\n", + "\n", + "mfc = MetaFairClassifier(tau=0, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Make Predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_bias_test = mfc.predict(dataset_orig_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Assess Bias" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Original training dataset" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference in mean outcomes between unprivileged and privileged groups = -0.062442\n", + "We can see that the mean outcome is still biased, towards the variable as before - sex: Male.\n" + ] + } + ], + "source": [ + "metric_orig_train = BinaryLabelDatasetMetric(dataset_bias_test, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + "display(Markdown(\"#### Original training dataset\"))\n", + "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % \n", + " metric_orig_train.mean_difference())\n", + "print(\"We can see that the mean outcome is still biased, towards the variable as before - sex: Male.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Accuracy of the Unconstrained Model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score: 0.778\n" + ] + } + ], + "source": [ + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in list(dataset_bias_test.labels)]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "print(\"Accuracy Score:\", round(accuracy_score(predictions, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Now we will change the tau values in the algorithm to introduce fairness into the model, to see how the accuracy score and the fairness metric changes with it." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#########################################\n", + "Tau: 0.01\n", + "Training Accuracy: 0.6112784813829009 , Training gamma: 0.6925396834753267\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.119845\n", + "Test Accuracy Score: 0.613\n", + "#########################################\n", + "Tau: 0.23\n", + "Training Accuracy: 0.6010705197578168 , Training gamma: 0.688744107329693\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.089704\n", + "Test Accuracy Score: 0.604\n", + "#########################################\n", + "Tau: 0.46\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.68\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.90\n", + "Training Accuracy: 0.4743923484161573 , Training gamma: 0.7351333138871446\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.081712\n", + "Test Accuracy Score: 0.479\n" + ] + } + ], + "source": [ + "tau_list = np.linspace(0.01, 0.90, 5)\n", + "accuracies = []\n", + "mean_difference = []\n", + "for tau in tau_list:\n", + " print(\"#########################################\")\n", + " print(\"Tau: %.2f\" % tau)\n", + " mfc = MetaFairClassifier(tau=tau, sensitive_attr=\"sex\")\n", + " mfc.fit(dataset_orig_train)\n", + " predictions_mfc = mfc.predict(dataset_orig_test)\n", + " pred_labels = predictions_mfc.labels\n", + " predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in pred_labels]\n", + " metric_orig_train = BinaryLabelDatasetMetric(predictions_mfc, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + " mean_diff = metric_orig_train.mean_difference()\n", + " print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % mean_diff)\n", + " acc = accuracy_score(predictions, y_test)\n", + " print(\"Test Accuracy Score:\", round(acc,3))\n", + " accuracies.append(acc)\n", + " mean_difference.append(mean_diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Results and Analysis" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "BEST TAU AND IT'S PRINT OUT FROM ABOVE\n", + "'''\n", + "#########################################\n", + "Tau: 0.68\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "'''" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Graphing (with unbiased algorithm)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAEWCAYAAACTwaluAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgUZbr38e9NEgwgIARQWYMCI4skQBBxQVlU4CiKMoijIrhwdERxnNeR4zYencOlvuNRGRUGN0ZH4HWDQcUNkEHFhUVUwC2iQkB22Qkk6fv9oyqh0+mQStLV6/25rr7oWrrqTpP+5amqp54WVcUYY7yoE+sCjDGJwwLDGOOZBYYxxjMLDGOMZxYYxhjPLDCMMZ5ZYBhjPLPASAAiskhEfhWRo2JdS6IQkb1Bj4CIHAiavjzW9SUqC4w4JyLZwJmAAsOivO/0aO4vklT16NIHsA64IGjei7GuL1FZYMS/0cAnwHTgquAFIlJPRB4WkZ9FZJeIfCgi9dxlZ4jIEhHZKSLrRWSMO3+RiFwbtI0xIvJh0LSKyI0i8j3wvTvvMXcbu0VkuYicGbR+mojcISI/iMged3kbEXlCRB4Oqfd1Ebkl9AcUkaki8teQef8SkVvd57eLyAZ3+9+KyMCavZXltn+6iHzqvm8bReSR4IAUkbNEZIW7/BMR6V3bfSYFVbVHHD+AfOD3QC+gCDg2aNkTwCKgFZAGnAYcBbQF9gCXARlAFpDrvmYRcG3QNsYAHwZNK/Ae0BSo5867wt1GOvBHYBOQ6S67DfgK+A0gQI677inARqCOu14zYH9w/UH77AesB8SdbgIcAFq6210PtHSXZQMnVvM9/AkYFDLvFKC3+76d6L7P17vLWgC7gZHuzzwG2Ao0jvXvQ6wfMS/AHkf4z4Ez3JBo5k5/A/zBfV7H/VDlhHndfwGzK9mml8AYUEVdv5buF/gWuLCS9b4GznGfjwfmVbKe4Bw29HOnrwMWus87AFuAQUBGDd/HCoERZp2JwMyg/S8OWf45MCrWvxOxftghSXy7CnhXVbe50zM4fFjSDMgEfgjzujaVzPdqffCEiPxRRL52m+c7gcbu/qva1z9wWie4/74QbiV1PpGzcFpEAL8DXnSX5QO3APcCW0Rkloi0rMkPFUxEuojIWyKyWUR2A/dw+GdqCfwc8pKfcVpyKc0CI0655yJGAmeJyCYR2QT8AcgRkRxgG1CI05wOtb6S+QD7gPpB08eFWafsFmb3fMXtbi1NVPUYYBdOq6Cqff0TuNCttzMwp5L1AGYCI0SkHdAHeLWsGNUZqnoG0M6t7cEjbMerp4AVOIc3jYD7OPwzbXT3FawtsCEC+01oFhjx6yKgBOgC5LqPzsAHwGhVDQDPAv8rIi3dk4993UuvLwKDRGSkiKSLSJaI5LrbXQlcLCL1RaQDcE0VdTQEinGO4dNF5B6gUdDyp4H7RaSjOLqLSBaAqhYAS3FaFq+q6oHKdqKqn7v7eBp4R1V3AojIb0RkgPtzFeIchpVU/fZVqSGwS1X3ikhXnMOQUnOBHiIywn3/RuMExtsR2G9ii/UxkT3CP3B+OR8OM38kzknHdKAe8CjOX75dwGIOn6g8E/gU5+TdeuAqd34z4F2ck6If4TT1Q89hdAiaTgOecbfzC/Angs4JuMvvAn50t7kUaB30+ivcbfb38DPf7a7726B53YHP3G3vAN7g8AnQy4HVHrZbVm/QvIHAd8BenPM6k4D5Qcv745y32OXu/9RY/07Ew6P0rLQxvhCRfjiHJtnqtIpMArNDEuMbEckAJgBPW1gkBwsM4wsR6QzsBI7HOWwyScAOSYwxnlkLwxjjWcLdXNSsWTPNzs6OdRnGJLXly5dvU9XmofMTLjCys7NZtmxZrMswJqmJSGhPV8AOSYwx1WCBYYzxzALDGONZwp3DCKeoqIiCggIKCwtjXYrxQWZmJq1btyYjIyPWpaS8pAiMgoICGjZsSHZ2NiJS9QtMwlBVtm/fTkFBAe3bt491OSnPt0MSEXlWRLaIyKpKlouITBaRfBH5UkR61nRfhYWFZGVlWVgkIREhKyvLWo9xws9zGNOBwUdYPgTo6D7GAVNqszMLi+Rl/7fxw7fAUNXFOLcjV+ZC4Hl1fAIcIyLH+1WPMakqEFAmTpzP2rW/1npbsbxK0oryQ8EVUMkQaCIyTkSWiciyrVu3RqW4mpg9ezYiwjfffBPrUmLif/7nf8jNzSU3N5e0tLSy55MnT451aSkrEFCuvXYuDz74EWefPb3WoRHLwAjXzgx7J5yqTlPVPFXNa968Qm/VuDFz5kzOOOMMZs2a5et+SkoiMeBU5N15552sXLmSlStXUq9evbLnN998c6xLS0mlYfHccysBWL9+N9OmLa/VNmMZGAU4A8iWao0zlmJC2rt3Lx999BHPPPNMhcB46KGHOPnkk8nJyWHixIkA5OfnM2jQIHJycujZsyc//PADixYt4vzzzy973fjx45k+fTrgdIm/7777OOOMM3j55Zd56qmn6N27Nzk5OVxyySXs378fgM2bNzN8+HBycnLIyclhyZIl3H333Tz22GNl273zzjsr/NW//fbbefLJJ8um7733Xh5++GF++eUX+vXrR25uLt26deODDz6o0fvzr3/9iz59+tCjRw/OPfdctmzZAsC2bdsYNmwY3bt357TTTmPVqrDnyE01hYYFwNixuUyaVLuvdInlZdW5wHgRmYUz6OsuVf2lthvNnvhmrQurzE8P/Eely+bMmcPgwYPp1KkTTZs2ZcWKFfTs2ZO33nqLOXPm8Omnn1K/fn127HBO61x++eVMnDiR4cOHU1hYSCAQYP369ZVuH5z+CB9+6Hzn0Pbt27nuOmcYyrvuuotnnnmGm266iZtvvpmzzjqL2bNnU1JSwt69e2nZsiUXX3wxEyZMIBAIMGvWLD777LNy2x41ahS33HILv//97wF46aWXePvtt5kxYwbnnXced955JyUlJWXBVF39+vVj2LBhiAhTp07l4Ycf5sEHH+Tuu++mT58+zJ07l3fffZcxY8bYvUK1VFlYPP30MOrUqd0JZN8CQ0RmAmcDzUSkAPgzzpfqoKpTgXnAUJwvkNkPjPWrlmiYOXMmt9zifKnXqFGjmDlzJj179mT+/PmMHTuW+vWdgbqbNm3Knj172LBhA8OHDwecIPDi0ksvLXu+atUq7rrrLnbu3MnevXs577zzAFi4cCHPP/88AGlpaTRu3JjGjRuTlZXF559/zubNm+nRowdZWVnltt2jRw+2bNnCxo0b2bp1K02aNKFt27b07t2bq6++mqKiIi666CJyc3OpiXXr1jFy5Eg2bdrEwYMH6dSpEwAffvghb77phPy5557LmDFj2LdvHw0aNKjRflKdn2EBPgaGql5WxXIFbvRr/9G0fft2Fi5cyKpVqxARSkpKEBEeeughZ+DUkMuClQ1alJ6eTiBweCS70L4HwR+iMWPGMGfOHHJycpg+fTqLFi06Yo3XXnst06dPZ9OmTVx99dVh1xkxYgSvvPIKmzZtYtSoUYDTMli8eDFvvvkmV155JbfddhujR48+4r7CufHGG7njjjsYOnQo8+fP54EHHgAqvhc2oFPt/OlP7/kWFkDijRreq1cvDbVmzZoK86Jp6tSpOm7cuHLz+vXrp4sXL9a33npL+/btq/v27VNV1e3bt6uqap8+fXT27NmqqlpYWKj79u3TdevWabt27bSwsFB37typ2dnZ+txzz6mqart27XTr1q1l28/KytLNmzfroUOHdNCgQXrVVVepquqll16qjzzyiKqqFhcX665du1RV9eDBg9qpUydt3769FhcXh/05Vq1apX379tWOHTvqxo0bVVX1p59+0qKiIlVVfeSRR3TChAme3pMGDRqUm+7WrZuuXLlSA4GAXnHFFTpw4EBVVb3hhht00qRJqqr63nvvaV5eXtjtxfr/OFGsWrVZW7T4vwr36tixc7SkJFCj7QDLNMznz24+i4CZM2eWHV6UuuSSS5gxYwaDBw9m2LBh5OXlkZuby1//6nzn8AsvvMDkyZPLTvZt2rSJNm3aMHLkSLp3787ll19Ojx49Kt3n/fffT58+fTjnnHM46aSTyuY/9thjvP/++5x88sn06tWL1atXA1C3bl369+/PyJEjSUtLC7vNrl27smfPHlq1asXxxztdYhYtWkRubi49evTg1VdfZcKECYDTYqnOuYZ7772X4cOHc9ZZZ3HssceWzb/vvvtYsmQJ3bt355577uG5557zvE1TUdeuLVi4cDR//GPfyLYsXAk3pmdeXp6G/qJ+/fXXdO7cOUYVJYZAIEDPnj15+eWX6dixY6zLqTb7P44uEVmuqnmh862FkQLWrFlDhw4dGDhwYEKGhQkvEFAmTHiLzz6L3jc4JsXdqubIunTpwtq1a2Ndhomg4Ksh//jHF7z77pWccor/3xWdNC2MRDu0Mt7Z/215oZdOd+06yIsvfhmVfSdFYGRmZrJ9+3b7xUpC6o6H4bWvSrKrrJ/FI48c6cbwyEmKQ5LWrVtTUFBAPN+YZmqudMStVOd3pywvkiIwMjIybDQmk9TiISwgSQ5JjElm8RIWYIFhTFyLp7AACwxj4poING58VNl0LMMCkuQchjHJSkT43/917kTetetgTMMCLDCMiXuloaFKTMMC7JDEmLgSCCjPPLOCkpJAufkiEvOwAAsMY+JG6QnOa699ndGj51QIjXhggWFMHAi9GjJjxldMn76yildFnwWGMTFW2aXTsWMrHw8lViwwjImheOtnURULDGNiJNHCAiwwjImJRAwLsMAwJuoSNSzAAsOYqPvllz28/XZ+2XSihAVYYBgTda1aNWLRojEcf/zRCRUWYF3DjYmJTp2yWLr0Oo4/vmHChAVYC8MY3wUCynffba8wv1WrRgkVFmCBYYyvSk9w5uVNY8mSI3/ZdiKwwDDGJ8FXQ/bsOcTgwf8M29JIJBYYxvgg3KXTESO60KFD0xhWVXsWGMZEWCL3s6iKBYYxEZTMYQE+B4aIDBaRb0UkX0QmhlneWEReF5EvRGS1iIz1sx5j/JTsYQE+BoaIpAFPAEOALsBlItIlZLUbgTWqmgOcDTwsInX9qskYv6RCWIC/LYxTgHxVXauqh4BZwIUh6yjQUEQEOBrYART7WJMxvpg9++ukDwvwNzBaAcEXngvcecEeBzoDG4GvgAmqWmFcMhEZJyLLRGSZfR2iiUcXX9yZiRNPB5I3LMDfruHh3q3Qb0s+D1gJDABOBN4TkQ9UdXe5F6lOA6YB5OXl2Tcum7gjIkyaNJC8vJYMH945KcMC/G1hFABtgqZb47Qkgo0FXlNHPvAjcJKPNRkTEYGAUlRUUm6eiHDJJV2SNizA38BYCnQUkfbuicxRwNyQddYBAwFE5FjgN8BaH2syptZKT3D+7nevVQiNZOfbIYmqFovIeOAdIA14VlVXi8j17vKpwP3AdBH5CucQ5nZV3eZXTcbUVrirITNmXExGRloMq4oeX29vV9V5wLyQeVODnm8EzvWzBmMiJVxYNGxYl7S01On/mDo/qTG1kCr9LKpigWFMFSwsDrPAMOYILCzKs8AwphIWFhVZYBhTiTvuWGBhEcICw5hKjBvXizZtGgEWFqVs1HBjKnHCCU1YtGgM06YtZ9KkgSkfFmCBYcwRnXBCEx54YFCsy4gbdkhiDM4JzvHj5/Hvf/8U61LimgWGSXmlV0OeeGIpQ4fOsNA4AgsMk9JCL53u31/Eyy+viXFV8csCw6SsyvpZTJ48JIZVxTcLDJOSrFNWzVhgmJRjYVFzFhgmpVhY1I4FhkkZqhYWtWWBYVJKaVdvsLCoCevpaVKGiHDvvWcDsH79bguLGrDAMCklODSc788y1WGHJCZpBQLKE098xqFDFb8OwMKiZiwwTFIqvRoyfvxb/Pa3L1cIDVMzFhgm6YReOp0791ueemp5jKtKDhYYJqlU1s/ihht6x7Cq5GGBYZKGdcrynwWGSQoWFtFhgWESnoVF9FhgmIRmYRFdFhgmoW3btp/33/+pbNrCwl8WGCahtWjRgEWLriI7+xgLiyhI2q7h76zexFOL11Kvbhr1MtKoXzeNenXTg567/2aUPk+nft00Mt3lpes466eTZr+Ecatdu2P49NNradasvoWFz3wNDBEZDDwGpAFPq+oDYdY5G3gUyAC2qepZkdj3hl8PsOznXyOxKQDqptcJCZg06mekk1k3jfpuyAQ/d8KpDvXrplcZTvUy0uwX3aNAQFmzZivdurUoN79FiwYxqii1+BYYIpIGPAGcAxQAS0VkrqquCVrnGOBJYLCqrhORFuG3Vn0HiiLbFfhQcYBDxQF2UhTR7ZbKzKhDvYw0GhyVTqPMDBrVK/03o2y6YWYGjTLTy80rXafhUelJHzqlJzhnzlzFG29cxsCBJ8S6pJTjZwvjFCBfVdcCiMgs4EIgeEjm3wGvqeo6AFXdEqmdX9KzNb2zm3KgqIQDh4rZf6iE/YdKKCwqKXt+4FAxB4pKn7v/FrnPi4o5UDq/qATVSFUWXmFRgMKiAL/uLwIO1GgbDY9ywqRhuFCpJGgaZTrrN8xMJz0tfk9phV4NOf/8mSxdel2FlobxV5WBISLjgRdVtbrt+1bA+qDpAqBPyDqdgAwRWQQ0BB5T1efD1DAOGAfQtm1bTzs/rnEmxzXOrGbJ4akqB4sDbtAUh4ROSVDoFJcPnXIhVXHZgaLS7QUiUueeg8XsOVhc49c3qJtWIVTChk9Q0GRmpOH3jZ+BgHLnre/y2qzVZfOGXvQb0rMy+W7znmpt69hGmTSulxHpElOGlxbGcTiHEyuAZ4F3VD39vQ33axT6unSgFzAQqAd8LCKfqOp35V6kOg2YBpCXl+fz3/qKRITMDOecQ9MGdSO+/UBAKSx2QmTfwWJ2Hyhmd2ERuw8UsbuwiD2Fxe7z4rJ5peuULqtNUJTad6iEfYdK+GVXYQR+qshQVba/9RP7vtpeNq/ByVksa5vJ4Mc+qPb20usII3q15sb+HWjTtH4kS00JVQaGqt4lIncD5wJjgcdF5CXgGVX94QgvLQDaBE23BjaGWWebqu4D9onIYiAH+I4UUqeOuCdC02l29FE12kZJQNlb6ITILk9BU1QumPYcLPb9sKu6KguLrCHZNR7PojigzFq6nleWF3Bxz1aM79+RtlkWHF55Ooehqioim4BNQDHQBHhFRN5T1T9V8rKlQEcRaQ9sAEbhnLMI9i+cAEoH6uIcsjxS/R/DpNURGtfPoHH9jHIp7VUgoOw95LZWPAbNrgNFHCqOzOFUKA0oa176tlxYHN/7OLqM/A1Sw5O7B4sDrNuxH3CC46VlBby6YgMX92jF+AEdaJdlV1qq4uUcxs3AVcA24GngNlUtEpE6wPdA2MBQ1WL3/Mc7OJdVn1XV1SJyvbt8qqp+LSJvA18CAZxLr6si8YOZ6qlTR9yTo7E/vi89wblg6aayeZHqlPXxD9t5dP53fPrjDsBpmb28vIDXPt/ARblOcLRvZsFRGanqdISI3Idz+PFzmGWdVfVrv4oLJy8vT5ctWxbNXZooe/PN7zj//Jll03704Pxk7XYem/89H6/dXm5+HaEsOE5ofnTE9pdoRGS5quZVmO8hME4FVqvqHne6IdBFVT/1pdIqWGCkhkmTPuDOOxf63t37sx938NiC7/gov2JwDMtpyfgBHenQIvWCozaB8TnQs/TKiHsoskxVe/pSaRUsMFLHm29+x5AhHaPSIW3ZTzt4bMH3fPD9tnLzxQ2OmwZ0oEOLhr7XES9qExgrVTU3ZN6Xqto9wjV6YoGRfAIBpaiohKOOiv2tTct/3sGj88MHx/ndW3LzgA50PDb5g6OywPDStW+tiNwsIhnuYwKwNvIlmlRUeoJz+PD/R2Fh7fuS1Favdk154Zo+vPb70zirU/Oy+arw+hcbOffRxdw4YwXfbqpeh7Fk4aWF0QKYDAzA6Xi1ALglkt24q8NaGMkjtLv3kCEdmD370rhoaZT6fN2vTF7wPe9/u7XCsqEnH8fNAzty0nGNwrwysVXWwvDScWsLTh8KYyIm3EhZxx13NBkZaTGsqqIebZvw3NhT+GL9TiYv+J4F3xz+Oznvq03M+2oTQ7o5wdH5+OQLjlBeWhiZwDVAV6Ds5gxVvdrf0sKzFkbiS+Rh9b4scIJj/tcVG9jndT2Wmwd2pGvLxjGoLLJqcw7jBZz7Sc4D/o3TxTs1D+BMrSVyWAB0b30MT1/VmzduOoNzuhxbbtk7qzfzH5M/5Lrnl7Fqw64YVegvT5dVVbVH6ZUREcnAuQFtQHRKLM9aGIkr0cMinNUbdzF5wfe8s3pzhWWDOrdgwsBOnNw68VoctWlhlI4Ys1NEugGNgewI1mZSQDKGBUDXlo35+5V5zLv5TAZ3Pa7csvlfb+GCxz/kmulL+bJgZ4wqjCwvgTFNRJoAdwFzcQbAedDXqkzS+fOf30+6sAjWpWUjpl7Zi7cmnMnQk8sHx4JvtjDs8Y8Y+9xnrFyf2MFxxEMSt1fnCFV9KXolHZkdkiSmDRt2c/bZ/yA/f0fShUU4327aw+SF3zPvq18qDBtwVqfmTBjUkZ5tm8SmOA9q09Nzsar2862yarLASFwbNuxm6tRl/Pd/90/qsAj23eY9/G1hPm98ubFCcJzZsRm3DOpIr3ZNY1PcEdQmMO7GGWTy/wH7Suer6o5IF+mFBYZJRN+7wfF6JcExYWBH8rLjJzhqExg/hpmtqhqTIZstMOJfIKDceOObDBv2G4YM6RjrcuJK/pa9PL7we+Z+sZFAyEfv9A5ZTBjYiVPaxz44ahwY8cYCI74FXw2pWzeNOXMutdAI44ete3liYT5zVm6oEBx9T8hiwqCOnHpCVmyKo3YtjNHh5ocb3TsaLDDiV7hLp9df34spU86PYVXx7cdt+3jcDY6SkOTo074ptwzqRN8Tox8ctQmMvwVNZuKM8L1CVUdEtkRvLDDiU7L2s4iWn7bt4/H385n9ecXgGNT5WJ6+qsJn11e1ufnsppANNcbpLm4MYGERCdnNGvDX3+Zw04AOPPF+Pq+uOBwcXVvGz01tNfmqq/2AHZQawMIi0tplNeChETks+j9nM6p3G5o2qMvVp7ePdVllvIwa/jqHv4CoDtAFiJuOXCZ2LCz806ZpfR64pDv3XFBM/brxMz6Il0r+GvS8GPhZVQt8qsckCFXluussLPwWT2EB3gJjHfCLqhYCiEg9EclW1Z98rczENRGha9fDX4RsYZEavATGy8BpQdMl7rzevlRkEsatt/YFYPXqLTz1lIVFKvASGOmqeqh0QlUPiUjkv5HYJKRbb+2Lqtb4u05NYvFylWSriAwrnRCRC3G+NtGkkEBAeeSRj9m/v6jCMguL1OElMK4H7hCRdSKyDrgd+E9/yzLxpPRqyK23vsuwYTPDhoZJDVUGhqr+oKqn4lxO7aqqp6lqvv+lmXgQeul0wYIf+fvfradtqqoyMERkkogco6p7VXWPiDQRkb9EozgTW5X1s5gw4dQYVmViycshyRBVLRtXTFV/BYb6V5KJB9Ypy4TjJTDSROSo0gkRqQccdYT1TYKzsDCV8RIY/wQWiMg1InIN8B7wDy8bF5HBIvKtiOSLyMQjrNdbREpEJCZ3wJrDLCzMkXi5W/UhEfkSGAQI8DbQrqrXiUga8ARwDlAALBWRuaq6Jsx6DwLvVL98E0kWFqYqXu9W3QQEgEtwxsP42sNrTgHyVXWt2/FrFnBhmPVuAl4FYvLlzuawnTsL+fjjw7cJWViYUJW2MESkE86XMF8GbMcZBFhUtb/HbbcC1gdNFwB9QvbRChiO883wlXY1F5FxwDiAtm3bety9qa6mTeuxcOFoBgx4nr59W1tYmAqOdEjyDfABcEFpvwsR+UM1th3uNy10eK9HgdtVteRIvQVVdRowDZwRt6pRg6mm449vyEcfXc0xx2RaWJgKjnRIcgnOocj7IvKUiAwkfAhUpgBoEzTdGtgYsk4eMEtEfgJGAE+KyEXV2IephUBAWbHilwrzmzatZ2Fhwqo0MFR1tqpeCpwELAL+ABwrIlNE5FwP214KdBSR9u7NaqNwvmoxeB/tVTVbVbOBV4Dfq+qcmv0opjpKT3CeeurTzJ37bazLMQnCS9fwfar6oqqej9NKWAlUeok06HXFwHicqx9fAy+p6moRuV5Erq9l3aYWgq+GFBUFGDHipbAtDWNCVWs4H/fbzv7uPrysPw+YFzJvaiXrjqlOLaZmwl06veKK7uTmHneEVxnjqMkgwCZBWT8LU1sWGCnCwsJEggVGCrCwMJFigZHkLCxMJFlgJLmFC3+0sDARY4GR5AYNOoG//W0IYGFhai++viXF+GL8+FM46aRmDBjQ3sLC1Iq1MJJMIKBhB+kdNOgECwtTaxYYSaT0BOeQIS+yb9+hql9gTDVZYCSJ4Kshixf/zNChM+zrAEzEWWAkgXCXTk88sQmZmXaKykSWBUaCs34WJposMBKYhYWJNguMBGVhYWLBAiMBWViYWLHASEB/+ctiCwsTExYYCeiGG/Lo1q0FYGFhosuuuyWg5s0bsHDhaKZMWcZdd/WzsDBRY4GRoJo3b8A995wV6zJMirFDkjgXCCj/+Z+v88ora6pe2RifWWDEsdKrIdOmrWDUqFcsNEzMWWDEqdBLpyUlynvv/RDjqkyqs8CIQ5X1s5gy5fwYVmWMBUbcsU5ZJp5ZYMQRCwsT7yww4oSFhUkEFhhxQFUZN+51CwsT9yww4oCI0KdPq7JpCwsTr6ynZ5y47rpeAHz66QamTbvAwsLEJVHVWNdQLXl5ebps2bJYl+EbVUXEwsLElogsV9W80Pm+HpKIyGAR+VZE8kVkYpjll4vIl+5jiYjk+FlPvAgElAcf/JDduw9WWGZhYeKZb4EhImnAE8AQoAtwmYh0CVntR+AsVe0O3A9M86ueeFF6NWTixAUMHvzPsKFhTLzys4VxCpCvqmtV9RAwC7gweAVVXaKqv7qTnwCtfawn5kIvnX78cQFTpiyNcVXGeOdnYLQC1gdNF7jzKnMN8Fa4BSIyTkSWiciyrVu3RrDE6Kmsn8Vtt50ew6qMqR4/AyPcwXjYM8YpPv0AAAfSSURBVKwi0h8nMG4Pt1xVp6lqnqrmNW/ePIIlRod1yjLJws/LqgVAm6Dp1sDG0JVEpDvwNDBEVbf7WE9MWFiYZOJnC2Mp0FFE2otIXWAUMDd4BRFpC7wGXKmq3/lYS0xYWJhk41sLQ1WLRWQ88A6QBjyrqqtF5Hp3+VTgHiALeNK9nFgc7tpvIrKwMMnI156eqjoPmBcyb2rQ82uBa/2sIVb27j3EypWbyqYtLEwysK7hPmnU6Cjmzx/NoEHPk5t7nIWFSQoWGD5q2rQeixaN4eij61pYmKRgd6tGSCCgLFmyvsL8Ro2OsrAwScMCIwJKT3CeeeZzvPjil7EuxxjfWGDUUvDVkEBAGT16TtiWhjHJwAKjFsJdOr3qqhxOPTWpb4kxKcwCo4asn4VJRRYYNWBhYVKVBUY1WViYVGaBUQ0WFibVWWBUw0cfrWP6dAsLk7osMKrhzDPb8eyzFyJiYWFSk3UNr6YxY3I58cQmnH56WwsLk3KshXEEgYCyZ0/FQXrPPLOdhYVJSRYYlSg9wTlw4PPs3FkY63KMiQsWGGEEXw1ZunQj5577QtiWhjGpxgIjRLhLp926taBBg7oxrMqY+GCBEcT6WRhzZBYYLgsLY6pmgYGFhTFepXxgWFgY413KB8aDD35oYWGMRykfGDfc0JvevVsCFhbGVCXlu4Yfc0wm7757JVOmLOX228+wsDDmCFI+MMAJjf/6rzNjXYYxcS+lDkkCAeW66+aWu0XdGONdyrQwgq+GPPPM54Bz56kxxruUaGGEXjpVdQbDMcZUT9IHRmX9LP7+9wtiWJUxiSmpA8M6ZRkTWUkbGBYWxkSer4EhIoNF5FsRyReRiWGWi4hMdpd/KSI9I7FfCwtj/OFbYIhIGvAEMAToAlwmIl1CVhsCdHQf44Apkdj39de/YWFhjA/8bGGcAuSr6lpVPQTMAi4MWedC4Hl1fAIcIyLH13bH/ftnl4WDhYUxkeNnP4xWQPDXmBcAfTys0wr4JXglERmH0wKhbdu2Ve74sstOBuD9939i6tTzLSyMiRA/WxjhPqVag3VQ1Wmqmqeqec2bN/e088suO5lp0y6wsDAmgvwMjAKgTdB0a2BjDdYxxsQJPwNjKdBRRNqLSF1gFDA3ZJ25wGj3asmpwC5V/SV0Q8aY+ODbOQxVLRaR8cA7QBrwrKquFpHr3eVTgXnAUCAf2A+M9aseY0zt+XrzmarOwwmF4HlTg54rcKOfNRhjIidpe3oaYyLPAsMY45kFhjHGMwsMY4xn4px3TBwishX42cOqzYBtPpdTG/FcXzzXBvFdXzzXBt7ra6eqFXpJJlxgeCUiy1Q1L9Z1VCae64vn2iC+64vn2qD29dkhiTHGMwsMY4xnyRwY02JdQBXiub54rg3iu754rg1qWV/SnsMwxkReMrcwjDERZoFhjPEs4QMjVgMNR6i2y92avhSRJSKSE63avNQXtF5vESkRkRHxVJuInC0iK0VktYj8O1q1ealPRBqLyOsi8oVbX9TuxBaRZ0Vki4isqmR5zT8TqpqwD5zb5n8ATgDqAl8AXULWGQq8hTO616nAp3FU22lAE/f5kGjV5rW+oPUW4tx1PCJeagOOAdYAbd3pFvH03gF3AA+6z5sDO4C6UaqvH9ATWFXJ8hp/JhK9hRGzgYYjUZuqLlHVX93JT3BGHIsWL+8dwE3Aq8CWOKvtd8BrqroOQFXjrT4FGoqIAEfjBEZxNIpT1cXu/ipT489EogdGZYMIV3cdP1R3v9fgpH60VFmfiLQChgNTiS4v710noImILBKR5SIyOmrVeavvcaAzzpCTXwETVDUQnfKqVOPPRKJ/e3vEBhr2gef9ikh/nMA4w9eKQnYbZl5ofY8Ct6tqifOHMmq81JYO9AIGAvWAj0XkE1X9zu/i8FbfecBKYABwIvCeiHygqrv9Ls6DGn8mEj0w4nmgYU/7FZHuwNPAEFXdHoW6SnmpLw+Y5YZFM2CoiBSr6pw4qK0A2Kaq+4B9IrIYyAGiERhe6hsLPKDOSYN8EfkROAn4LAr1VaXmn4lonSjy6eROOrAWaM/hk09dQ9b5D8qf4PksjmprizOe6Wnx+N6FrD+d6J309PLedQYWuOvWB1YB3eKovinAve7zY4ENQLMo/v9mU/lJzxp/JhK6haFxPNCwx9ruAbKAJ92/4sUapTsdPdYXE15qU9WvReRt4EsgADytqmEvI8aiPuB+YLqIfIXzwbxdVaNy27uIzATOBpqJSAHwZyAjqLYafyasa7gxxrNEv0pijIkiCwxjjGcWGMYYzywwjDGeWWAYYzxL6MuqJvZEJAunPwTAcUAJsNWdPkWdey1MkrDLqiZiROReYK+q/jXWtRh/2CGJ8Y07HsRydzyIa4PmXyEiX4nIKhGZFMsaTfXYIYnx01WqukNE6gPLRORVoAHwF5z7VHYB80XkfFV9I5aFGm+shWH89AcR+QL4GOcGpxOBPsBCVd2mqkXADJwBX0wCsBaG8YWIDMIJglNV9YCIfAhkEv7WapMgrIVh/NIY2OGGRVegtzv/E6C/iGSJSDowCojqeJym5qyFYfzyJjDOPST5BvgUQFULROQeYBFOa+N1VX0zZlWaarHLqsYYz+yQxBjjmQWGMcYzCwxjjGcWGMYYzywwjDGeWWAYYzyzwDDGePb/Adi09yrwRv5gAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(tau_list, accuracies, lw=3, label=\"Accuracy vs. Tao\")\n", + "axe.set_xlabel(\"Tao\")\n", + "axe.set_ylabel(\"Accuracy\")\n", + "axe.set_title(\"Accuracy vs. Tao\")\n", + "axe.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(tau_list, mean_difference, lw=3,color='orange', label=\"Mean Bias Difference vs. Tao\")\n", + "axe.set_xlabel(\"Tao\")\n", + "axe.set_ylabel(\"Mean Bias Difference\")\n", + "axe.set_title(\"Mean Bias Difference vs. Tao\")\n", + "axe.plot([0, 1], [0, 0.1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEWCAYAAABPON1ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUZfb48c8hoUsLIEgHRXqCEFARBQURERG7iKuoi7q2XbEAispafpZ1V3R1l0XhixUVXRRd1AiIiiBNqoAEkA4CCYTekvP7494kM2EymZTJ3Jmc9+uVF3PbzJlLcuaZ5577PKKqGGOM8a5ykQ7AGGNMcJaojTHG4yxRG2OMx1miNsYYj7NEbYwxHmeJ2hhjPM4StSkxInJARFpEOo5YFKlzKyJjReRxn+U/icjvbjy1ReQ8EUl1lweWdnxlhVgdtfeIyCwgCaivqkcjHE7MEZENQAOggaru9lm/BOe8N1fVDaUYT09gJnDIXZUBvKmqT4b5dTcA9YATQCawEngbGKeqWQH2Lw/sA85R1aXuuhnAVFV9JZyxlnXWovYYEWkGnA8oMKCUXzu+NF8vwn4DBmUviEgHoHLkwmGbqp6iqqcA3YHbS6mFermqVgOaAs8Dw4Hx+exbD6gE/OKzrmme5ZCVsd+3YrFE7T03Az8BE4FbfDeISGUR+buIbBSRDBGZLSKV3W3dRWSOiOwVkc0iMsRdP0tE/ujzHENEZLbPsorIPSKSCqS6615xn2OfiCwSkfN99o8TkUdFZJ2I7He3N/Z5rjPcxxVF5CUR2eR+VR7rE2sdEfnCjTVdRH4QkZN+F91jXsqz7jMRGeY+Hi4iW904fhWRXoU4z++45zrbLTitSd/XCvYearnvYZeI7HEfN/I5dpaIPC0iP7rxpYhInVACU9XfgDlAW5/n8z23l4nIYvf/Z7OIjPbZr5KIvCsiae75XSAi9UJ4zQxVnQpcD9wiIu3d55soIs+IyJnAr+7ue0VkpoisA1oAn7tdHxVFpIaIjBeR7e7/zTMiEuc+1xD3fLwsIunAaHf9bSKyyj2PX4tI0zzv+y63e2WPiLwuIuKzfah77H4RWSkindz1DUTkE/f/5zcRuT+Uc+9Zqmo/HvoB1gJ3A52B40A9n22vA7OAhkAc0A2oCDQB9uO0EMsDtYGO7jGzgD/6PMcQYLbPsgLfAAlAZXfdTe5zxAMPAjuASu62h4HlQCtAcLoKavs81xnu4zHAVPd5qwGfA8+5254Dxrqxlsf5BiEBzsUFwObsbUAt4DBOt0Urd1sDd1sz4PQQz/EGoDdO4mnjnsvNOK1DBZqF8B5qA1cDVdxtk4FPfV5jFrAOOBOnpT4LeD6feHoCW3yWWwJbgYvy/D+d4bN/B5yGViLwOzDQ3XanG2cV9311BqoHOw8B1m8C/uQ+ngg843OOFYjP7zmAT4H/AFWBU4H5wJ0+v3sngPtwfrcqAwNxfufbuOtGAXPyvO8vgJo4v+e7gL7utmvd89QF53fxDPf/sBywCHgCqIDzYbIeuCTSf99FzguRDsB+fP4znK+8x4E67vJq4AH3cTmcJJUU4LiRwJR8nnMWBSfqiwqIa0/26+Iktyvy2U/dPxYBDuKTOIFzgd/cx08Bn2UnniCvK27SuMBdHgrMdB+fAezESbjlC3meN7jHjcL50OiL82EV776HZgW9hwDP2RHYk+e8j/JZvhv4Kp9jewJZwF6cPmAF/gtUyHtu8zl+DPCy+/g2nNZ4YqjnIcD6n4DH3McTCTFR43SNHMX9wHfXDQK+9fnd25Tntb4EbvdZLofTV9/U531399n+ETDCffw18OcA8Z8d4HVGAv9X2L9Jr/xY14e33AKkaO4FrvfJ7f6og9M/uC7AcY3zWR+qzb4LIvKg+3UyQ0T2AjXc1w/1teritOgWuV+/9wJfuesB/obTikoRkfUiMiLQk6jzF/YBuX3JNwLvudvWAn/B+fq8U0Q+EJEGob5h1zvucw4hT7dHQe9BRKqIyH/E6YbaB3wP1Mz+mu/a4fP4EHBKkFi2qWpNVa2O03o8DLwVaEcROVtEvnW/1mcAd5H7//MOTgL7QES2iciL4lwELIyGQHohjwGnNVse2O5zzv6D07LOtjnAMa/47J+O8yHZ0Gef/M5jfr+LTYEG2c/pPu+jOB8kUckStUe4fZ/XAT1EZIeI7AAeAJJEJAnYDRwBTg9w+OZ81oPTKqzis1w/wD45pT/i9EcPd2Oppao1caoQsvsFg71Wtt04iaadm3xqqmoNdS6Uoar7VfVBVW0BXA4MC9K/PAm4xu23PBv4JCdo1fdVtTu5XRYvFBCX/5tW3YhzUbEfTgs25PeA0yXUCjjbTa4XuOuFYlLVDJwP6cvz2eV9nC6ZxqpaA6cbSdxjj6vqX1W1LU7XWH/8++KDEpEuOElydkH7BrAZp0Vdx+ecVVfVdj775C0z24zTNVLT56eyqs4J8fXy+3v4Lc9zVlPVfkV4T55gido7BuKUSLXF+RrdEaff7gfgZnXKpSYA/3AvlMSJyLkiUhGnldlbRK4TkXhx6ls7us+7BLjKbQGeAdxeQBzVcPoRdwHxIvIEUN1n+5vA0yLSUhyJIlLb9wncWN8AXhaRUwFEpKGIXOI+7i8iZ7gXhfa57zszUDCqutiN5U3ga1Xd6z5HKxG5yH3/R3CSasDnKMDtOF0/BwvzHtzzdBjnwloCUGKldCJyCnAD+VdTVAPSVfWIiHTF+VaQfeyFItLBbdnvw+lKK/C8iEh1EemP8w3mXVVdXti4VXU7kAL83X2+ciJyuoj0CHLYWGCkiLRz46ghIteG+JJvAg+JSGf3d/EM9wN9PrBPnIvNld2/lfbuh1BUskTtHbfg9KFtUtUd2T/Aa8BgcUqZHsK5kLcA5yviC0A5Vd2E0yp80F2fXQ8M8DJwDOeC01u4XQdBfI3Tb7gG2IiTBH2/rv4Dp58wBScRjCdwWdtwnO6Nn9yugek4LVBwLpZNBw4Ac4F/qeqsIDFNwulTft9nXUWccrLdOF+NT8X5eouIDBaRkErGVHWdqi7MZ3Ow9zAG533vxunT/SqU1wuigTiVEwdwznsCMDiffe8GnhKR/TgXzD7y2VYf+Bjn/2YV8B3wbpDX/dx9ns3AYzj/v7cW433cjHMBbyXOtY2PgdPy21lVp+D8Hn/gnuMVwKWhvJCqTgaexfm92I9zITNBVTNxvo10xPnGtBsnqdco2luKPLvhxRhjPM5a1MYY43GWqI0xxuMsURtjjMdZojbGGI8rU4Oi1KlTR5s1axbpMIwxJqBFixbtVtW6edeXqUTdrFkzFi7MrxLLGGMiS0Q2BlpvXR/GGONxlqiNMcbjLFEbY4zHlak+6kCOHz/Oli1bOHLkSKRDMSYmVapUiUaNGlG+fGEH8TPZynyi3rJlC9WqVaNZs2b4TBxhjCkBqkpaWhpbtmyhefPmkQ4nakW060NE+oozhdLaQGMSi0hrEZkrIkdF5KHCHBuqI0eOULt2bUvSxoSBiFC7dm37xlpMEUvU7jCMr+OMlNUWGCQibfPslg7cD+SdNy+UYwsTS1EPNcYUoKz9fYVjoLtItqi7AmtVdb2qHsMZB/cK3x1UdaeqLsAZU7dQxxpjTGnKzMzi5ZfnctNNU0o8WUcyUTfEf5zjLfhPv1Mix4rIHSKyUEQW7tq1q0iBloYpU6YgIqxevTrSoZykW7dukQ4BgA0bNiAiPP744znrdu/eTfny5bn33nvD+to9e/akVatWdOzYkTZt2jBu3Licbf369WPv3r3Fev7Ro0fTsGFDOnbsSMuWLbnqqqtYuXJlzvY//vGPOcuTJ0+mTZs2XHjhhQAMGjSIxMREXn755WLFYIpuzZo0evSYyLBhKbz//nImTVpRos8fyUQd6PtQqB9DIR+rquNUNVlVk+vWPenOTM+YNGkS3bt354MPPgjr62RmFn4SlDlzQpkVqXS0aNGCL774Imd58uTJtGvXLsgRJee9995jyZIl/PjjjwwfPpxjx44BMG3aNGrWrFns53/ggQdYsmQJqampXH/99Vx00UVkNy7efPNN2rZ1evfGjx/Pv/71L7799lt27NjBnDlzWLZsGQ888EBIr3PixIlix2oc2a3opKSx/Phjbttx/PjFJdqqjmSi3oIzOWW2RsC2UjjWcw4cOMCPP/7I+PHjT0rUL774Ih06dCApKYkRI5xrpmvXrqV3794kJSXRqVMn1q1bx6xZs+jfv3/Ocffeey8TJ04EnFvnn3rqKbp3787kyZN544036NKlC0lJSVx99dUcOnQIgN9//50rr7ySpKQkkpKSchL0Kafkzsn6t7/9jS5dupCYmMiTTzqzTx08eJDLLruMpKQk2rdvz4cffuj3HlatWkXXrl1zljds2EBiYiIAI0aMoG3btiQmJvLQQ37XiwOqXLkybdq0yRkK4MMPP+S6667L2b5r1y6uvvpqunTpQpcuXfjxxx8BmD9/Pt26deOss86iW7du/PrrrwBMnDiRq666ir59+9KyZUseeeSRAmM4cOAAVatWJS4uLuf87t7tzEc8cOBAOnfuTLt27XJa3ZmZmQwZMoT27dvToUOHkFq+119/PX369OH9951JbXr27MnChQt56qmnmD17NnfddRcPP/wwffr0YefOnXTs2JEffviBdevW0bdvXzp37sz555+f8w1tyJAhDBs2jAsvvJDhw4dz8OBBbrvtNrp06cJZZ53FZ599VuD5+Oqrr+jUqRNJSUn06uVMcZnf85QVDz2UwrBhKRw54nz4xceXY/ToHnz55eAS7ZuPZHneAqCliDQHtuLMEXdj8ENK5Nj8vR/Gix435v/p+umnn9K3b1/OPPNMEhIS+Pnnn+nUqRNffvkln376KfPmzaNKlSqkpzsTQw8ePJgRI0Zw5ZVXcuTIEbKysti8Oe/kzv4qVarE7NnOfKVpaWkMHToUgFGjRjF+/Hjuu+8+7r//fnr06MGUKVPIzMzkwIEDfs+RkpJCamoq8+fPR1UZMGAA33//Pbt27aJBgwb873//AyAjI8PvuDZt2nDs2DHWr19PixYtcpJreno6U6ZMYfXq1YhIyN0HN9xwAx988AH169cnLi6OBg0asG2b8zn95z//mQceeIDu3buzadMmLrnkElatWkXr1q35/vvviY+PZ/r06Tz66KN88okzT+6SJUtYvHgxFStWpFWrVtx33300btz4pNcdPHgwFStWJDU1lTFjxuQkal8TJkwgISGBw4cP06VLF66++mo2bNjA1q1bWbHC+Toc6vvs1KnTSV1hTzzxBDNnzuSll14iOTmZe+65h/79+7NkyRIAevXqxdixY2nZsiXz5s3j7rvvZubMmQCsWbOG6dOnExcXx6OPPspFF13EhAkT2Lt3L127dqV37975no9KlSoxdOhQvv/+e5o3b57zu/jss88GfJ6qVauG9B6j3b33dmXcuJ85dOg4SUn1mDhxIB07Bpo/ungilqhV9YSI3IszR18cMEFVfxGRu9ztY0WkPrAQZ3LVLBH5C9BWVfcFOjYy76T4Jk2axF/+8hfASUKTJk2iU6dOTJ8+nVtvvZUqVZxJxBMSEti/fz9bt27lyiuvBJwEHIrrr78+5/GKFSsYNWoUe/fu5cCBA1xyiTNf68yZM3n77bcBiIuLo0YN/ynmUlJSSElJ4ayzzgKclmVqairnn38+Dz30EMOHD6d///6cf/75J73+ddddx0cffcSIESP48MMP+fDDD6levTqVKlXij3/8I5dddpnfN4Jg+vbty+OPP069evX83hfA9OnT/fp29+3bx/79+8nIyOCWW24hNTUVEeH48dzr07169cp5r23btmXjxo0BE/V7771HcnIyu3btolu3bvTt25emTZv67fPqq68yZcoUADZv3kxqaiqtWrVi/fr13HfffVx22WX06dMnpPdZ2K/OBw4cYM6cOVx7be7csEePHs15fO211+Z8uKSkpDB16lReeskpqDpy5AibNm0CAp+PPXv2cMEFF+TUQickJAR9njZt2hQq9mh1+ukJvPTSxezceZCRI8+nQoWTP7xLQkRveFHVacC0POvG+jzegdOtEdKx0SgtLY2ZM2eyYsUKRITMzExEhBdffBFVPenrU35/vPHx8WRlZeUs561b9W3hDBkyhE8//ZSkpCQmTpzIrFmzQopVVRk5ciR33nnnSdsWLVrEtGnTGDlyJH369OGJJ57w23799ddz7bXXctVVVyEitGzZEnC6JGbMmMEHH3zAa6+9ltP6C6ZChQp07tyZv//97/zyyy98/vnnOduysrKYO3culSv7z7d73333ceGFFzJlyhQ2bNhAz549c7ZVrFgx53FcXFyBfbh169alU6dOzJs3zy9Rz5o1i+nTpzN37lyqVKlCz549OXLkCLVq1WLp0qV8/fXXvP7663z00UdMmDChwPe5ePFikpOTC9wvW1ZWFjVr1sxpXefl+zugqnzyySe0atXKb5958+YFPB+BfheDPU+syczM4tVX56EKw4ad67ftT38K/+TmNtaHrxs1fD/5+Pjjj7n55pvZuHEjGzZsYPPmzTRv3pzZs2fTp08fJkyYkNOHnJ6eTvXq1WnUqBGffvop4LSYDh06RNOmTVm5ciVHjx4lIyODGTNm5Pua+/fv57TTTuP48eO8917upOS9evXi3//+N+D0q+7bt8/vuEsuuYQJEybkdIls3bqVnTt3sm3bNqpUqcJNN93EQw89xM8//3zSa55++unExcXx9NNP57SCDxw4QEZGBv369WPMmDH5JphAHnzwQV544QVq167tt75Pnz689tprOcvZz5mRkUHDhk5hUHbffVEdOnSIxYsXc/rpp/utz8jIoFatWlSpUoXVq1fz008/AU5lSlZWFldffTVPP/10wPOT1yeffEJKSgqDBg0KOa7q1avTvHlzJk+eDDhJdOnSpQH3veSSS/jnP/+Z88G/ePHioM997rnn8t133/Hbb78B5HR9FPZ5olFqam5Fx8iRM1ixYmepx2CJOsImTZqU042R7eqrr+b999+nb9++DBgwgOTkZDp27Jjz9fKdd97h1VdfJTExkW7durFjxw4aN27MddddR2JiIoMHD87pngjk6aef5uyzz+biiy+mdevWOetfeeUVvv32Wzp06EDnzp355Rf/3qQ+ffpw4403cu6559KhQweuueYa9u/fz/Lly+natSsdO3bk2WefZdSoUQFf9/rrr+fdd9/Nufi3f/9++vfvT2JiIj169Mi5yDZ16tSTWuR5tWvXjltuueWk9a+++ioLFy4kMTGRtm3bMnas8wXtkUceYeTIkZx33nlFqnwBp4+6Y8eOdO7cmSFDhtC5c2e/7X379uXEiRMkJiby+OOPc8455wDOB1rPnj3p2LEjQ4YM4bnnngv4/C+//HJOed67777LzJkzKWyl0nvvvcf48eNJSkqiXbt2+V7ce/zxxzl+/DiJiYm0b9/er+QxkLp16zJu3DiuuuoqkpKScj5sC/s80SS7oiMxMbei49ixTF56qfSroCQcd9F4VXJysuadOGDVqlVlpj/NmEiJtr+z1NQ0br31M7+Su/j4cjz22Pk8+mj4+qJFZJGqntTfVeYHZTLGmGzZfdGPPjozp+QOCGtFRygsURtjDJFrRYfCEjXke0XbGFN80dC9qqoMGvQJixZtz1kX6Va0rzJ/MbFSpUqkpaVFxS+TMdEmezzqUOv9I0VEeP31fpQrJ8THl+PJJ3swf/5QTyRpsBY1jRo1YsuWLXh5wCZjoln2DC9ekpWliPgPwXr22Y149dW+nHdeE88k6GxlPlGXL1/eZp4wpgzJ7ou+887O/OEPSX7b7rmnaz5HRVaZT9TGmLIhb0XHL7/solevFjRoUC3SoRWozPdRG2Nin+/dhdlldwcOHOOHHzZGOLLQWIvaGBOzvFoXXViWqI0xMcnLddGFZYnaGBNTYqUV7csStTEmpuzbd5QXXvjRb9aVaGxF+7KLicaYmFKrVmXGjnUmoUhKqseCBUMZPbpn1CZpsBa1MSbK7dp1kLp1/af+GjiwNZMnX8uAAa2iOkFnsxa1MSYqZWZm8Y9/zKVp0zHMmLH+pO3XXNM2JpI0WKI2xkShNWvSuOCCiTz4YAqHD5/g9tunsn//0YIPjFKWqI0xUSN71pWkpLHMmZNbdlezZiV27ToUwcjCy/qojTFRYc2aNG677eS66FGjzg/rDOBeYInaGONpsVgXXViWqI0xnrV+/R5uvnlKmWxF+7JEbYzxtMWLd+Q8LkutaF92MdEY41ktWtTihRd6e3LWldJkLWpjjCdkZmaxaNF2unZt6Lf+7ru70Lt3C1q3rhOhyCLPWtTGmIhbs8YZL7p79wksX/6737Zy5aRMJ2mwRG2MiSDfuugff9zM8eNZDBnyGcePZ0Y6NE+xrg9jTETkVxd9+eVnRjAqb7JEbYwpVVYXXXiWqI0xpSa/VnS0jxcdbpaojTGl4t13lzF06OfWii4CS9TGmFJx5pm1OXbMuUhorejCsURtjCkVXbs25OGHu/HVV2utFV1IoqqRjqHUJCcn68KFCyMdhjExb82aNNatS+fSS1v6rT969AQiYq3ofIjIIlVNzrve6qiNMSXGty560KBP2Lp1n9/2ihXjLUkXQUQTtYj0FZFfRWStiIwIsF1E5FV3+zIR6eSzbYOILBeRJSJizWRjIiz77sJhw1I4cuQEGRlHueeeaZEOKyZErI9aROKA14GLgS3AAhGZqqorfXa7FGjp/pwN/Nv9N9uFqrq7lEI2xgQQrC569OiekQsshkTyYmJXYK2qrgcQkQ+AKwDfRH0F8LY6Hek/iUhNETlNVbeXfrjGmLysLrp0RDJRNwQ2+yxvwb+1nN8+DYHtgAIpIqLAf1R1XKAXEZE7gDsAmjRpUjKRG1PG2d2FpSuSiVoCrMtbghJsn/NUdZuInAp8IyKrVfX7k3Z2Evg4cKo+ihOwMcZxww2f8PHHuV9+rRUdXiFdTBSRpiLS231cWUSqlcBrbwEa+yw3AraFuo+qZv+7E5iC05VijCkFf/hDYs7jpKR6LFgwlNGje1qSDpMCE7WIDAU+Bv7jrmoEfFoCr70AaCkizUWkAnADMDXPPlOBm93qj3OADFXdLiJVsz8sRKQq0AdYUQIxGWNCMGBAK269tWOZnnWlNIXS9XEPTmt1HoCqprrdDcWiqidE5F7gayAOmKCqv4jIXe72scA0oB+wFjgE3OoeXg+YIiLZ7+F9Vf2quDEZY/xl90UnJtajV68WftvGjx+A+zdowiyURH1UVY9l/4eISDwn9yUXiapOw0nGvuvG+jxWnA+KvMetB5JKIgZjTGC+FR1NmtRgxYo/Ua1axZztlqRLTyh91N+JyKNAZRG5GJgMfB7esIwxkZJ31hWATZsyGDPmpwhHVnaF0qIeAdwOLAfuxGkBvxnOoIwxkRGsLnr48O4RjKxsCyVRV8bpP34Dcu4orIzTZ2yMiQFWF+1toSTqGUBv4IC7XBlIAbqFKyhjTOmxuwu9L5REXUlVs5M0qnpARKqEMSZjTCk5fPg43btPYNeu3C/I1or2nlAuJh7MM2pdZ+Bw+EIyxpSWypXL5wycFB9fzuqiPSqUFvVfgMkikn3X4GnA9eELyRhTmu66K5lfftnJ0KGdLUF7VIEtalVdALQG/gTcDbRR1UXhDswYU7LWrEnjooveYvny3/3WlysnvP76ZZakPSzUiQO6AInAWcAgEbk5fCEZY0qSb130t99uYMiQzzh+PDPSYZlCKLDrQ0TeAU4HlgDZ/7sKvB3GuIwxJSBQRceyZb8zd+4WLrigaQQjM4URSh91MtBWy9IsuMZEOauLji2hJOoVQH2cwfqNMR5nddGxJ5REXQdYKSLzgaPZK1V1QNiiMsYUmrWiY1coiXp0uIMwxhTfypW7ePjhb8jMdHoprRUdO0Ipz/sO2ACUdx8vAH4Oc1zGmELq0KEejzxyHmCzrsSaUKo+huJMDpuAU/3REBgL9ApvaMaYYI4cOUGlSv5/wk8+2YP69U/hrruSLUHHkFDqqO8BzgP2gTPDC1DsGV6MMUWTXRfdosUrbN26z29bxYrx3H//2ZakY0woifqoqh7LXijJGV6MMYWTmppGjx4TGTYshe3bDzB06OdY5WzssxlejIkC2a3oxMSxfmV327btJz3dxkiLdTbDizEeZ3XRJmiidmdzeUtVbwLeKJ2QjDFgddEmV9BEraqZIlJXRCr49lMbY8LLWtHGVyhdHxuAH0VkKnAwe6Wq/iNcQRlT1v36626/JG2t6LItlIuJ24Av3H2r+fwYY8Lk8stbcdNNiTbrigFAQi3tEZGqqnqw4D29Kzk5WRcuXBjpMIzxk5mZxebN+2jWrKbf+vT0w2zalGEJugwRkUWqmpx3fYEtahE5V0RWAqvc5SQR+VcYYjSmzMmui77ggv9j376jftsSEipbkjZAaF0fY4BLgDQAVV0KXBDOoIyJdXnrojdv3sfDD6dEOizjUaFcTERVN4uI7yqbx8eYIsqvouO006qhquT5WzMmpES9WUS6ASoiFYD7cbtBjDGhs7poU1ShJOq7gFdwRs3bAqTgDNRkjAmR1UWb4sg3UYvIC6o6HLhQVQeXYkzGxJR//3sBw4alWCvaFFmwi4n9RKQ8MLK0gjEmFqmSk6StLtoURbCuj6+A3UBVEdkHCM7wpgKoqlYvhfiMiXp33ZXMxx+vJD39sLWiTZHke8OLiFRU1aMi8pmqXlHKcYWF3fBiwm3NmjQyM7No06au3/pduw5So0Yl64s2QRXlhpe57r/7guxjjCG3LjopaSyDB/+X48f9K1jr1q1qSdoUWbCujwoicgvQTUSuyrtRVf8bvrCMiR55KzoWL97Bc8/N5oknekQ4MhMrgrWo7wLOAWoCl+f56V8SLy4ifUXkVxFZKyIjAmwXEXnV3b5MRDqFeqwx4ebbis470t2AAa0iGJmJNfm2qFV1NjBbRBaq6viSfmF3UoLXgYtx6rMXiMhUVV3ps9ulQEv352zg38DZIR5rTNjkVxc9atT5jBxpddGmZAWro75IVWcCe8LU9dEVWKuq693X+wC4AvBNtlcAb6tzxWcsauIAABw3SURBVPMnEakpIqcBzUI41pgSZ3cXmkgI1kfdA5iJ09WRlwLFTdQNgc0+y1twWs0F7dMwxGMBEJE7gDsAmjRpUryITZmWmZlF797vMGvWhpx1dnehKQ3Buj6edP+9NUyvHWjkmby1gvntE8qxzkrVccA4cMrzChOgMb7i4srRrVujnERtrWhTWgqa3LYVTmu0tbtqFTBOVdeUwGtvARr7LDfCmU0mlH0qhHCsMSXuiSd68OWXaxkwoJW1ok2pybfqQ0TOBWYBB3BapG/gzJk4S0TOKYHXXgC0FJHm7qh8NwBT8+wzFbjZrf44B8hQ1e0hHmtMkWVmZvHKKz+xZYv/bQQVK8Yzb94fGT26pyVpU2qCtaifAAap6iyfdZ+KyEzgSZyKjCJT1RMici/wNRAHTFDVX0TkLnf7WGAa0A9YCxwCbg12bHHiMSabb0XHV1+tY9q0G/3GiC5f3hK0KV3BbiFfo6pn5rPtV1WNukJRu4XcBJNfRcdHH13Dtde2i2BkpqzI7xbyYC3q/UG2RfUkt8bkFawu+oorWgc50pjwC5aoG4vIqwHWC055nDFRz+qiTTQIlqgfDrLN+g9M1LNZV0y0CFZH/VZpBmJMadq0KYOOHcdy+LC1oo33BRuUyZiY1aRJjZwLhDbrivG6UCa3NSYmjRlzCTt3HuS553pZgjaeZi1qE/PWrEnjuusmk5FxxG99rVqV+fLLwZakjecVmKhF5EURqS4i5UVkhojsFpGbSiM4Y4rDd7zoyZNX8tBDKZEOyZgiCaVF3UdV9+FMFrAFOJPgFSHGRNyaNWn06DGRYcNScsru3nprKb/9tifCkRlTeKEk6vLuv/2ASaqaHsZ4jCmWYLOuzJ8/lObNa0UwOmOKJpSLiZ+LyGrgMHC3iNQFjhRwjDGlzuqiTawqMFGr6ggReQHYp6qZInIQZzYVYzzB7i40sS7U8ryGwMUiUsln3dthiMeYQpsyZTXDhuVeKCxzcxeqwu65kHkY6l0EEmheDRPNCkzUIvIk0BNoizPs6KXAbCxRG4+4+uo29OrVnBkzfit7rehdc2DJcNg121k+ZyK0uCWiIZmSF8rFxGuAXsAOd1quJKBiWKMyJoisLP+heUWE8eMH8NRTPcvO3YUZq+D7K+Gb83KTNMCWTyMXkwmbUBL1YVXNAk6ISHVgJ9AivGEZc7Lsio7u3Sdw/Him37amTWvy+OM9Yr+r49BWmDcUprUPnJTTFpR+TCbsQknUC0WkJs5UXIuAn4H5YY3KmDx866Lnzt3C//t/P0Q6pNJ1bC8sGQmfnwHr3gTNyt3WdBDEVXEeH94Kh7dHJkYTNgUmalW9W1X3ulNjXQzcEsaZyY3xk19d9BdfpHLiRFaQI2NE5hFY9XeY2gJWPu8sZ6vfB/ougvPeh4SzctenLyr9OE1Y5XsxUURaq+pqEekUYFsnVf05vKGZsq6guuj4+BgeqiYrEza8C8seh0Ob/bcldIaOz0P93j7rkmHXj87jtIXQsH/pxWrCLljVxzDgDuDvAbYpcFFYIjJlXpmui1aFbdNgyQjIWOG/7ZTTIelZaHItSJ4PqYQuuY/TrZ861gSbOOAO998LSy8cU9YFm7sw5uuid//klNrt/N5/faVTof0TcPpQiKsQ+NjaPvOhpi90Er7VU8eMoHXUItIUOKiqu0XkHKA7sFZVrQbIhMW77y47aYyOmG9FZ6yGpY/Clin+6+NPgTYPQethUL5a8Oeo1hLiq8GJ/XBkJxzaAlUbhy9mU6ry7eQTkSeAmcBPIvIMMAaoA/xZRMaUUnymjHnssfNp3/5U4uPLMXp0jM+6cmgbzLvDLbXzSdISD2feC5evhQ5PFpykwekKSeicu2zdHzElWIv6BqANUAXYBNRX1UMiEg8sKY3gTGzLzMxi//5j1KyZOzJBxYrxvPvulagSuwn62F5Y+SL8Osa57dtX0xsg8Rmodnrhn7d2F9g5y3mcthAaX1XsUI03BEvUR1T1GHBMRNap6iEAVT0hIsdKJzwTq1JT07j11s845ZQKfPnlYMSnPzUpKUYTdOYRWPMv+OVZOJZntOD6vZ1KDt9WcWEl5OmnNjEjWKKuKSJXAQJUdx/jLtcIe2QmJgWq6JgwYTG3335SFWjsyMqEDe+5pXab/LfVOgs6vgCnXVz817ELijErWKL+Drjcffy9z+PsZWMKJbsVnbeiIz39cJCjopgqbPsSlo6Avcv9t53SAhKfhabXnVxqV1RVm0OFBKe1fmwPHFhftC4U4znByvPs7kNTIspkXfTueW6p3Xf+6yvWdUrtzrgj/1K7ohJxuj92uEO+pi+0RB0jQh2P2pgiya8VHbOzruz7FZY+Bps/8V8fXxVaPwRtHgytiqOoaudJ1E2vD99rmVJjidqEhaoyZsxPZacVfXg7LP+rO2CSz8h+Eg9n3AntH4fK9cIfh+8FxTS7oBgrLFGbsBARli3bmZOkY7YVfSwDVr0Iq18+udSuyfWQ9AxUO6P04qnteyv5ImeUvZLqAzcRE1KiFpFuQDPf/VXVZngxQb388iWkpKyjbt0qsdeKzjwKqW6p3dE0/231LnIqOXyrMEpL5YZQqR4c+d25S3HfGqjRuvTjMCUqlKm43gFOx7nJJfs7nWJTcRkfqalpnHpqVWrUyL15pWbNSsyYcTMtWtSKnVZ0ViZsfN8ptTu40X9brY5Ogq5/ceTK4rIvKG77n7OcvtASdQwIpUWdDLRVVS1wT1Pm+FZ0DB7cgTffHOC3vXXrOhGKrISpwvavnFHt9i7z31a1udPF0fQGb3Qz+CbqtAXQ/KbIxmOKLZREvQKoD9i0EcZP3oqO8eMXc+21bbnkklLsky0Nu+e7pXaz/NdXrOOW2t1Z8qV2xeHXT20XFGNBKIm6DrBSROYDR7NXquqA/A8xsSxYXXT9+qdEMLIStm+NW2r3sf/6+KrQ+kG31K56ZGILxvc29D2LIesElLO6gWgWyv/e6JJ+URFJAD7EuUC5AbhOVfcE2K8v8AoQB7ypqs+760cDQ4Fd7q6Pquq0ko7TnKxM1EUf3g7Ln4J1bwQotbvDLbXz8IXRyvWhSiNnqNPMw7BvFdTsEOmoTDEUmKhV9buC9imCEcAMVX1eREa4y8N9dxCROOB1nHkatwALRGSqqq50d3lZVV8KQ2wmgDJxd+HxfbDyb7D6H5B5yH9bk+ucUe2qt4xMbIWVkOwkanD6qS1RR7UCr3yIyDkiskBEDojIMRHJFJF9xXzdK4C33MdvAQMD7NMVZ5KC9e4ofh+4x5lStmfP4ZwZwH3rop98MkbGi848CqvHOBPI/vKMf5KudyFcMh+6fxg9SRqsnzrGhNL18RrO2NSTcSpAbgaK+xtbT1W3A6jqdhE5NcA+DQHfWT23AGf7LN8rIjcDC4EHA3WdAIjIHThzP9KkSZNihl021axZicqVy+csx0wrWrNgQ3ap3Qb/bTWT3FHt+kTnCHR2h2JMCamWSFXXAnGqmqmq/wf0LOgYEZkuIisC/ITaKg7015FdIvhvnNrujjjVKIEm4M2OfZyqJqtqct26dUN8aeNLRHjzzctJSKgcG61oVdj2FXzZCeb+wT9JV20G574Ll/4MDS6JziQN/hcU9y6FTBtCPpqF0qI+JCIVgCUi8iJOYqxa0EGq2ju/bSLyu4ic5ramTwN2BthtC+A76VsjYJv73L/7PNcbwBchvA8TgszMLN55Zxk33tjB78Jg06Y1+e23P1O9esUIRlcC0hY4pXa/f+u/vmJtaPc4tLwL4qL8PYLzfk5p4Qx1mnUMMpYXb1ICE1GhtKj/4O53L3AQJ3leXczXnQrc4j6+BfgswD4LgJYi0tz9oLjBPQ43uWe7EqfW2xRTamoaPXpM5NZbP+O55344aXtUJ+l9qTD7Ovi6q3+SjqviVHEMWA+t/xwbSTqbdX/EjAITtapuxOmGOE1V/6qqw9yukOJ4HrhYRFJxqjqyy+4aiMg093VP4Hw4fA2sAj5S1V/c418UkeUisgy4EHigmPGUaZmZWbz88lwSE8fmlN0988wPrFgR6ItOlDm8AxbcDf9rA5sm566XOGj5JxiwFhKf8mY9dHHZ1FwxI5SxPi4HXgIqAM1FpCPwVHFueFHVNKBXgPXbgH4+y9OAk+qjVfUPRX1t4y9YXfSZZ9aOYGTFdHwfrHoJVv09QKndtW6p3ZmRia20+A4KlWazkkezUG946QrMAlDVJSLSLGwRmVIRs3XRmUdh7X9gxdNwdLf/tlN7OpUcdbpGJLRS59snnbECThyG+MqRi8cUWSiJ+oSqZki0Xv02J4nJuws1CzZMckvtfvPfVjPRLbWL4iqOoihfHaq3cmad0Uyn+qPOOZGOyhRBSIMyiciNQJyItATuB+aENywTLnPmbKZXr7djpxWtCttTnAlk9yzx31a1qdPF0exGb4xqFwkJyU6iBueCoiXqqBTKb+99QDucAZkmAfuAv4QzKBM+yckNaNkyAYiBuwvTFsLM3jCrr3+SrlgbOr0M/X91hvgsq0ka8lxQtH7qaBXKWB+HgMfcHxPlKlSIY+LEgdxxx+e8+eaA6EzQ+9c6o9pt+sh/fVxlaD0M2jwMFWpEJjavsVvJY0K+iVpEpgY70IY59b41a9IYP/5nnn++N77XGDp1Oo0FC4YSddcdDv8OK56CteNAc7tukDg4/Y/O2NBVGkQuPi+q1dH5RqFZkLEKjh+A8jE0FG0ZEaxFfS7OWBuTgHkEvqXbeFDeio4zzkhg6FD/u9KiKkkf3+eU2a3+O5w46L+t8dWQ9Kxz0cycLL4qVG/rVH2gzvjUp54f6ahMIQXrvKsPPAq0xxkT+mJgt6p+F6ahT00JWLMm7aSR7kaMmMHBg1E41kPmMfj1nzD1DKcl7ZukT+0BfebC+R9bki6I1VNHvXwTtTsA01eqegtwDrAWmCUi95VadCZk2XcXJiWN9Su7S0qqx4wZN1O1qoemiipIdqndF61h0f1wdFfutpodoOc06PWtVTCEKsH6qaNd0IuJIlIRuAwYhDMby6vAf8MflimMNWvSuO22GKmL3v6NM2jSnsX+66s0cSeQvRHKRdH78QK7lTzqBbuY+BZOt8eXwF9V1QY+8piYurswfREsHg6/z/BfXyEB2o9yxuWIqxSZ2KJdrURnGjE9AftT4dheqFAz0lGZQgjWov4Dzmh5ZwL3+1x8EkBVNQZHsYkuL7zwI489NjNnOSpb0fvXwtJRsOlD//VxlaH1A9DmESu1K664Ss7dmXt+dpbTF0H9k4baMR6Wb6JW1TJ8l0B0uPvuLrz++gK2bdsffa3ow78743Gs/U+AUrvbof2TVmpXkmon+yTqhZaoo4zNIR/FatasxBtvXM78+VujpxV9fL9bavdSgFK7qyDxWajROjKxxbKEZGCc89jGpo46lqijQHZf9Lp1e3jttX5+2/r1a0m/flEw6WrmMedGlRVP+VdxANQ9H8560ao4wsluJY9qlqg9Lm9Fx2WXteTSS6MgMWfTLNj4ESx7zJkWyleN9tDxeWjQr2yNahcJNdtDuYqQdRQOboQju6CSzSEaLawf2qPyq4t+/fUoag3tmA5fdYE5g/yTdJXGcM5EuHQJNLzMknRpKFfeuZ08W/qiyMViCs1a1B60Zo0zXvScOf510aNGnc/IkVFw+2/6z7BkBOz4xn99hVrQ7jE48x4rtYuEhGRIm+c8Tl8IDfpGNh4TMkvUHpKZmcUrr8zjsceitC56/zpYNgo2fuC/Pq4StPoLtB1u9buRVDsZUt3Hdit5VLFE7REFtaI9XdFxZKdTapc6Nk+pXTlocRt0GA1VGkYsPOOyW8mjliVqj3j44W/8knRUtKKP74fV/3AmkT1xwH9boyudUe1qtIlMbOZk1VtDXBVnst/D2+DQNqtVjxJ2MdEj/vnPS6lWrQLx8eUYPdrjs65kHoM1r8PnZ8Dy0f5Jum53uHgOXPBfS9JeUy4OEjrlLlurOmpYizoCMjOzyMpSypfP7c5o0qQGb701kObNa3k3QWc7lg6LH3FaZtlqtHNL7ayKw9MSkmHXbOdx+kJoZPN/RANrUZey7PGin3nm+5O2XXllG+8naYDK9aHNg87jKo3gnP+DS5dCw/6WpL3Od2ouu0MxaliLupTkHelu3rytDBzYmrPOOi3SoRVNm4ecke3OuBPiK0c6GhOqvEOeqtqHaxSwFnUpCDTrCsDChdsiGFUxla8Orf9iSTraVDvD+b8D51b+Q5siG48JiSXqMAo268qCBUNPmsfQmLCTcv6tauv+iAqWqMMkUCs6Pr4cTz7p8YoOE/tsxpeoY33UJSymZl0xsam2JepoY4m6hGVmKm+9tdSvFR11s66Y2ObX9TEfso47gzYZz7KujxJWoUIcEycOJD6+XE5f9OjRPS1JG++o2gwqu7f0H9+XW1dtPMsSdTFt2pSBqvqt69ixPt988wfrizbeJAKNrshd3vJZ5GIxIbFEXUTZFR2tWr3Gm2/+fNL2nj2bWSvaeJdfov7Uqac2nmWJugjyVnQ8+GAKmzZlRDosY0J3as/ceuqDG2HvsoiGY4KzRF0I+dVFt2hRi4MHj0UwMmMKKa6CMwVaNuv+8DRL1CEqqC66TRubf85EmYZ5uj+MZ0UkUYtIgoh8IyKp7r+18tlvgojsFJEVRTm+JBR0d6FVdJio1eDS3LK8PYvhoN1O7lWRalGPAGaoaktghrscyEQg0MRuoR5fLJs3Z9jdhSZ2VagBp16Yu2zdH54VqUR9BfCW+/gtYGCgnVT1eyC9qMcXV/XqFdm4MfciobWiTcxp7POnY4nasyKVqOup6nYA999Tw3W8iNwhIgtFZOGuXbsK9SI1alRi3Lj+1oo2sauhz8QBO7+DY3siF4vJV9huIReR6UCgrPZYuF4zEFUdB4wDSE5OLnSx6KWXtmT9+vtp3LhGicdmTMRVaehMepu+wJmYeOs0aD440lGZPMKWqFW1d37bROR3ETlNVbeLyGnAzkI+fXGPLxRL0iamNbrCSdQAWz+zRO1Bker6mArc4j6+BShs51hxjzfGZPO9S3Hbl5B5NHKxmIAilaifBy4WkVTgYncZEWkgItOydxKRScBcoJWIbBGR24Mdb4wpghrt4JTTnccnDsDvMyMbjzlJRIY5VdU0oFeA9duAfj7LgwpzvDGmCLIHaVr9D2d5y2dOjbXxDLsz0RgDjfKU6WlW5GIxJ7FEbYyBOt2gYh3n8ZEdkLYgsvEYP5aojTFQLg4a9s9dtrE/PMUStTHGkbf7w3iGJWpjjKP+xRBX2Xm8bxXsWxPZeEwOS9TGGEd8FTitT+6ytao9wxK1MSaX7xjVWy1Re4UlamNMrob9Qdy0sGsOHP49svEYwBK1McZXpbpQ5zx3QWHbFxENxzgsURtj/PnNUG7dH15gidoY4883Ue/4Bk4cjFwsBrBEbYzJq9oZzkBNAJlHYHtKZOMxlqiNMQH43vyy1fqpIy0io+cZYzyuyTVwZKfTDVLfBqqMNEvUxpiT1eoIZ4+LdBTGZV0fxhjjcZaojTHG4yxRG2OMx1miNsYYj7NEbYwxHmeJ2hhjPM4StTHGeJyoaqRjKDUisgvYWMjD6gC7wxBOOFnMpcNiLh1lKeamqlo378oylaiLQkQWqmpypOMoDIu5dFjMpcNitq4PY4zxPEvUxhjjcZaoCxaNAx5YzKXDYi4dZT5m66M2xhiPsxa1McZ4nCVqY4zxuDKdqEWkr4j8KiJrRWREkP26iEimiFxT2GNLWjFj3iAiy0VkiYgsLJ2IC45ZRHqKSIYb1xIReSLUYz0asyfPs0/cS0TkFxH5rjDHejBmT55nEXnY5/dihft3mBDKsflS1TL5A8QB64AWQAVgKdA2n/1mAtOAawpzrJdidtdvAOp47TwDPYEvivp+vRSzx89zTWAl0MRdPjUKznPAmL18nvPsfzkws7jnuSy3qLsCa1V1vaoeAz4Argiw333AJ8DOIhxb0ooTc6QU51x5/Tx7SSgx3wj8V1U3AajqzkIc67WYI6Ww52oQMKmIx+Yoy4m6IbDZZ3mLuy6HiDQErgTGFvbYMClOzAAKpIjIIhG5I2xR+gv1XJ0rIktF5EsRaVfIY0tacWIG757nM4FaIjLLje3mQhwbDsWJGbx7ngEQkSpAX5xGU6GOzassz5koAdblrVUcAwxX1UwRv91DOTYcihMzwHmquk1ETgW+EZHVqvp9OAL1EUrMP+OMcXBARPoBnwItQzw2HIoTM3j3PMcDnYFeQGVgroj8FOKx4VDkmFV1Dd49z9kuB35U1fQiHOunLCfqLUBjn+VGwLY8+yQDH7gJrw7QT0ROhHhsOBQ5ZlX9VFW3gfP1UUSm4HwVC/cvdoExq+o+n8fTRORfIlInlGPDpMgxq+pur55nd5/dqnoQOCgi3wNJIR4bDsWJeY2Hz3O2G8jt9ijssf5KsyPeSz84H1Lrgebkduy3C7L/RHIvJhbqWI/EXBWo5vN4DtDXCzED9cm9+aorsAmn9eHZ8xwkZi+f5zbADHffKsAKoL3Hz3N+MXv2PLv71QDSgaqFPTbQT5ltUavqCRG5F/ga52rsBFX9RUTucrcH6uMNeqyXYwbqAVPclnY88L6qfuWRmK8B/uR+WzkM3KDOb7aXz3PAmEXEs+dZVVeJyFfAMiALeFNVVwB49TznF7OItMCj59nd9UogRZ1vAkGPDeV17RZyY4zxuLJc9WGMMVHBErUxxnicJWpjjPE4S9TGGONxlqiNMcbjLFGbmCQiV4qIikjrSMfiS0TmRDoGE30sUZtYNQiYjXN3WFiISFxhj1HVbuGIxcQ2S9Qm5ojIKcB5wO34JGoRecQdv3ipiDzvrjtDRKa7634WkdPd8Y+/8DnuNREZ4j7eICJPiMhs4FoRGSoiC9zjP3EH4kFE6onIFHf9UhHp5q4/4PO8D7vHLhORv7rrqorI/9xjVojI9WE/YcbzyuydiSamDQS+UtU1IpIuIp1w7swcCJytqoeyB3IH3gOeV9UpIlIJp/HSOPDT5jiiqt0BRKS2qr7hPn4G58Phn8CrwHeqeqXb8j7F9wlEpA/OIE5dcW49nyoiFwB1gW2qepm7X43inQoTCyxRm1g0CGcUQXDG/B2Ek4D/T1UPAahquohUAxqq6hR33RGAAKMO5vWhz+P2boKuiZOMv3bXXwTc7D5vJpCR5zn6uD+L3eVTcBL3D8BLIvICzsQEP4T4nk0Ms0RtYoqI1MZJku1FRHHGVFCcMYHzjpeQX0Y+gX+3YKU82w/6PJ4IDFTVpW73SM9QQwWeU9X/nLRBpDPQD3hORFJU9akQn9PEKOujNrHmGuBtVW2qqs1UtTHwG85IZrf59CEnqDNU6RYRGeiuq+hu3wi0dZdr4IyFnJ9qwHYRKQ8M9lk/A/iT+7xxIlI9z3Ffu/Gc4u7TUEROFZEGwCFVfRd4CehUnJNhYoMlahNrBgFT8qz7BGgATAUWisgS4CF32x+A+0VkGc5QmfVVdTPwEc6Ibe+R2z0RyOPAPOAbYLXP+j8DF4rIcmAR4DsDDKqaAryPMxD+cuBjnKTfAZjvxvgY8Ezob93EKhs9zxhjPM5a1MYY43GWqI0xxuMsURtjjMdZojbGGI+zRG2MMR5nidoYYzzOErUxxnjc/wcbHxctzxzLFgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(accuracies, mean_difference, lw=3,color='orange', label=\"Accuracies vs. Mean Bias Difference\")\n", + "axe.set_xlabel(\"Accuracies\")\n", + "axe.set_ylabel(\"Mean Bias Difference\")\n", + "axe.set_title(\"Accuracies vs. Mean Bias Difference\")\n", + "axe.plot([0.4, 0.7], [-0.1, 0.1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Interpretation\n", + "\n", + "The graph shows that, for almost all the Tao values, the accuracy goes down as the tao goes up. The mean bia difference is loves when tao is 0.88, coming all the way down to -0.015 from -0.19 biased model. We can clearly observe a trade-off between the accuracy score and bias reduction, as expected." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Other Metrices to Use\n", + "1. FDR (False Discovery Rate): is a method of conceptualizing the rate of type I errors in null hypothesis testing when conducting multiple comparisons. (from Wikipedia) The idea of false discovery rate is to make the total number of FALSE discoveries (false positive + false negative) LOWER as a percentage of the overall discoveries." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### FDR Code Implementation\n", + "\n", + "We implement the FDR metric to assess the \"fairness\" of the model from another bias related metric. Below we compare the best model from the iterations above, with the unconstrained model. This proves the reduction in bias. " + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy : 3254 14653 0.777929434245547\n", + "SR tau : 0.24060062422550277\n", + "FPR tau : 0.43600739741090616\n", + "FNR tau : 0.8609189723320158\n", + "TPR tau : 0.31004901960784315\n", + "TNR tau : 0.980340417865866\n", + "AR tau : 0.8224361467717171\n", + "FDR tau : 0.4288194444444444\n", + "FOR tau : 0.3872744450835204\n", + "PPR tau : 0.4588815789473684\n", + "NPR tau : 0.8136439105306991\n", + "0.4288194444444444\n" + ] + } + ], + "source": [ + "from aif360.algorithms.inprocessing.celisMeta.utils import getStats\n", + "\n", + "#Unconstrained Model\n", + "mfc = MetaFairClassifier(tau=0, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)\n", + "dataset_bias_test = mfc.predict(dataset_orig_test)\n", + "\n", + "\n", + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in list(dataset_bias_test.labels)]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "x_control_test = pd.DataFrame(data=dataset_orig_test.features, columns=dataset_orig_test.feature_names)[\"sex\"]\n", + "\n", + "acc, sr, unconstrainedFDR = getStats(y_test, predictions, x_control_test)\n", + "print(unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy : 6660 14653 0.5454855660956801\n", + "SR tau : 0.9683859899039216\n", + "FPR tau : 0.9120522139618913\n", + "FNR tau : 0.6842461122379986\n", + "TPR tau : 0.9711585968379447\n", + "TNR tau : 0.8803995524643298\n", + "AR tau : 0.7673932778424438\n", + "FDR tau : 0.6863407379682039\n", + "FOR tau : 0.2292871682218505\n", + "PPR tau : 0.3786416244779234\n", + "NPR tau : 0.9379752173348904\n", + "0.6863407379682039 0.4288194444444444\n" + ] + } + ], + "source": [ + "# the best model from the iterations above had a TAO of 0.88.\n", + "mfc = MetaFairClassifier(tau=0.88, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)\n", + "dataset_bias_test = mfc.predict(dataset_orig_test)\n", + "\n", + "predictions = list(dataset_bias_test.labels)\n", + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in predictions]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "x_control_test = pd.DataFrame(data=dataset_orig_test.features, columns=dataset_orig_test.feature_names)[\"sex\"]\n", + "\n", + "acc, sr, fdr = getStats(y_test, predictions, x_control_test)\n", + "print(fdr, unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "assert(fdr >= unconstrainedFDR)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### The statement above checks that the fdr with the balancing TAO gives better results than the unconstrained model in terms of fairness. " + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tau': 0.88, 'sensitive_attr': 'sex'}" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mfc.__dict__['_params']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Comparison with a Logistic Regression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### The comparison of the model at hand with the logistic regression comes from their similarity. Similar to Logistic Regression Classifier, Meta Fair Classfier calculates loss by trying to minimize a term (bias). And therefore, the algorithms work very similarly. We do not preprocess the data for the model to be similar to the Meta Fair Classifier example.\n", + "\n", + "#### In the cells below you will see the logistic regression with the sensitive vairable \"sex\" and without the variable (variable was deleted). " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## With sensitive variable \"sex\"" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score: 0.793\n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "\n", + "df_test = pd.DataFrame.from_records(dataset_orig_test.features)\n", + "df_test[\"labels\"] = np.concatenate(dataset_orig_test.labels, axis=0)\n", + "y_test = df_test[\"labels\"]\n", + "X_test = df_test.drop([\"labels\"],axis=1) \n", + "\n", + "df_train = pd.DataFrame.from_records(dataset_orig_train.features)\n", + "df_train[\"labels\"] = np.concatenate(dataset_orig_train.labels, axis=0)\n", + "y_train = df_train[\"labels\"]\n", + "X_train = df_train.drop([\"labels\"],axis=1)\n", + "\n", + "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n", + "preds = clf.predict(X_test)\n", + "print(\"Accuracy Score:\", round(accuracy_score(preds, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Feature Importance (Permutation Test)" + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
WeightFeature
\n", + " 0.0649\n", + " \n", + " ± 0.0030\n", + " \n", + " \n", + " x1\n", + "
\n", + " 0.0242\n", + " \n", + " ± 0.0015\n", + " \n", + " \n", + " x0\n", + "
\n", + " 0.0199\n", + " \n", + " ± 0.0011\n", + " \n", + " \n", + " x2\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 94, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import eli5\n", + "from eli5.sklearn import PermutationImportance\n", + "\n", + "perm = PermutationImportance(clf, random_state=0).fit(X_train, y_train)\n", + "eli5.show_weights(perm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Without sensitive variable \"sex\"" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score without the sensitive variable: 0.778\n" + ] + } + ], + "source": [ + "y_train = df_train[\"labels\"]\n", + "X_train = df_train.drop([\"labels\", 2],axis=1)\n", + "y_test = df_test[\"labels\"]\n", + "X_test = df_test.drop([\"labels\", 2],axis=1) \n", + "\n", + "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n", + "preds = clf.predict(X_test)\n", + "print(\"Accuracy Score without the sensitive variable:\", round(accuracy_score(preds, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Feature Importance (Permutation Test)" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
WeightFeature
\n", + " 0.0584\n", + " \n", + " ± 0.0031\n", + " \n", + " \n", + " x1\n", + "
\n", + " 0.0219\n", + " \n", + " ± 0.0027\n", + " \n", + " \n", + " x0\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 96, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "perm = PermutationImportance(clf, random_state=0).fit(X_train, y_train)\n", + "eli5.show_weights(perm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Correlation Among Variables\n", + "#### \"2\" is the SEX variable" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012labels
01.0000.0320.0910.231
10.0321.0000.0120.335
20.0910.0121.0000.214
labels0.2310.3350.2141.000
\n", + "
" + ], + "text/plain": [ + " 0 1 2 labels\n", + "0 1.000 0.032 0.091 0.231\n", + "1 0.032 1.000 0.012 0.335\n", + "2 0.091 0.012 1.000 0.214\n", + "labels 0.231 0.335 0.214 1.000" + ] + }, + "execution_count": 97, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "round(df.corr(),3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpretation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### From the analysis and comparison above we come to a very important conclusion, that serves as the krux of our bias/fairness discussion. As we can see above, the logistic regression model, that works similar to Meta Fair Classifier (through minimizing an error term), achieves a much higher accuracy than MFC. Accuracy of the unconstrained model in MFC is almost the same as the accuracy of the Logistic Regression without the SEX variable. We can also see this in the Global Feature Importance graphs of the two logistic regression models. The reason both of these models have higher and very similar accuracies is because SEX is actually not a very preditive variable. This is proven by the global feature importance diagrams (0.0199) concoted through a Permuttion test.\n", + "\n", + "### While the imputation of the sensitive variable serves as a loss of data, we can see that it also erases bias. However, erasing sensitive variables is not enough to erase bias in most of the cases. To make sure that there is no inter variable dependency such that the deleted variable is highly corraleted with another variable, which vauses a indirect bias, we created a correlation graph above. On it, we see that none of the other two predictors are strongly correlated with the sensitive variable \"Sex\". \n", + "\n", + "### Overall it is important to keep these strategies in mind and decide WHEN and HOW to use IBM's algorithms. In some cases such as the one I just walked you through, using a logistic regression and imputing the SEX variale yields less bias and higher accuracy than any IBM algorithm." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/IBM_AIF360_Module.ipynb b/IBM_AIF360_Module.ipynb new file mode 100644 index 00000000..99fcd939 --- /dev/null +++ b/IBM_AIF360_Module.ipynb @@ -0,0 +1,1223 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Understanding Fairness Through Analyzing IBM's AIF360" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Welcome to a introductory module to IBM's AIF360. AIF360 is a AI Fairness package that allows users to evaluate their models for bias during preprocessing, during processing and post-processing. Furthermore, the package allows interference with the model in the three stages mentioned afore, and also serves as a tool to create fairness in your own AI or ML model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Importing Packages and Data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting aif360\n", + " Using cached https://files.pythonhosted.org/packages/54/a7/de16a858cbd70d9d7b9c79c06286d79bcc6ca58507f919c656e8c324286c/aif360-0.2.3-py3-none-any.whl\n", + "Requirement already satisfied: pandas>=0.23.3 in /opt/conda/lib/python3.7/site-packages (from aif360) (0.25.3)\n", + "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from aif360) (0.21.3)\n", + "Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from aif360) (1.3.2)\n", + "Requirement already satisfied: numpy>=1.16 in /opt/conda/lib/python3.7/site-packages (from aif360) (1.17.3)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23.3->aif360) (2.8.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23.3->aif360) (2019.3)\n", + "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->aif360) (0.14.1)\n", + "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas>=0.23.3->aif360) (1.13.0)\n", + "Installing collected packages: aif360\n", + "Successfully installed aif360-0.2.3\n" + ] + } + ], + "source": [ + "%%bash\n", + "pip install aif360\n", + "pip install eli5" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.metrics import accuracy_score\n", + "# setting random seed\n", + "np.random.seed(218)\n", + "from aif360.datasets import AdultDataset\n", + "from aif360.metrics import BinaryLabelDatasetMetric\n", + "from aif360.metrics import DatasetMetric\n", + "from aif360.algorithms.preprocessing import Reweighing\n", + "from IPython.display import Markdown, display\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### We will be using the AdultDataSet which serves as a CENSUS income dataset for adults with demographic information. The sensitive variable, or in other words, the variable that creates the biased \"priviledge\" is male." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['age', 'education-num', 'sex']\n", + "['income-per-year']\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.7/site-packages/aif360/datasets/standard_dataset.py:121: FutureWarning: outer method for ufunc is not implemented on pandas objects. Returning an ndarray, but in the future this will raise a 'NotImplementedError'. Consider explicitly converting the Series to an array with '.array' first.\n", + " priv = np.logical_or.reduce(np.equal.outer(vals, df[attr]))\n", + "/opt/conda/lib/python3.7/site-packages/aif360/datasets/standard_dataset.py:142: FutureWarning: outer method for ufunc is not implemented on pandas objects. Returning an ndarray, but in the future this will raise a 'NotImplementedError'. Consider explicitly converting the Series to an array with '.array' first.\n", + " df[label_name]))\n" + ] + } + ], + "source": [ + "single_protected = ['sex'] \n", + "single_privileged = [['Male']]\n", + "ad = AdultDataset(protected_attribute_names=single_protected,\n", + " privileged_classes = single_privileged, \n", + " categorical_features=[], \n", + " # even if the priviledged features are not specified, they are kept\n", + " features_to_keep=['age', 'education-num'])\n", + "print(ad.feature_names)\n", + "print(ad.label_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_orig_train, dataset_orig_test = ad.split([0.7], shuffle=True)\n", + "privileged_groups = [{'sex': 1}]\n", + "unprivileged_groups = [{'sex': 0}]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Explore the Data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Training Dataset shape" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(34189, 3)\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Protected attribute names" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['sex']\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Privileged and unprivileged protected attribute values" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[array([1.])] [array([0.])]\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Dataset feature names" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['age', 'education-num', 'sex']\n" + ] + } + ], + "source": [ + "display(Markdown(\"#### Training Dataset shape\"))\n", + "print(dataset_orig_train.features.shape)\n", + "display(Markdown(\"#### Protected attribute names\"))\n", + "print(dataset_orig_train.protected_attribute_names)\n", + "display(Markdown(\"#### Privileged and unprivileged protected attribute values\"))\n", + "print(dataset_orig_train.privileged_protected_attributes, \n", + " dataset_orig_train.unprivileged_protected_attributes)\n", + "display(Markdown(\"#### Dataset feature names\"))\n", + "print(dataset_orig_train.feature_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### After we assign the priviledged and under-priviledged groups, we calculate the baseline fairness." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Original training dataset" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference in mean outcomes between unprivileged and privileged groups = -0.192812\n" + ] + } + ], + "source": [ + "metric_orig_train = BinaryLabelDatasetMetric(dataset_orig_train, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + "display(Markdown(\"#### Original training dataset\"))\n", + "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % \n", + " metric_orig_train.mean_difference())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This value shows that there is a mean difference between priviledged and underpriviledged groups, and the biad towards the privildeged group." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction of Fainess Algorithm: Inprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### I will be introducing, executing and walking you through one of the inprocessing techniques that IBM AI360 has created to introduce fairness to the model. The interesting aspect of inprocess algorithm is the intervention the algorithm makes in the model, not post or pre analysis. \n", + "\n", + "#### Meta Fair Classifier takes the fairness metric, BinaryLabelDatasetMetric, as an input and optimizes the algorithm for it. It is essentially passing the fairness metric as a loss function, and trianing the model, while sacrificing from some of the accuracy, to be more fair. The example code for this introductory modelu is inspired by https://github.com/IBM/AIF360/blob/master/aif360/algorithms/inprocessing/meta_fair_classifier.py.\n", + "\n", + "#### Below we create the MetaFairClassifier class, and then create the fit and predict functions for the model. Here, we train the biased model, where algorithm has a tau of 0, motivating it to not be fair:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Import error: No module named 'tensorflow'\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "import copy\n", + "from aif360.algorithms import Transformer\n", + "from aif360.algorithms.inprocessing.celisMeta.FalseDiscovery import FalseDiscovery\n", + "from aif360.algorithms.inprocessing.celisMeta.StatisticalRate import StatisticalRate\n", + "from aif360.algorithms.inprocessing.meta_fair_classifier import MetaFairClassifier\n", + "\n", + "mfc = MetaFairClassifier(tau=0, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Make Predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_bias_test = mfc.predict(dataset_orig_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Assess Bias" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Original training dataset" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference in mean outcomes between unprivileged and privileged groups = -0.062442\n", + "We can see that the mean outcome is still biased, towards the variable as before - sex: Male.\n" + ] + } + ], + "source": [ + "metric_orig_train = BinaryLabelDatasetMetric(dataset_bias_test, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + "display(Markdown(\"#### Original training dataset\"))\n", + "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % \n", + " metric_orig_train.mean_difference())\n", + "print(\"We can see that the mean outcome is still biased, towards the variable as before - sex: Male.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Accuracy of the Unconstrained Model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score: 0.778\n" + ] + } + ], + "source": [ + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in list(dataset_bias_test.labels)]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "print(\"Accuracy Score:\", round(accuracy_score(predictions, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Now we will change the tau values in the algorithm to introduce fairness into the model, to see how the accuracy score and the fairness metric changes with it." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#########################################\n", + "Tau: 0.01\n", + "Training Accuracy: 0.6112784813829009 , Training gamma: 0.6925396834753267\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.119845\n", + "Test Accuracy Score: 0.613\n", + "#########################################\n", + "Tau: 0.23\n", + "Training Accuracy: 0.6010705197578168 , Training gamma: 0.688744107329693\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.089704\n", + "Test Accuracy Score: 0.604\n", + "#########################################\n", + "Tau: 0.46\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.68\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.90\n", + "Training Accuracy: 0.4743923484161573 , Training gamma: 0.7351333138871446\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.081712\n", + "Test Accuracy Score: 0.479\n" + ] + } + ], + "source": [ + "tau_list = np.linspace(0.01, 0.90, 5)\n", + "accuracies = []\n", + "mean_difference = []\n", + "for tau in tau_list:\n", + " print(\"#########################################\")\n", + " print(\"Tau: %.2f\" % tau)\n", + " mfc = MetaFairClassifier(tau=tau, sensitive_attr=\"sex\")\n", + " mfc.fit(dataset_orig_train)\n", + " predictions_mfc = mfc.predict(dataset_orig_test)\n", + " pred_labels = predictions_mfc.labels\n", + " predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in pred_labels]\n", + " metric_orig_train = BinaryLabelDatasetMetric(predictions_mfc, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + " mean_diff = metric_orig_train.mean_difference()\n", + " print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % mean_diff)\n", + " acc = accuracy_score(predictions, y_test)\n", + " print(\"Test Accuracy Score:\", round(acc,3))\n", + " accuracies.append(acc)\n", + " mean_difference.append(mean_diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Results and Analysis" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "BEST TAU AND IT'S PRINT OUT FROM ABOVE\n", + "'''\n", + "#########################################\n", + "Tau: 0.68\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "'''" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Graphing (with unbiased algorithm)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAEWCAYAAACTwaluAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgUZbr38e9NEgwgIARQWYMCI4skQBBxQVlU4CiKMoijIrhwdERxnNeR4zYencOlvuNRGRUGN0ZH4HWDQcUNkEHFhUVUwC2iQkB22Qkk6fv9oyqh0+mQStLV6/25rr7oWrrqTpP+5amqp54WVcUYY7yoE+sCjDGJwwLDGOOZBYYxxjMLDGOMZxYYxhjPLDCMMZ5ZYBhjPLPASAAiskhEfhWRo2JdS6IQkb1Bj4CIHAiavjzW9SUqC4w4JyLZwJmAAsOivO/0aO4vklT16NIHsA64IGjei7GuL1FZYMS/0cAnwHTgquAFIlJPRB4WkZ9FZJeIfCgi9dxlZ4jIEhHZKSLrRWSMO3+RiFwbtI0xIvJh0LSKyI0i8j3wvTvvMXcbu0VkuYicGbR+mojcISI/iMged3kbEXlCRB4Oqfd1Ebkl9AcUkaki8teQef8SkVvd57eLyAZ3+9+KyMCavZXltn+6iHzqvm8bReSR4IAUkbNEZIW7/BMR6V3bfSYFVbVHHD+AfOD3QC+gCDg2aNkTwCKgFZAGnAYcBbQF9gCXARlAFpDrvmYRcG3QNsYAHwZNK/Ae0BSo5867wt1GOvBHYBOQ6S67DfgK+A0gQI677inARqCOu14zYH9w/UH77AesB8SdbgIcAFq6210PtHSXZQMnVvM9/AkYFDLvFKC3+76d6L7P17vLWgC7gZHuzzwG2Ao0jvXvQ6wfMS/AHkf4z4Ez3JBo5k5/A/zBfV7H/VDlhHndfwGzK9mml8AYUEVdv5buF/gWuLCS9b4GznGfjwfmVbKe4Bw29HOnrwMWus87AFuAQUBGDd/HCoERZp2JwMyg/S8OWf45MCrWvxOxftghSXy7CnhXVbe50zM4fFjSDMgEfgjzujaVzPdqffCEiPxRRL52m+c7gcbu/qva1z9wWie4/74QbiV1PpGzcFpEAL8DXnSX5QO3APcCW0Rkloi0rMkPFUxEuojIWyKyWUR2A/dw+GdqCfwc8pKfcVpyKc0CI0655yJGAmeJyCYR2QT8AcgRkRxgG1CI05wOtb6S+QD7gPpB08eFWafsFmb3fMXtbi1NVPUYYBdOq6Cqff0TuNCttzMwp5L1AGYCI0SkHdAHeLWsGNUZqnoG0M6t7cEjbMerp4AVOIc3jYD7OPwzbXT3FawtsCEC+01oFhjx6yKgBOgC5LqPzsAHwGhVDQDPAv8rIi3dk4993UuvLwKDRGSkiKSLSJaI5LrbXQlcLCL1RaQDcE0VdTQEinGO4dNF5B6gUdDyp4H7RaSjOLqLSBaAqhYAS3FaFq+q6oHKdqKqn7v7eBp4R1V3AojIb0RkgPtzFeIchpVU/fZVqSGwS1X3ikhXnMOQUnOBHiIywn3/RuMExtsR2G9ii/UxkT3CP3B+OR8OM38kzknHdKAe8CjOX75dwGIOn6g8E/gU5+TdeuAqd34z4F2ck6If4TT1Q89hdAiaTgOecbfzC/Angs4JuMvvAn50t7kUaB30+ivcbfb38DPf7a7726B53YHP3G3vAN7g8AnQy4HVHrZbVm/QvIHAd8BenPM6k4D5Qcv745y32OXu/9RY/07Ew6P0rLQxvhCRfjiHJtnqtIpMArNDEuMbEckAJgBPW1gkBwsM4wsR6QzsBI7HOWwyScAOSYwxnlkLwxjjWcLdXNSsWTPNzs6OdRnGJLXly5dvU9XmofMTLjCys7NZtmxZrMswJqmJSGhPV8AOSYwx1WCBYYzxzALDGONZwp3DCKeoqIiCggIKCwtjXYrxQWZmJq1btyYjIyPWpaS8pAiMgoICGjZsSHZ2NiJS9QtMwlBVtm/fTkFBAe3bt491OSnPt0MSEXlWRLaIyKpKlouITBaRfBH5UkR61nRfhYWFZGVlWVgkIREhKyvLWo9xws9zGNOBwUdYPgTo6D7GAVNqszMLi+Rl/7fxw7fAUNXFOLcjV+ZC4Hl1fAIcIyLH+1WPMakqEFAmTpzP2rW/1npbsbxK0oryQ8EVUMkQaCIyTkSWiciyrVu3RqW4mpg9ezYiwjfffBPrUmLif/7nf8jNzSU3N5e0tLSy55MnT451aSkrEFCuvXYuDz74EWefPb3WoRHLwAjXzgx7J5yqTlPVPFXNa968Qm/VuDFz5kzOOOMMZs2a5et+SkoiMeBU5N15552sXLmSlStXUq9evbLnN998c6xLS0mlYfHccysBWL9+N9OmLa/VNmMZGAU4A8iWao0zlmJC2rt3Lx999BHPPPNMhcB46KGHOPnkk8nJyWHixIkA5OfnM2jQIHJycujZsyc//PADixYt4vzzzy973fjx45k+fTrgdIm/7777OOOMM3j55Zd56qmn6N27Nzk5OVxyySXs378fgM2bNzN8+HBycnLIyclhyZIl3H333Tz22GNl273zzjsr/NW//fbbefLJJ8um7733Xh5++GF++eUX+vXrR25uLt26deODDz6o0fvzr3/9iz59+tCjRw/OPfdctmzZAsC2bdsYNmwY3bt357TTTmPVqrDnyE01hYYFwNixuUyaVLuvdInlZdW5wHgRmYUz6OsuVf2lthvNnvhmrQurzE8P/Eely+bMmcPgwYPp1KkTTZs2ZcWKFfTs2ZO33nqLOXPm8Omnn1K/fn127HBO61x++eVMnDiR4cOHU1hYSCAQYP369ZVuH5z+CB9+6Hzn0Pbt27nuOmcYyrvuuotnnnmGm266iZtvvpmzzjqL2bNnU1JSwt69e2nZsiUXX3wxEyZMIBAIMGvWLD777LNy2x41ahS33HILv//97wF46aWXePvtt5kxYwbnnXced955JyUlJWXBVF39+vVj2LBhiAhTp07l4Ycf5sEHH+Tuu++mT58+zJ07l3fffZcxY8bYvUK1VFlYPP30MOrUqd0JZN8CQ0RmAmcDzUSkAPgzzpfqoKpTgXnAUJwvkNkPjPWrlmiYOXMmt9zifKnXqFGjmDlzJj179mT+/PmMHTuW+vWdgbqbNm3Knj172LBhA8OHDwecIPDi0ksvLXu+atUq7rrrLnbu3MnevXs577zzAFi4cCHPP/88AGlpaTRu3JjGjRuTlZXF559/zubNm+nRowdZWVnltt2jRw+2bNnCxo0b2bp1K02aNKFt27b07t2bq6++mqKiIi666CJyc3OpiXXr1jFy5Eg2bdrEwYMH6dSpEwAffvghb77phPy5557LmDFj2LdvHw0aNKjRflKdn2EBPgaGql5WxXIFbvRr/9G0fft2Fi5cyKpVqxARSkpKEBEeeughZ+DUkMuClQ1alJ6eTiBweCS70L4HwR+iMWPGMGfOHHJycpg+fTqLFi06Yo3XXnst06dPZ9OmTVx99dVh1xkxYgSvvPIKmzZtYtSoUYDTMli8eDFvvvkmV155JbfddhujR48+4r7CufHGG7njjjsYOnQo8+fP54EHHgAqvhc2oFPt/OlP7/kWFkDijRreq1cvDbVmzZoK86Jp6tSpOm7cuHLz+vXrp4sXL9a33npL+/btq/v27VNV1e3bt6uqap8+fXT27NmqqlpYWKj79u3TdevWabt27bSwsFB37typ2dnZ+txzz6mqart27XTr1q1l28/KytLNmzfroUOHdNCgQXrVVVepquqll16qjzzyiKqqFhcX665du1RV9eDBg9qpUydt3769FhcXh/05Vq1apX379tWOHTvqxo0bVVX1p59+0qKiIlVVfeSRR3TChAme3pMGDRqUm+7WrZuuXLlSA4GAXnHFFTpw4EBVVb3hhht00qRJqqr63nvvaV5eXtjtxfr/OFGsWrVZW7T4vwr36tixc7SkJFCj7QDLNMznz24+i4CZM2eWHV6UuuSSS5gxYwaDBw9m2LBh5OXlkZuby1//6nzn8AsvvMDkyZPLTvZt2rSJNm3aMHLkSLp3787ll19Ojx49Kt3n/fffT58+fTjnnHM46aSTyuY/9thjvP/++5x88sn06tWL1atXA1C3bl369+/PyJEjSUtLC7vNrl27smfPHlq1asXxxztdYhYtWkRubi49evTg1VdfZcKECYDTYqnOuYZ7772X4cOHc9ZZZ3HssceWzb/vvvtYsmQJ3bt355577uG5557zvE1TUdeuLVi4cDR//GPfyLYsXAk3pmdeXp6G/qJ+/fXXdO7cOUYVJYZAIEDPnj15+eWX6dixY6zLqTb7P44uEVmuqnmh862FkQLWrFlDhw4dGDhwYEKGhQkvEFAmTHiLzz6L3jc4JsXdqubIunTpwtq1a2Ndhomg4Ksh//jHF7z77pWccor/3xWdNC2MRDu0Mt7Z/215oZdOd+06yIsvfhmVfSdFYGRmZrJ9+3b7xUpC6o6H4bWvSrKrrJ/FI48c6cbwyEmKQ5LWrVtTUFBAPN+YZmqudMStVOd3pywvkiIwMjIybDQmk9TiISwgSQ5JjElm8RIWYIFhTFyLp7AACwxj4poING58VNl0LMMCkuQchjHJSkT43/917kTetetgTMMCLDCMiXuloaFKTMMC7JDEmLgSCCjPPLOCkpJAufkiEvOwAAsMY+JG6QnOa699ndGj51QIjXhggWFMHAi9GjJjxldMn76yildFnwWGMTFW2aXTsWMrHw8lViwwjImheOtnURULDGNiJNHCAiwwjImJRAwLsMAwJuoSNSzAAsOYqPvllz28/XZ+2XSihAVYYBgTda1aNWLRojEcf/zRCRUWYF3DjYmJTp2yWLr0Oo4/vmHChAVYC8MY3wUCynffba8wv1WrRgkVFmCBYYyvSk9w5uVNY8mSI3/ZdiKwwDDGJ8FXQ/bsOcTgwf8M29JIJBYYxvgg3KXTESO60KFD0xhWVXsWGMZEWCL3s6iKBYYxEZTMYQE+B4aIDBaRb0UkX0QmhlneWEReF5EvRGS1iIz1sx5j/JTsYQE+BoaIpAFPAEOALsBlItIlZLUbgTWqmgOcDTwsInX9qskYv6RCWIC/LYxTgHxVXauqh4BZwIUh6yjQUEQEOBrYART7WJMxvpg9++ukDwvwNzBaAcEXngvcecEeBzoDG4GvgAmqWmFcMhEZJyLLRGSZfR2iiUcXX9yZiRNPB5I3LMDfruHh3q3Qb0s+D1gJDABOBN4TkQ9UdXe5F6lOA6YB5OXl2Tcum7gjIkyaNJC8vJYMH945KcMC/G1hFABtgqZb47Qkgo0FXlNHPvAjcJKPNRkTEYGAUlRUUm6eiHDJJV2SNizA38BYCnQUkfbuicxRwNyQddYBAwFE5FjgN8BaH2syptZKT3D+7nevVQiNZOfbIYmqFovIeOAdIA14VlVXi8j17vKpwP3AdBH5CucQ5nZV3eZXTcbUVrirITNmXExGRloMq4oeX29vV9V5wLyQeVODnm8EzvWzBmMiJVxYNGxYl7S01On/mDo/qTG1kCr9LKpigWFMFSwsDrPAMOYILCzKs8AwphIWFhVZYBhTiTvuWGBhEcICw5hKjBvXizZtGgEWFqVs1HBjKnHCCU1YtGgM06YtZ9KkgSkfFmCBYcwRnXBCEx54YFCsy4gbdkhiDM4JzvHj5/Hvf/8U61LimgWGSXmlV0OeeGIpQ4fOsNA4AgsMk9JCL53u31/Eyy+viXFV8csCw6SsyvpZTJ48JIZVxTcLDJOSrFNWzVhgmJRjYVFzFhgmpVhY1I4FhkkZqhYWtWWBYVJKaVdvsLCoCevpaVKGiHDvvWcDsH79bguLGrDAMCklODSc788y1WGHJCZpBQLKE098xqFDFb8OwMKiZiwwTFIqvRoyfvxb/Pa3L1cIDVMzFhgm6YReOp0791ueemp5jKtKDhYYJqlU1s/ihht6x7Cq5GGBYZKGdcrynwWGSQoWFtFhgWESnoVF9FhgmIRmYRFdFhgmoW3btp/33/+pbNrCwl8WGCahtWjRgEWLriI7+xgLiyhI2q7h76zexFOL11Kvbhr1MtKoXzeNenXTg567/2aUPk+nft00Mt3lpes466eTZr+Ecatdu2P49NNradasvoWFz3wNDBEZDDwGpAFPq+oDYdY5G3gUyAC2qepZkdj3hl8PsOznXyOxKQDqptcJCZg06mekk1k3jfpuyAQ/d8KpDvXrplcZTvUy0uwX3aNAQFmzZivdurUoN79FiwYxqii1+BYYIpIGPAGcAxQAS0VkrqquCVrnGOBJYLCqrhORFuG3Vn0HiiLbFfhQcYBDxQF2UhTR7ZbKzKhDvYw0GhyVTqPMDBrVK/03o2y6YWYGjTLTy80rXafhUelJHzqlJzhnzlzFG29cxsCBJ8S6pJTjZwvjFCBfVdcCiMgs4EIgeEjm3wGvqeo6AFXdEqmdX9KzNb2zm3KgqIQDh4rZf6iE/YdKKCwqKXt+4FAxB4pKn7v/FrnPi4o5UDq/qATVSFUWXmFRgMKiAL/uLwIO1GgbDY9ywqRhuFCpJGgaZTrrN8xMJz0tfk9phV4NOf/8mSxdel2FlobxV5WBISLjgRdVtbrt+1bA+qDpAqBPyDqdgAwRWQQ0BB5T1efD1DAOGAfQtm1bTzs/rnEmxzXOrGbJ4akqB4sDbtAUh4ROSVDoFJcPnXIhVXHZgaLS7QUiUueeg8XsOVhc49c3qJtWIVTChk9Q0GRmpOH3jZ+BgHLnre/y2qzVZfOGXvQb0rMy+W7znmpt69hGmTSulxHpElOGlxbGcTiHEyuAZ4F3VD39vQ33axT6unSgFzAQqAd8LCKfqOp35V6kOg2YBpCXl+fz3/qKRITMDOecQ9MGdSO+/UBAKSx2QmTfwWJ2Hyhmd2ERuw8UsbuwiD2Fxe7z4rJ5peuULqtNUJTad6iEfYdK+GVXYQR+qshQVba/9RP7vtpeNq/ByVksa5vJ4Mc+qPb20usII3q15sb+HWjTtH4kS00JVQaGqt4lIncD5wJjgcdF5CXgGVX94QgvLQDaBE23BjaGWWebqu4D9onIYiAH+I4UUqeOuCdC02l29FE12kZJQNlb6ITILk9BU1QumPYcLPb9sKu6KguLrCHZNR7PojigzFq6nleWF3Bxz1aM79+RtlkWHF55Ooehqioim4BNQDHQBHhFRN5T1T9V8rKlQEcRaQ9sAEbhnLMI9i+cAEoH6uIcsjxS/R/DpNURGtfPoHH9jHIp7VUgoOw95LZWPAbNrgNFHCqOzOFUKA0oa176tlxYHN/7OLqM/A1Sw5O7B4sDrNuxH3CC46VlBby6YgMX92jF+AEdaJdlV1qq4uUcxs3AVcA24GngNlUtEpE6wPdA2MBQ1WL3/Mc7OJdVn1XV1SJyvbt8qqp+LSJvA18CAZxLr6si8YOZ6qlTR9yTo7E/vi89wblg6aayeZHqlPXxD9t5dP53fPrjDsBpmb28vIDXPt/ARblOcLRvZsFRGanqdISI3Idz+PFzmGWdVfVrv4oLJy8vT5ctWxbNXZooe/PN7zj//Jll03704Pxk7XYem/89H6/dXm5+HaEsOE5ofnTE9pdoRGS5quZVmO8hME4FVqvqHne6IdBFVT/1pdIqWGCkhkmTPuDOOxf63t37sx938NiC7/gov2JwDMtpyfgBHenQIvWCozaB8TnQs/TKiHsoskxVe/pSaRUsMFLHm29+x5AhHaPSIW3ZTzt4bMH3fPD9tnLzxQ2OmwZ0oEOLhr7XES9qExgrVTU3ZN6Xqto9wjV6YoGRfAIBpaiohKOOiv2tTct/3sGj88MHx/ndW3LzgA50PDb5g6OywPDStW+tiNwsIhnuYwKwNvIlmlRUeoJz+PD/R2Fh7fuS1Favdk154Zo+vPb70zirU/Oy+arw+hcbOffRxdw4YwXfbqpeh7Fk4aWF0QKYDAzA6Xi1ALglkt24q8NaGMkjtLv3kCEdmD370rhoaZT6fN2vTF7wPe9/u7XCsqEnH8fNAzty0nGNwrwysVXWwvDScWsLTh8KYyIm3EhZxx13NBkZaTGsqqIebZvw3NhT+GL9TiYv+J4F3xz+Oznvq03M+2oTQ7o5wdH5+OQLjlBeWhiZwDVAV6Ds5gxVvdrf0sKzFkbiS+Rh9b4scIJj/tcVG9jndT2Wmwd2pGvLxjGoLLJqcw7jBZz7Sc4D/o3TxTs1D+BMrSVyWAB0b30MT1/VmzduOoNzuhxbbtk7qzfzH5M/5Lrnl7Fqw64YVegvT5dVVbVH6ZUREcnAuQFtQHRKLM9aGIkr0cMinNUbdzF5wfe8s3pzhWWDOrdgwsBOnNw68VoctWlhlI4Ys1NEugGNgewI1mZSQDKGBUDXlo35+5V5zLv5TAZ3Pa7csvlfb+GCxz/kmulL+bJgZ4wqjCwvgTFNRJoAdwFzcQbAedDXqkzS+fOf30+6sAjWpWUjpl7Zi7cmnMnQk8sHx4JvtjDs8Y8Y+9xnrFyf2MFxxEMSt1fnCFV9KXolHZkdkiSmDRt2c/bZ/yA/f0fShUU4327aw+SF3zPvq18qDBtwVqfmTBjUkZ5tm8SmOA9q09Nzsar2862yarLASFwbNuxm6tRl/Pd/90/qsAj23eY9/G1hPm98ubFCcJzZsRm3DOpIr3ZNY1PcEdQmMO7GGWTy/wH7Suer6o5IF+mFBYZJRN+7wfF6JcExYWBH8rLjJzhqExg/hpmtqhqTIZstMOJfIKDceOObDBv2G4YM6RjrcuJK/pa9PL7we+Z+sZFAyEfv9A5ZTBjYiVPaxz44ahwY8cYCI74FXw2pWzeNOXMutdAI44ete3liYT5zVm6oEBx9T8hiwqCOnHpCVmyKo3YtjNHh5ocb3TsaLDDiV7hLp9df34spU86PYVXx7cdt+3jcDY6SkOTo074ptwzqRN8Tox8ctQmMvwVNZuKM8L1CVUdEtkRvLDDiU7L2s4iWn7bt4/H385n9ecXgGNT5WJ6+qsJn11e1ufnsppANNcbpLm4MYGERCdnNGvDX3+Zw04AOPPF+Pq+uOBwcXVvGz01tNfmqq/2AHZQawMIi0tplNeChETks+j9nM6p3G5o2qMvVp7ePdVllvIwa/jqHv4CoDtAFiJuOXCZ2LCz806ZpfR64pDv3XFBM/brxMz6Il0r+GvS8GPhZVQt8qsckCFXluussLPwWT2EB3gJjHfCLqhYCiEg9EclW1Z98rczENRGha9fDX4RsYZEavATGy8BpQdMl7rzevlRkEsatt/YFYPXqLTz1lIVFKvASGOmqeqh0QlUPiUjkv5HYJKRbb+2Lqtb4u05NYvFylWSriAwrnRCRC3G+NtGkkEBAeeSRj9m/v6jCMguL1OElMK4H7hCRdSKyDrgd+E9/yzLxpPRqyK23vsuwYTPDhoZJDVUGhqr+oKqn4lxO7aqqp6lqvv+lmXgQeul0wYIf+fvfradtqqoyMERkkogco6p7VXWPiDQRkb9EozgTW5X1s5gw4dQYVmViycshyRBVLRtXTFV/BYb6V5KJB9Ypy4TjJTDSROSo0gkRqQccdYT1TYKzsDCV8RIY/wQWiMg1InIN8B7wDy8bF5HBIvKtiOSLyMQjrNdbREpEJCZ3wJrDLCzMkXi5W/UhEfkSGAQI8DbQrqrXiUga8ARwDlAALBWRuaq6Jsx6DwLvVL98E0kWFqYqXu9W3QQEgEtwxsP42sNrTgHyVXWt2/FrFnBhmPVuAl4FYvLlzuawnTsL+fjjw7cJWViYUJW2MESkE86XMF8GbMcZBFhUtb/HbbcC1gdNFwB9QvbRChiO883wlXY1F5FxwDiAtm3bety9qa6mTeuxcOFoBgx4nr59W1tYmAqOdEjyDfABcEFpvwsR+UM1th3uNy10eK9HgdtVteRIvQVVdRowDZwRt6pRg6mm449vyEcfXc0xx2RaWJgKjnRIcgnOocj7IvKUiAwkfAhUpgBoEzTdGtgYsk4eMEtEfgJGAE+KyEXV2IephUBAWbHilwrzmzatZ2Fhwqo0MFR1tqpeCpwELAL+ABwrIlNE5FwP214KdBSR9u7NaqNwvmoxeB/tVTVbVbOBV4Dfq+qcmv0opjpKT3CeeurTzJ37bazLMQnCS9fwfar6oqqej9NKWAlUeok06HXFwHicqx9fAy+p6moRuV5Erq9l3aYWgq+GFBUFGDHipbAtDWNCVWs4H/fbzv7uPrysPw+YFzJvaiXrjqlOLaZmwl06veKK7uTmHneEVxnjqMkgwCZBWT8LU1sWGCnCwsJEggVGCrCwMJFigZHkLCxMJFlgJLmFC3+0sDARY4GR5AYNOoG//W0IYGFhai++viXF+GL8+FM46aRmDBjQ3sLC1Iq1MJJMIKBhB+kdNOgECwtTaxYYSaT0BOeQIS+yb9+hql9gTDVZYCSJ4Kshixf/zNChM+zrAEzEWWAkgXCXTk88sQmZmXaKykSWBUaCs34WJposMBKYhYWJNguMBGVhYWLBAiMBWViYWLHASEB/+ctiCwsTExYYCeiGG/Lo1q0FYGFhosuuuyWg5s0bsHDhaKZMWcZdd/WzsDBRY4GRoJo3b8A995wV6zJMirFDkjgXCCj/+Z+v88ora6pe2RifWWDEsdKrIdOmrWDUqFcsNEzMWWDEqdBLpyUlynvv/RDjqkyqs8CIQ5X1s5gy5fwYVmWMBUbcsU5ZJp5ZYMQRCwsT7yww4oSFhUkEFhhxQFUZN+51CwsT9yww4oCI0KdPq7JpCwsTr6ynZ5y47rpeAHz66QamTbvAwsLEJVHVWNdQLXl5ebps2bJYl+EbVUXEwsLElogsV9W80Pm+HpKIyGAR+VZE8kVkYpjll4vIl+5jiYjk+FlPvAgElAcf/JDduw9WWGZhYeKZb4EhImnAE8AQoAtwmYh0CVntR+AsVe0O3A9M86ueeFF6NWTixAUMHvzPsKFhTLzys4VxCpCvqmtV9RAwC7gweAVVXaKqv7qTnwCtfawn5kIvnX78cQFTpiyNcVXGeOdnYLQC1gdNF7jzKnMN8Fa4BSIyTkSWiciyrVu3RrDE6Kmsn8Vtt50ew6qMqR4/AyPcwXjYM8YpPv0AAAfSSURBVKwi0h8nMG4Pt1xVp6lqnqrmNW/ePIIlRod1yjLJws/LqgVAm6Dp1sDG0JVEpDvwNDBEVbf7WE9MWFiYZOJnC2Mp0FFE2otIXWAUMDd4BRFpC7wGXKmq3/lYS0xYWJhk41sLQ1WLRWQ88A6QBjyrqqtF5Hp3+VTgHiALeNK9nFgc7tpvIrKwMMnI156eqjoPmBcyb2rQ82uBa/2sIVb27j3EypWbyqYtLEwysK7hPmnU6Cjmzx/NoEHPk5t7nIWFSQoWGD5q2rQeixaN4eij61pYmKRgd6tGSCCgLFmyvsL8Ro2OsrAwScMCIwJKT3CeeeZzvPjil7EuxxjfWGDUUvDVkEBAGT16TtiWhjHJwAKjFsJdOr3qqhxOPTWpb4kxKcwCo4asn4VJRRYYNWBhYVKVBUY1WViYVGaBUQ0WFibVWWBUw0cfrWP6dAsLk7osMKrhzDPb8eyzFyJiYWFSk3UNr6YxY3I58cQmnH56WwsLk3KshXEEgYCyZ0/FQXrPPLOdhYVJSRYYlSg9wTlw4PPs3FkY63KMiQsWGGEEXw1ZunQj5577QtiWhjGpxgIjRLhLp926taBBg7oxrMqY+GCBEcT6WRhzZBYYLgsLY6pmgYGFhTFepXxgWFgY413KB8aDD35oYWGMRykfGDfc0JvevVsCFhbGVCXlu4Yfc0wm7757JVOmLOX228+wsDDmCFI+MMAJjf/6rzNjXYYxcS+lDkkCAeW66+aWu0XdGONdyrQwgq+GPPPM54Bz56kxxruUaGGEXjpVdQbDMcZUT9IHRmX9LP7+9wtiWJUxiSmpA8M6ZRkTWUkbGBYWxkSer4EhIoNF5FsRyReRiWGWi4hMdpd/KSI9I7FfCwtj/OFbYIhIGvAEMAToAlwmIl1CVhsCdHQf44Apkdj39de/YWFhjA/8bGGcAuSr6lpVPQTMAi4MWedC4Hl1fAIcIyLH13bH/ftnl4WDhYUxkeNnP4xWQPDXmBcAfTys0wr4JXglERmH0wKhbdu2Ve74sstOBuD9939i6tTzLSyMiRA/WxjhPqVag3VQ1Wmqmqeqec2bN/e088suO5lp0y6wsDAmgvwMjAKgTdB0a2BjDdYxxsQJPwNjKdBRRNqLSF1gFDA3ZJ25wGj3asmpwC5V/SV0Q8aY+ODbOQxVLRaR8cA7QBrwrKquFpHr3eVTgXnAUCAf2A+M9aseY0zt+XrzmarOwwmF4HlTg54rcKOfNRhjIidpe3oaYyLPAsMY45kFhjHGMwsMY4xn4px3TBwishX42cOqzYBtPpdTG/FcXzzXBvFdXzzXBt7ra6eqFXpJJlxgeCUiy1Q1L9Z1VCae64vn2iC+64vn2qD29dkhiTHGMwsMY4xnyRwY02JdQBXiub54rg3iu754rg1qWV/SnsMwxkReMrcwjDERZoFhjPEs4QMjVgMNR6i2y92avhSRJSKSE63avNQXtF5vESkRkRHxVJuInC0iK0VktYj8O1q1ealPRBqLyOsi8oVbX9TuxBaRZ0Vki4isqmR5zT8TqpqwD5zb5n8ATgDqAl8AXULWGQq8hTO616nAp3FU22lAE/f5kGjV5rW+oPUW4tx1PCJeagOOAdYAbd3pFvH03gF3AA+6z5sDO4C6UaqvH9ATWFXJ8hp/JhK9hRGzgYYjUZuqLlHVX93JT3BGHIsWL+8dwE3Aq8CWOKvtd8BrqroOQFXjrT4FGoqIAEfjBEZxNIpT1cXu/ipT489EogdGZYMIV3cdP1R3v9fgpH60VFmfiLQChgNTiS4v710noImILBKR5SIyOmrVeavvcaAzzpCTXwETVDUQnfKqVOPPRKJ/e3vEBhr2gef9ikh/nMA4w9eKQnYbZl5ofY8Ct6tqifOHMmq81JYO9AIGAvWAj0XkE1X9zu/i8FbfecBKYABwIvCeiHygqrv9Ls6DGn8mEj0w4nmgYU/7FZHuwNPAEFXdHoW6SnmpLw+Y5YZFM2CoiBSr6pw4qK0A2Kaq+4B9IrIYyAGiERhe6hsLPKDOSYN8EfkROAn4LAr1VaXmn4lonSjy6eROOrAWaM/hk09dQ9b5D8qf4PksjmprizOe6Wnx+N6FrD+d6J309PLedQYWuOvWB1YB3eKovinAve7zY4ENQLMo/v9mU/lJzxp/JhK6haFxPNCwx9ruAbKAJ92/4sUapTsdPdYXE15qU9WvReRt4EsgADytqmEvI8aiPuB+YLqIfIXzwbxdVaNy27uIzATOBpqJSAHwZyAjqLYafyasa7gxxrNEv0pijIkiCwxjjGcWGMYYzywwjDGeWWAYYzxL6MuqJvZEJAunPwTAcUAJsNWdPkWdey1MkrDLqiZiROReYK+q/jXWtRh/2CGJ8Y07HsRydzyIa4PmXyEiX4nIKhGZFMsaTfXYIYnx01WqukNE6gPLRORVoAHwF5z7VHYB80XkfFV9I5aFGm+shWH89AcR+QL4GOcGpxOBPsBCVd2mqkXADJwBX0wCsBaG8YWIDMIJglNV9YCIfAhkEv7WapMgrIVh/NIY2OGGRVegtzv/E6C/iGSJSDowCojqeJym5qyFYfzyJjDOPST5BvgUQFULROQeYBFOa+N1VX0zZlWaarHLqsYYz+yQxBjjmQWGMcYzCwxjjGcWGMYYzywwjDGeWWAYYzyzwDDGePb/Adi09yrwRv5gAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(tau_list, accuracies, lw=3, label=\"Accuracy vs. Tao\")\n", + "axe.set_xlabel(\"Tao\")\n", + "axe.set_ylabel(\"Accuracy\")\n", + "axe.set_title(\"Accuracy vs. Tao\")\n", + "axe.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(tau_list, mean_difference, lw=3,color='orange', label=\"Mean Bias Difference vs. Tao\")\n", + "axe.set_xlabel(\"Tao\")\n", + "axe.set_ylabel(\"Mean Bias Difference\")\n", + "axe.set_title(\"Mean Bias Difference vs. Tao\")\n", + "axe.plot([0, 1], [0, 0.1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEWCAYAAABPON1ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUZfb48c8hoUsLIEgHRXqCEFARBQURERG7iKuoi7q2XbEAispafpZ1V3R1l0XhixUVXRRd1AiIiiBNqoAEkA4CCYTekvP7494kM2EymZTJ3Jmc9+uVF3PbzJlLcuaZ5577PKKqGGOM8a5ykQ7AGGNMcJaojTHG4yxRG2OMx1miNsYYj7NEbYwxHmeJ2hhjPM4StSkxInJARFpEOo5YFKlzKyJjReRxn+U/icjvbjy1ReQ8EUl1lweWdnxlhVgdtfeIyCwgCaivqkcjHE7MEZENQAOggaru9lm/BOe8N1fVDaUYT09gJnDIXZUBvKmqT4b5dTcA9YATQCawEngbGKeqWQH2Lw/sA85R1aXuuhnAVFV9JZyxlnXWovYYEWkGnA8oMKCUXzu+NF8vwn4DBmUviEgHoHLkwmGbqp6iqqcA3YHbS6mFermqVgOaAs8Dw4Hx+exbD6gE/OKzrmme5ZCVsd+3YrFE7T03Az8BE4FbfDeISGUR+buIbBSRDBGZLSKV3W3dRWSOiOwVkc0iMsRdP0tE/ujzHENEZLbPsorIPSKSCqS6615xn2OfiCwSkfN99o8TkUdFZJ2I7He3N/Z5rjPcxxVF5CUR2eR+VR7rE2sdEfnCjTVdRH4QkZN+F91jXsqz7jMRGeY+Hi4iW904fhWRXoU4z++45zrbLTitSd/XCvYearnvYZeI7HEfN/I5dpaIPC0iP7rxpYhInVACU9XfgDlAW5/n8z23l4nIYvf/Z7OIjPbZr5KIvCsiae75XSAi9UJ4zQxVnQpcD9wiIu3d55soIs+IyJnAr+7ue0VkpoisA1oAn7tdHxVFpIaIjBeR7e7/zTMiEuc+1xD3fLwsIunAaHf9bSKyyj2PX4tI0zzv+y63e2WPiLwuIuKzfah77H4RWSkindz1DUTkE/f/5zcRuT+Uc+9Zqmo/HvoB1gJ3A52B40A9n22vA7OAhkAc0A2oCDQB9uO0EMsDtYGO7jGzgD/6PMcQYLbPsgLfAAlAZXfdTe5zxAMPAjuASu62h4HlQCtAcLoKavs81xnu4zHAVPd5qwGfA8+5254Dxrqxlsf5BiEBzsUFwObsbUAt4DBOt0Urd1sDd1sz4PQQz/EGoDdO4mnjnsvNOK1DBZqF8B5qA1cDVdxtk4FPfV5jFrAOOBOnpT4LeD6feHoCW3yWWwJbgYvy/D+d4bN/B5yGViLwOzDQ3XanG2cV9311BqoHOw8B1m8C/uQ+ngg843OOFYjP7zmAT4H/AFWBU4H5wJ0+v3sngPtwfrcqAwNxfufbuOtGAXPyvO8vgJo4v+e7gL7utmvd89QF53fxDPf/sBywCHgCqIDzYbIeuCTSf99FzguRDsB+fP4znK+8x4E67vJq4AH3cTmcJJUU4LiRwJR8nnMWBSfqiwqIa0/26+Iktyvy2U/dPxYBDuKTOIFzgd/cx08Bn2UnniCvK27SuMBdHgrMdB+fAezESbjlC3meN7jHjcL50OiL82EV776HZgW9hwDP2RHYk+e8j/JZvhv4Kp9jewJZwF6cPmAF/gtUyHtu8zl+DPCy+/g2nNZ4YqjnIcD6n4DH3McTCTFR43SNHMX9wHfXDQK+9fnd25Tntb4EbvdZLofTV9/U531399n+ETDCffw18OcA8Z8d4HVGAv9X2L9Jr/xY14e33AKkaO4FrvfJ7f6og9M/uC7AcY3zWR+qzb4LIvKg+3UyQ0T2AjXc1w/1teritOgWuV+/9wJfuesB/obTikoRkfUiMiLQk6jzF/YBuX3JNwLvudvWAn/B+fq8U0Q+EJEGob5h1zvucw4hT7dHQe9BRKqIyH/E6YbaB3wP1Mz+mu/a4fP4EHBKkFi2qWpNVa2O03o8DLwVaEcROVtEvnW/1mcAd5H7//MOTgL7QES2iciL4lwELIyGQHohjwGnNVse2O5zzv6D07LOtjnAMa/47J+O8yHZ0Gef/M5jfr+LTYEG2c/pPu+jOB8kUckStUe4fZ/XAT1EZIeI7AAeAJJEJAnYDRwBTg9w+OZ81oPTKqzis1w/wD45pT/i9EcPd2Oppao1caoQsvsFg71Wtt04iaadm3xqqmoNdS6Uoar7VfVBVW0BXA4MC9K/PAm4xu23PBv4JCdo1fdVtTu5XRYvFBCX/5tW3YhzUbEfTgs25PeA0yXUCjjbTa4XuOuFYlLVDJwP6cvz2eV9nC6ZxqpaA6cbSdxjj6vqX1W1LU7XWH/8++KDEpEuOElydkH7BrAZp0Vdx+ecVVfVdj775C0z24zTNVLT56eyqs4J8fXy+3v4Lc9zVlPVfkV4T55gido7BuKUSLXF+RrdEaff7gfgZnXKpSYA/3AvlMSJyLkiUhGnldlbRK4TkXhx6ls7us+7BLjKbQGeAdxeQBzVcPoRdwHxIvIEUN1n+5vA0yLSUhyJIlLb9wncWN8AXhaRUwFEpKGIXOI+7i8iZ7gXhfa57zszUDCqutiN5U3ga1Xd6z5HKxG5yH3/R3CSasDnKMDtOF0/BwvzHtzzdBjnwloCUGKldCJyCnAD+VdTVAPSVfWIiHTF+VaQfeyFItLBbdnvw+lKK/C8iEh1EemP8w3mXVVdXti4VXU7kAL83X2+ciJyuoj0CHLYWGCkiLRz46ghIteG+JJvAg+JSGf3d/EM9wN9PrBPnIvNld2/lfbuh1BUskTtHbfg9KFtUtUd2T/Aa8BgcUqZHsK5kLcA5yviC0A5Vd2E0yp80F2fXQ8M8DJwDOeC01u4XQdBfI3Tb7gG2IiTBH2/rv4Dp58wBScRjCdwWdtwnO6Nn9yugek4LVBwLpZNBw4Ac4F/qeqsIDFNwulTft9nXUWccrLdOF+NT8X5eouIDBaRkErGVHWdqi7MZ3Ow9zAG533vxunT/SqU1wuigTiVEwdwznsCMDiffe8GnhKR/TgXzD7y2VYf+Bjn/2YV8B3wbpDX/dx9ns3AYzj/v7cW433cjHMBbyXOtY2PgdPy21lVp+D8Hn/gnuMVwKWhvJCqTgaexfm92I9zITNBVTNxvo10xPnGtBsnqdco2luKPLvhxRhjPM5a1MYY43GWqI0xxuMsURtjjMdZojbGGI8rU4Oi1KlTR5s1axbpMIwxJqBFixbtVtW6edeXqUTdrFkzFi7MrxLLGGMiS0Q2BlpvXR/GGONxlqiNMcbjLFEbY4zHlak+6kCOHz/Oli1bOHLkSKRDMSYmVapUiUaNGlG+fGEH8TPZynyi3rJlC9WqVaNZs2b4TBxhjCkBqkpaWhpbtmyhefPmkQ4nakW060NE+oozhdLaQGMSi0hrEZkrIkdF5KHCHBuqI0eOULt2bUvSxoSBiFC7dm37xlpMEUvU7jCMr+OMlNUWGCQibfPslg7cD+SdNy+UYwsTS1EPNcYUoKz9fYVjoLtItqi7AmtVdb2qHsMZB/cK3x1UdaeqLsAZU7dQxxpjTGnKzMzi5ZfnctNNU0o8WUcyUTfEf5zjLfhPv1Mix4rIHSKyUEQW7tq1q0iBloYpU6YgIqxevTrSoZykW7dukQ4BgA0bNiAiPP744znrdu/eTfny5bn33nvD+to9e/akVatWdOzYkTZt2jBu3Licbf369WPv3r3Fev7Ro0fTsGFDOnbsSMuWLbnqqqtYuXJlzvY//vGPOcuTJ0+mTZs2XHjhhQAMGjSIxMREXn755WLFYIpuzZo0evSYyLBhKbz//nImTVpRos8fyUQd6PtQqB9DIR+rquNUNVlVk+vWPenOTM+YNGkS3bt354MPPgjr62RmFn4SlDlzQpkVqXS0aNGCL774Imd58uTJtGvXLsgRJee9995jyZIl/PjjjwwfPpxjx44BMG3aNGrWrFns53/ggQdYsmQJqampXH/99Vx00UVkNy7efPNN2rZ1evfGjx/Pv/71L7799lt27NjBnDlzWLZsGQ888EBIr3PixIlix2oc2a3opKSx/Phjbttx/PjFJdqqjmSi3oIzOWW2RsC2UjjWcw4cOMCPP/7I+PHjT0rUL774Ih06dCApKYkRI5xrpmvXrqV3794kJSXRqVMn1q1bx6xZs+jfv3/Ocffeey8TJ04EnFvnn3rqKbp3787kyZN544036NKlC0lJSVx99dUcOnQIgN9//50rr7ySpKQkkpKSchL0Kafkzsn6t7/9jS5dupCYmMiTTzqzTx08eJDLLruMpKQk2rdvz4cffuj3HlatWkXXrl1zljds2EBiYiIAI0aMoG3btiQmJvLQQ37XiwOqXLkybdq0yRkK4MMPP+S6667L2b5r1y6uvvpqunTpQpcuXfjxxx8BmD9/Pt26deOss86iW7du/PrrrwBMnDiRq666ir59+9KyZUseeeSRAmM4cOAAVatWJS4uLuf87t7tzEc8cOBAOnfuTLt27XJa3ZmZmQwZMoT27dvToUOHkFq+119/PX369OH9951JbXr27MnChQt56qmnmD17NnfddRcPP/wwffr0YefOnXTs2JEffviBdevW0bdvXzp37sz555+f8w1tyJAhDBs2jAsvvJDhw4dz8OBBbrvtNrp06cJZZ53FZ599VuD5+Oqrr+jUqRNJSUn06uVMcZnf85QVDz2UwrBhKRw54nz4xceXY/ToHnz55eAS7ZuPZHneAqCliDQHtuLMEXdj8ENK5Nj8vR/Gix435v/p+umnn9K3b1/OPPNMEhIS+Pnnn+nUqRNffvkln376KfPmzaNKlSqkpzsTQw8ePJgRI0Zw5ZVXcuTIEbKysti8Oe/kzv4qVarE7NnOfKVpaWkMHToUgFGjRjF+/Hjuu+8+7r//fnr06MGUKVPIzMzkwIEDfs+RkpJCamoq8+fPR1UZMGAA33//Pbt27aJBgwb873//AyAjI8PvuDZt2nDs2DHWr19PixYtcpJreno6U6ZMYfXq1YhIyN0HN9xwAx988AH169cnLi6OBg0asG2b8zn95z//mQceeIDu3buzadMmLrnkElatWkXr1q35/vvviY+PZ/r06Tz66KN88okzT+6SJUtYvHgxFStWpFWrVtx33300btz4pNcdPHgwFStWJDU1lTFjxuQkal8TJkwgISGBw4cP06VLF66++mo2bNjA1q1bWbHC+Toc6vvs1KnTSV1hTzzxBDNnzuSll14iOTmZe+65h/79+7NkyRIAevXqxdixY2nZsiXz5s3j7rvvZubMmQCsWbOG6dOnExcXx6OPPspFF13EhAkT2Lt3L127dqV37975no9KlSoxdOhQvv/+e5o3b57zu/jss88GfJ6qVauG9B6j3b33dmXcuJ85dOg4SUn1mDhxIB07Bpo/ungilqhV9YSI3IszR18cMEFVfxGRu9ztY0WkPrAQZ3LVLBH5C9BWVfcFOjYy76T4Jk2axF/+8hfASUKTJk2iU6dOTJ8+nVtvvZUqVZxJxBMSEti/fz9bt27lyiuvBJwEHIrrr78+5/GKFSsYNWoUe/fu5cCBA1xyiTNf68yZM3n77bcBiIuLo0YN/ynmUlJSSElJ4ayzzgKclmVqairnn38+Dz30EMOHD6d///6cf/75J73+ddddx0cffcSIESP48MMP+fDDD6levTqVKlXij3/8I5dddpnfN4Jg+vbty+OPP069evX83hfA9OnT/fp29+3bx/79+8nIyOCWW24hNTUVEeH48dzr07169cp5r23btmXjxo0BE/V7771HcnIyu3btolu3bvTt25emTZv67fPqq68yZcoUADZv3kxqaiqtWrVi/fr13HfffVx22WX06dMnpPdZ2K/OBw4cYM6cOVx7be7csEePHs15fO211+Z8uKSkpDB16lReeskpqDpy5AibNm0CAp+PPXv2cMEFF+TUQickJAR9njZt2hQq9mh1+ukJvPTSxezceZCRI8+nQoWTP7xLQkRveFHVacC0POvG+jzegdOtEdKx0SgtLY2ZM2eyYsUKRITMzExEhBdffBFVPenrU35/vPHx8WRlZeUs561b9W3hDBkyhE8//ZSkpCQmTpzIrFmzQopVVRk5ciR33nnnSdsWLVrEtGnTGDlyJH369OGJJ57w23799ddz7bXXctVVVyEitGzZEnC6JGbMmMEHH3zAa6+9ltP6C6ZChQp07tyZv//97/zyyy98/vnnOduysrKYO3culSv7z7d73333ceGFFzJlyhQ2bNhAz549c7ZVrFgx53FcXFyBfbh169alU6dOzJs3zy9Rz5o1i+nTpzN37lyqVKlCz549OXLkCLVq1WLp0qV8/fXXvP7663z00UdMmDChwPe5ePFikpOTC9wvW1ZWFjVr1sxpXefl+zugqnzyySe0atXKb5958+YFPB+BfheDPU+syczM4tVX56EKw4ad67ftT38K/+TmNtaHrxs1fD/5+Pjjj7n55pvZuHEjGzZsYPPmzTRv3pzZs2fTp08fJkyYkNOHnJ6eTvXq1WnUqBGffvop4LSYDh06RNOmTVm5ciVHjx4lIyODGTNm5Pua+/fv57TTTuP48eO8917upOS9evXi3//+N+D0q+7bt8/vuEsuuYQJEybkdIls3bqVnTt3sm3bNqpUqcJNN93EQw89xM8//3zSa55++unExcXx9NNP57SCDxw4QEZGBv369WPMmDH5JphAHnzwQV544QVq167tt75Pnz689tprOcvZz5mRkUHDhk5hUHbffVEdOnSIxYsXc/rpp/utz8jIoFatWlSpUoXVq1fz008/AU5lSlZWFldffTVPP/10wPOT1yeffEJKSgqDBg0KOa7q1avTvHlzJk+eDDhJdOnSpQH3veSSS/jnP/+Z88G/ePHioM997rnn8t133/Hbb78B5HR9FPZ5olFqam5Fx8iRM1ixYmepx2CJOsImTZqU042R7eqrr+b999+nb9++DBgwgOTkZDp27Jjz9fKdd97h1VdfJTExkW7durFjxw4aN27MddddR2JiIoMHD87pngjk6aef5uyzz+biiy+mdevWOetfeeUVvv32Wzp06EDnzp355Rf/3qQ+ffpw4403cu6559KhQweuueYa9u/fz/Lly+natSsdO3bk2WefZdSoUQFf9/rrr+fdd9/Nufi3f/9++vfvT2JiIj169Mi5yDZ16tSTWuR5tWvXjltuueWk9a+++ioLFy4kMTGRtm3bMnas8wXtkUceYeTIkZx33nlFqnwBp4+6Y8eOdO7cmSFDhtC5c2e/7X379uXEiRMkJiby+OOPc8455wDOB1rPnj3p2LEjQ4YM4bnnngv4/C+//HJOed67777LzJkzKWyl0nvvvcf48eNJSkqiXbt2+V7ce/zxxzl+/DiJiYm0b9/er+QxkLp16zJu3DiuuuoqkpKScj5sC/s80SS7oiMxMbei49ixTF56qfSroCQcd9F4VXJysuadOGDVqlVlpj/NmEiJtr+z1NQ0br31M7+Su/j4cjz22Pk8+mj4+qJFZJGqntTfVeYHZTLGmGzZfdGPPjozp+QOCGtFRygsURtjDJFrRYfCEjXke0XbGFN80dC9qqoMGvQJixZtz1kX6Va0rzJ/MbFSpUqkpaVFxS+TMdEmezzqUOv9I0VEeP31fpQrJ8THl+PJJ3swf/5QTyRpsBY1jRo1YsuWLXh5wCZjoln2DC9ekpWliPgPwXr22Y149dW+nHdeE88k6GxlPlGXL1/eZp4wpgzJ7ou+887O/OEPSX7b7rmnaz5HRVaZT9TGmLIhb0XHL7/solevFjRoUC3SoRWozPdRG2Nin+/dhdlldwcOHOOHHzZGOLLQWIvaGBOzvFoXXViWqI0xMcnLddGFZYnaGBNTYqUV7csStTEmpuzbd5QXXvjRb9aVaGxF+7KLicaYmFKrVmXGjnUmoUhKqseCBUMZPbpn1CZpsBa1MSbK7dp1kLp1/af+GjiwNZMnX8uAAa2iOkFnsxa1MSYqZWZm8Y9/zKVp0zHMmLH+pO3XXNM2JpI0WKI2xkShNWvSuOCCiTz4YAqHD5/g9tunsn//0YIPjFKWqI0xUSN71pWkpLHMmZNbdlezZiV27ToUwcjCy/qojTFRYc2aNG677eS66FGjzg/rDOBeYInaGONpsVgXXViWqI0xnrV+/R5uvnlKmWxF+7JEbYzxtMWLd+Q8LkutaF92MdEY41ktWtTihRd6e3LWldJkLWpjjCdkZmaxaNF2unZt6Lf+7ru70Lt3C1q3rhOhyCLPWtTGmIhbs8YZL7p79wksX/6737Zy5aRMJ2mwRG2MiSDfuugff9zM8eNZDBnyGcePZ0Y6NE+xrg9jTETkVxd9+eVnRjAqb7JEbYwpVVYXXXiWqI0xpSa/VnS0jxcdbpaojTGl4t13lzF06OfWii4CS9TGmFJx5pm1OXbMuUhorejCsURtjCkVXbs25OGHu/HVV2utFV1IoqqRjqHUJCcn68KFCyMdhjExb82aNNatS+fSS1v6rT969AQiYq3ofIjIIlVNzrve6qiNMSXGty560KBP2Lp1n9/2ihXjLUkXQUQTtYj0FZFfRWStiIwIsF1E5FV3+zIR6eSzbYOILBeRJSJizWRjIiz77sJhw1I4cuQEGRlHueeeaZEOKyZErI9aROKA14GLgS3AAhGZqqorfXa7FGjp/pwN/Nv9N9uFqrq7lEI2xgQQrC569OiekQsshkTyYmJXYK2qrgcQkQ+AKwDfRH0F8LY6Hek/iUhNETlNVbeXfrjGmLysLrp0RDJRNwQ2+yxvwb+1nN8+DYHtgAIpIqLAf1R1XKAXEZE7gDsAmjRpUjKRG1PG2d2FpSuSiVoCrMtbghJsn/NUdZuInAp8IyKrVfX7k3Z2Evg4cKo+ihOwMcZxww2f8PHHuV9+rRUdXiFdTBSRpiLS231cWUSqlcBrbwEa+yw3AraFuo+qZv+7E5iC05VijCkFf/hDYs7jpKR6LFgwlNGje1qSDpMCE7WIDAU+Bv7jrmoEfFoCr70AaCkizUWkAnADMDXPPlOBm93qj3OADFXdLiJVsz8sRKQq0AdYUQIxGWNCMGBAK269tWOZnnWlNIXS9XEPTmt1HoCqprrdDcWiqidE5F7gayAOmKCqv4jIXe72scA0oB+wFjgE3OoeXg+YIiLZ7+F9Vf2quDEZY/xl90UnJtajV68WftvGjx+A+zdowiyURH1UVY9l/4eISDwn9yUXiapOw0nGvuvG+jxWnA+KvMetB5JKIgZjTGC+FR1NmtRgxYo/Ua1axZztlqRLTyh91N+JyKNAZRG5GJgMfB7esIwxkZJ31hWATZsyGDPmpwhHVnaF0qIeAdwOLAfuxGkBvxnOoIwxkRGsLnr48O4RjKxsCyVRV8bpP34Dcu4orIzTZ2yMiQFWF+1toSTqGUBv4IC7XBlIAbqFKyhjTOmxuwu9L5REXUlVs5M0qnpARKqEMSZjTCk5fPg43btPYNeu3C/I1or2nlAuJh7MM2pdZ+Bw+EIyxpSWypXL5wycFB9fzuqiPSqUFvVfgMkikn3X4GnA9eELyRhTmu66K5lfftnJ0KGdLUF7VIEtalVdALQG/gTcDbRR1UXhDswYU7LWrEnjooveYvny3/3WlysnvP76ZZakPSzUiQO6AInAWcAgEbk5fCEZY0qSb130t99uYMiQzzh+PDPSYZlCKLDrQ0TeAU4HlgDZ/7sKvB3GuIwxJSBQRceyZb8zd+4WLrigaQQjM4URSh91MtBWy9IsuMZEOauLji2hJOoVQH2cwfqNMR5nddGxJ5REXQdYKSLzgaPZK1V1QNiiMsYUmrWiY1coiXp0uIMwxhTfypW7ePjhb8jMdHoprRUdO0Ipz/sO2ACUdx8vAH4Oc1zGmELq0KEejzxyHmCzrsSaUKo+huJMDpuAU/3REBgL9ApvaMaYYI4cOUGlSv5/wk8+2YP69U/hrruSLUHHkFDqqO8BzgP2gTPDC1DsGV6MMUWTXRfdosUrbN26z29bxYrx3H//2ZakY0woifqoqh7LXijJGV6MMYWTmppGjx4TGTYshe3bDzB06OdY5WzssxlejIkC2a3oxMSxfmV327btJz3dxkiLdTbDizEeZ3XRJmiidmdzeUtVbwLeKJ2QjDFgddEmV9BEraqZIlJXRCr49lMbY8LLWtHGVyhdHxuAH0VkKnAwe6Wq/iNcQRlT1v36626/JG2t6LItlIuJ24Av3H2r+fwYY8Lk8stbcdNNiTbrigFAQi3tEZGqqnqw4D29Kzk5WRcuXBjpMIzxk5mZxebN+2jWrKbf+vT0w2zalGEJugwRkUWqmpx3fYEtahE5V0RWAqvc5SQR+VcYYjSmzMmui77ggv9j376jftsSEipbkjZAaF0fY4BLgDQAVV0KXBDOoIyJdXnrojdv3sfDD6dEOizjUaFcTERVN4uI7yqbx8eYIsqvouO006qhquT5WzMmpES9WUS6ASoiFYD7cbtBjDGhs7poU1ShJOq7gFdwRs3bAqTgDNRkjAmR1UWb4sg3UYvIC6o6HLhQVQeXYkzGxJR//3sBw4alWCvaFFmwi4n9RKQ8MLK0gjEmFqmSk6StLtoURbCuj6+A3UBVEdkHCM7wpgKoqlYvhfiMiXp33ZXMxx+vJD39sLWiTZHke8OLiFRU1aMi8pmqXlHKcYWF3fBiwm3NmjQyM7No06au3/pduw5So0Yl64s2QRXlhpe57r/7guxjjCG3LjopaSyDB/+X48f9K1jr1q1qSdoUWbCujwoicgvQTUSuyrtRVf8bvrCMiR55KzoWL97Bc8/N5oknekQ4MhMrgrWo7wLOAWoCl+f56V8SLy4ifUXkVxFZKyIjAmwXEXnV3b5MRDqFeqwx4ebbis470t2AAa0iGJmJNfm2qFV1NjBbRBaq6viSfmF3UoLXgYtx6rMXiMhUVV3ps9ulQEv352zg38DZIR5rTNjkVxc9atT5jBxpddGmZAWro75IVWcCe8LU9dEVWKuq693X+wC4AvBNtlcAb6tzxWcsauIAABw3SURBVPMnEakpIqcBzUI41pgSZ3cXmkgI1kfdA5iJ09WRlwLFTdQNgc0+y1twWs0F7dMwxGMBEJE7gDsAmjRpUryITZmWmZlF797vMGvWhpx1dnehKQ3Buj6edP+9NUyvHWjkmby1gvntE8qxzkrVccA4cMrzChOgMb7i4srRrVujnERtrWhTWgqa3LYVTmu0tbtqFTBOVdeUwGtvARr7LDfCmU0mlH0qhHCsMSXuiSd68OWXaxkwoJW1ok2pybfqQ0TOBWYBB3BapG/gzJk4S0TOKYHXXgC0FJHm7qh8NwBT8+wzFbjZrf44B8hQ1e0hHmtMkWVmZvHKKz+xZYv/bQQVK8Yzb94fGT26pyVpU2qCtaifAAap6iyfdZ+KyEzgSZyKjCJT1RMici/wNRAHTFDVX0TkLnf7WGAa0A9YCxwCbg12bHHiMSabb0XHV1+tY9q0G/3GiC5f3hK0KV3BbiFfo6pn5rPtV1WNukJRu4XcBJNfRcdHH13Dtde2i2BkpqzI7xbyYC3q/UG2RfUkt8bkFawu+oorWgc50pjwC5aoG4vIqwHWC055nDFRz+qiTTQIlqgfDrLN+g9M1LNZV0y0CFZH/VZpBmJMadq0KYOOHcdy+LC1oo33BRuUyZiY1aRJjZwLhDbrivG6UCa3NSYmjRlzCTt3HuS553pZgjaeZi1qE/PWrEnjuusmk5FxxG99rVqV+fLLwZakjecVmKhF5EURqS4i5UVkhojsFpGbSiM4Y4rDd7zoyZNX8tBDKZEOyZgiCaVF3UdV9+FMFrAFOJPgFSHGRNyaNWn06DGRYcNScsru3nprKb/9tifCkRlTeKEk6vLuv/2ASaqaHsZ4jCmWYLOuzJ8/lObNa0UwOmOKJpSLiZ+LyGrgMHC3iNQFjhRwjDGlzuqiTawqMFGr6ggReQHYp6qZInIQZzYVYzzB7i40sS7U8ryGwMUiUsln3dthiMeYQpsyZTXDhuVeKCxzcxeqwu65kHkY6l0EEmheDRPNCkzUIvIk0BNoizPs6KXAbCxRG4+4+uo29OrVnBkzfit7rehdc2DJcNg121k+ZyK0uCWiIZmSF8rFxGuAXsAOd1quJKBiWKMyJoisLP+heUWE8eMH8NRTPcvO3YUZq+D7K+Gb83KTNMCWTyMXkwmbUBL1YVXNAk6ISHVgJ9AivGEZc7Lsio7u3Sdw/Him37amTWvy+OM9Yr+r49BWmDcUprUPnJTTFpR+TCbsQknUC0WkJs5UXIuAn4H5YY3KmDx866Lnzt3C//t/P0Q6pNJ1bC8sGQmfnwHr3gTNyt3WdBDEVXEeH94Kh7dHJkYTNgUmalW9W1X3ulNjXQzcEsaZyY3xk19d9BdfpHLiRFaQI2NE5hFY9XeY2gJWPu8sZ6vfB/ougvPeh4SzctenLyr9OE1Y5XsxUURaq+pqEekUYFsnVf05vKGZsq6guuj4+BgeqiYrEza8C8seh0Ob/bcldIaOz0P93j7rkmHXj87jtIXQsH/pxWrCLljVxzDgDuDvAbYpcFFYIjJlXpmui1aFbdNgyQjIWOG/7ZTTIelZaHItSJ4PqYQuuY/TrZ861gSbOOAO998LSy8cU9YFm7sw5uuid//klNrt/N5/faVTof0TcPpQiKsQ+NjaPvOhpi90Er7VU8eMoHXUItIUOKiqu0XkHKA7sFZVrQbIhMW77y47aYyOmG9FZ6yGpY/Clin+6+NPgTYPQethUL5a8Oeo1hLiq8GJ/XBkJxzaAlUbhy9mU6ry7eQTkSeAmcBPIvIMMAaoA/xZRMaUUnymjHnssfNp3/5U4uPLMXp0jM+6cmgbzLvDLbXzSdISD2feC5evhQ5PFpykwekKSeicu2zdHzElWIv6BqANUAXYBNRX1UMiEg8sKY3gTGzLzMxi//5j1KyZOzJBxYrxvPvulagSuwn62F5Y+SL8Osa57dtX0xsg8Rmodnrhn7d2F9g5y3mcthAaX1XsUI03BEvUR1T1GHBMRNap6iEAVT0hIsdKJzwTq1JT07j11s845ZQKfPnlYMSnPzUpKUYTdOYRWPMv+OVZOJZntOD6vZ1KDt9WcWEl5OmnNjEjWKKuKSJXAQJUdx/jLtcIe2QmJgWq6JgwYTG3335SFWjsyMqEDe+5pXab/LfVOgs6vgCnXVz817ELijErWKL+Drjcffy9z+PsZWMKJbsVnbeiIz39cJCjopgqbPsSlo6Avcv9t53SAhKfhabXnVxqV1RVm0OFBKe1fmwPHFhftC4U4znByvPs7kNTIspkXfTueW6p3Xf+6yvWdUrtzrgj/1K7ohJxuj92uEO+pi+0RB0jQh2P2pgiya8VHbOzruz7FZY+Bps/8V8fXxVaPwRtHgytiqOoaudJ1E2vD99rmVJjidqEhaoyZsxPZacVfXg7LP+rO2CSz8h+Eg9n3AntH4fK9cIfh+8FxTS7oBgrLFGbsBARli3bmZOkY7YVfSwDVr0Iq18+udSuyfWQ9AxUO6P04qnteyv5ImeUvZLqAzcRE1KiFpFuQDPf/VXVZngxQb388iWkpKyjbt0qsdeKzjwKqW6p3dE0/231LnIqOXyrMEpL5YZQqR4c+d25S3HfGqjRuvTjMCUqlKm43gFOx7nJJfs7nWJTcRkfqalpnHpqVWrUyL15pWbNSsyYcTMtWtSKnVZ0ViZsfN8ptTu40X9brY5Ogq5/ceTK4rIvKG77n7OcvtASdQwIpUWdDLRVVS1wT1Pm+FZ0DB7cgTffHOC3vXXrOhGKrISpwvavnFHt9i7z31a1udPF0fQGb3Qz+CbqtAXQ/KbIxmOKLZREvQKoD9i0EcZP3oqO8eMXc+21bbnkklLsky0Nu+e7pXaz/NdXrOOW2t1Z8qV2xeHXT20XFGNBKIm6DrBSROYDR7NXquqA/A8xsSxYXXT9+qdEMLIStm+NW2r3sf/6+KrQ+kG31K56ZGILxvc29D2LIesElLO6gWgWyv/e6JJ+URFJAD7EuUC5AbhOVfcE2K8v8AoQB7ypqs+760cDQ4Fd7q6Pquq0ko7TnKxM1EUf3g7Ln4J1bwQotbvDLbXz8IXRyvWhSiNnqNPMw7BvFdTsEOmoTDEUmKhV9buC9imCEcAMVX1eREa4y8N9dxCROOB1nHkatwALRGSqqq50d3lZVV8KQ2wmgDJxd+HxfbDyb7D6H5B5yH9bk+ucUe2qt4xMbIWVkOwkanD6qS1RR7UCr3yIyDkiskBEDojIMRHJFJF9xXzdK4C33MdvAQMD7NMVZ5KC9e4ofh+4x5lStmfP4ZwZwH3rop98MkbGi848CqvHOBPI/vKMf5KudyFcMh+6fxg9SRqsnzrGhNL18RrO2NSTcSpAbgaK+xtbT1W3A6jqdhE5NcA+DQHfWT23AGf7LN8rIjcDC4EHA3WdAIjIHThzP9KkSZNihl021axZicqVy+csx0wrWrNgQ3ap3Qb/bTWT3FHt+kTnCHR2h2JMCamWSFXXAnGqmqmq/wf0LOgYEZkuIisC/ITaKg7015FdIvhvnNrujjjVKIEm4M2OfZyqJqtqct26dUN8aeNLRHjzzctJSKgcG61oVdj2FXzZCeb+wT9JV20G574Ll/4MDS6JziQN/hcU9y6FTBtCPpqF0qI+JCIVgCUi8iJOYqxa0EGq2ju/bSLyu4ic5ramTwN2BthtC+A76VsjYJv73L/7PNcbwBchvA8TgszMLN55Zxk33tjB78Jg06Y1+e23P1O9esUIRlcC0hY4pXa/f+u/vmJtaPc4tLwL4qL8PYLzfk5p4Qx1mnUMMpYXb1ICE1GhtKj/4O53L3AQJ3leXczXnQrc4j6+BfgswD4LgJYi0tz9oLjBPQ43uWe7EqfW2xRTamoaPXpM5NZbP+O55344aXtUJ+l9qTD7Ovi6q3+SjqviVHEMWA+t/xwbSTqbdX/EjAITtapuxOmGOE1V/6qqw9yukOJ4HrhYRFJxqjqyy+4aiMg093VP4Hw4fA2sAj5S1V/c418UkeUisgy4EHigmPGUaZmZWbz88lwSE8fmlN0988wPrFgR6ItOlDm8AxbcDf9rA5sm566XOGj5JxiwFhKf8mY9dHHZ1FwxI5SxPi4HXgIqAM1FpCPwVHFueFHVNKBXgPXbgH4+y9OAk+qjVfUPRX1t4y9YXfSZZ9aOYGTFdHwfrHoJVv09QKndtW6p3ZmRia20+A4KlWazkkezUG946QrMAlDVJSLSLGwRmVIRs3XRmUdh7X9gxdNwdLf/tlN7OpUcdbpGJLRS59snnbECThyG+MqRi8cUWSiJ+oSqZki0Xv02J4nJuws1CzZMckvtfvPfVjPRLbWL4iqOoihfHaq3cmad0Uyn+qPOOZGOyhRBSIMyiciNQJyItATuB+aENywTLnPmbKZXr7djpxWtCttTnAlk9yzx31a1qdPF0exGb4xqFwkJyU6iBueCoiXqqBTKb+99QDucAZkmAfuAv4QzKBM+yckNaNkyAYiBuwvTFsLM3jCrr3+SrlgbOr0M/X91hvgsq0ka8lxQtH7qaBXKWB+HgMfcHxPlKlSIY+LEgdxxx+e8+eaA6EzQ+9c6o9pt+sh/fVxlaD0M2jwMFWpEJjavsVvJY0K+iVpEpgY70IY59b41a9IYP/5nnn++N77XGDp1Oo0FC4YSddcdDv8OK56CteNAc7tukDg4/Y/O2NBVGkQuPi+q1dH5RqFZkLEKjh+A8jE0FG0ZEaxFfS7OWBuTgHkEvqXbeFDeio4zzkhg6FD/u9KiKkkf3+eU2a3+O5w46L+t8dWQ9Kxz0cycLL4qVG/rVH2gzvjUp54f6ahMIQXrvKsPPAq0xxkT+mJgt6p+F6ahT00JWLMm7aSR7kaMmMHBg1E41kPmMfj1nzD1DKcl7ZukT+0BfebC+R9bki6I1VNHvXwTtTsA01eqegtwDrAWmCUi95VadCZk2XcXJiWN9Su7S0qqx4wZN1O1qoemiipIdqndF61h0f1wdFfutpodoOc06PWtVTCEKsH6qaNd0IuJIlIRuAwYhDMby6vAf8MflimMNWvSuO22GKmL3v6NM2jSnsX+66s0cSeQvRHKRdH78QK7lTzqBbuY+BZOt8eXwF9V1QY+8piYurswfREsHg6/z/BfXyEB2o9yxuWIqxSZ2KJdrURnGjE9AftT4dheqFAz0lGZQgjWov4Dzmh5ZwL3+1x8EkBVNQZHsYkuL7zwI489NjNnOSpb0fvXwtJRsOlD//VxlaH1A9DmESu1K664Ss7dmXt+dpbTF0H9k4baMR6Wb6JW1TJ8l0B0uPvuLrz++gK2bdsffa3ow78743Gs/U+AUrvbof2TVmpXkmon+yTqhZaoo4zNIR/FatasxBtvXM78+VujpxV9fL9bavdSgFK7qyDxWajROjKxxbKEZGCc89jGpo46lqijQHZf9Lp1e3jttX5+2/r1a0m/flEw6WrmMedGlRVP+VdxANQ9H8560ao4wsluJY9qlqg9Lm9Fx2WXteTSS6MgMWfTLNj4ESx7zJkWyleN9tDxeWjQr2yNahcJNdtDuYqQdRQOboQju6CSzSEaLawf2qPyq4t+/fUoag3tmA5fdYE5g/yTdJXGcM5EuHQJNLzMknRpKFfeuZ08W/qiyMViCs1a1B60Zo0zXvScOf510aNGnc/IkVFw+2/6z7BkBOz4xn99hVrQ7jE48x4rtYuEhGRIm+c8Tl8IDfpGNh4TMkvUHpKZmcUrr8zjsceitC56/zpYNgo2fuC/Pq4StPoLtB1u9buRVDsZUt3Hdit5VLFE7REFtaI9XdFxZKdTapc6Nk+pXTlocRt0GA1VGkYsPOOyW8mjliVqj3j44W/8knRUtKKP74fV/3AmkT1xwH9boyudUe1qtIlMbOZk1VtDXBVnst/D2+DQNqtVjxJ2MdEj/vnPS6lWrQLx8eUYPdrjs65kHoM1r8PnZ8Dy0f5Jum53uHgOXPBfS9JeUy4OEjrlLlurOmpYizoCMjOzyMpSypfP7c5o0qQGb701kObNa3k3QWc7lg6LH3FaZtlqtHNL7ayKw9MSkmHXbOdx+kJoZPN/RANrUZey7PGin3nm+5O2XXllG+8naYDK9aHNg87jKo3gnP+DS5dCw/6WpL3Od2ouu0MxaliLupTkHelu3rytDBzYmrPOOi3SoRVNm4ecke3OuBPiK0c6GhOqvEOeqtqHaxSwFnUpCDTrCsDChdsiGFUxla8Orf9iSTraVDvD+b8D51b+Q5siG48JiSXqMAo268qCBUNPmsfQmLCTcv6tauv+iAqWqMMkUCs6Pr4cTz7p8YoOE/tsxpeoY33UJSymZl0xsam2JepoY4m6hGVmKm+9tdSvFR11s66Y2ObX9TEfso47gzYZz7KujxJWoUIcEycOJD6+XE5f9OjRPS1JG++o2gwqu7f0H9+XW1dtPMsSdTFt2pSBqvqt69ixPt988wfrizbeJAKNrshd3vJZ5GIxIbFEXUTZFR2tWr3Gm2/+fNL2nj2bWSvaeJdfov7Uqac2nmWJugjyVnQ8+GAKmzZlRDosY0J3as/ceuqDG2HvsoiGY4KzRF0I+dVFt2hRi4MHj0UwMmMKKa6CMwVaNuv+8DRL1CEqqC66TRubf85EmYZ5uj+MZ0UkUYtIgoh8IyKp7r+18tlvgojsFJEVRTm+JBR0d6FVdJio1eDS3LK8PYvhoN1O7lWRalGPAGaoaktghrscyEQg0MRuoR5fLJs3Z9jdhSZ2VagBp16Yu2zdH54VqUR9BfCW+/gtYGCgnVT1eyC9qMcXV/XqFdm4MfciobWiTcxp7POnY4nasyKVqOup6nYA999Tw3W8iNwhIgtFZOGuXbsK9SI1alRi3Lj+1oo2sauhz8QBO7+DY3siF4vJV9huIReR6UCgrPZYuF4zEFUdB4wDSE5OLnSx6KWXtmT9+vtp3LhGicdmTMRVaehMepu+wJmYeOs0aD440lGZPMKWqFW1d37bROR3ETlNVbeLyGnAzkI+fXGPLxRL0iamNbrCSdQAWz+zRO1Bker6mArc4j6+BShs51hxjzfGZPO9S3Hbl5B5NHKxmIAilaifBy4WkVTgYncZEWkgItOydxKRScBcoJWIbBGR24Mdb4wpghrt4JTTnccnDsDvMyMbjzlJRIY5VdU0oFeA9duAfj7LgwpzvDGmCLIHaVr9D2d5y2dOjbXxDLsz0RgDjfKU6WlW5GIxJ7FEbYyBOt2gYh3n8ZEdkLYgsvEYP5aojTFQLg4a9s9dtrE/PMUStTHGkbf7w3iGJWpjjKP+xRBX2Xm8bxXsWxPZeEwOS9TGGEd8FTitT+6ytao9wxK1MSaX7xjVWy1Re4UlamNMrob9Qdy0sGsOHP49svEYwBK1McZXpbpQ5zx3QWHbFxENxzgsURtj/PnNUG7dH15gidoY4883Ue/4Bk4cjFwsBrBEbYzJq9oZzkBNAJlHYHtKZOMxlqiNMQH43vyy1fqpIy0io+cZYzyuyTVwZKfTDVLfBqqMNEvUxpiT1eoIZ4+LdBTGZV0fxhjjcZaojTHG4yxRG2OMx1miNsYYj7NEbYwxHmeJ2hhjPM4StTHGeJyoaqRjKDUisgvYWMjD6gC7wxBOOFnMpcNiLh1lKeamqlo378oylaiLQkQWqmpypOMoDIu5dFjMpcNitq4PY4zxPEvUxhjjcZaoCxaNAx5YzKXDYi4dZT5m66M2xhiPsxa1McZ4nCVqY4zxuDKdqEWkr4j8KiJrRWREkP26iEimiFxT2GNLWjFj3iAiy0VkiYgsLJ2IC45ZRHqKSIYb1xIReSLUYz0asyfPs0/cS0TkFxH5rjDHejBmT55nEXnY5/dihft3mBDKsflS1TL5A8QB64AWQAVgKdA2n/1mAtOAawpzrJdidtdvAOp47TwDPYEvivp+vRSzx89zTWAl0MRdPjUKznPAmL18nvPsfzkws7jnuSy3qLsCa1V1vaoeAz4Argiw333AJ8DOIhxb0ooTc6QU51x5/Tx7SSgx3wj8V1U3AajqzkIc67WYI6Ww52oQMKmIx+Yoy4m6IbDZZ3mLuy6HiDQErgTGFvbYMClOzAAKpIjIIhG5I2xR+gv1XJ0rIktF5EsRaVfIY0tacWIG757nM4FaIjLLje3mQhwbDsWJGbx7ngEQkSpAX5xGU6GOzassz5koAdblrVUcAwxX1UwRv91DOTYcihMzwHmquk1ETgW+EZHVqvp9OAL1EUrMP+OMcXBARPoBnwItQzw2HIoTM3j3PMcDnYFeQGVgroj8FOKx4VDkmFV1Dd49z9kuB35U1fQiHOunLCfqLUBjn+VGwLY8+yQDH7gJrw7QT0ROhHhsOBQ5ZlX9VFW3gfP1UUSm4HwVC/cvdoExq+o+n8fTRORfIlInlGPDpMgxq+pur55nd5/dqnoQOCgi3wNJIR4bDsWJeY2Hz3O2G8jt9ijssf5KsyPeSz84H1Lrgebkduy3C7L/RHIvJhbqWI/EXBWo5vN4DtDXCzED9cm9+aorsAmn9eHZ8xwkZi+f5zbADHffKsAKoL3Hz3N+MXv2PLv71QDSgaqFPTbQT5ltUavqCRG5F/ga52rsBFX9RUTucrcH6uMNeqyXYwbqAVPclnY88L6qfuWRmK8B/uR+WzkM3KDOb7aXz3PAmEXEs+dZVVeJyFfAMiALeFNVVwB49TznF7OItMCj59nd9UogRZ1vAkGPDeV17RZyY4zxuLJc9WGMMVHBErUxxnicJWpjjPE4S9TGGONxlqiNMcbjLFGbmCQiV4qIikjrSMfiS0TmRDoGE30sUZtYNQiYjXN3WFiISFxhj1HVbuGIxcQ2S9Qm5ojIKcB5wO34JGoRecQdv3ipiDzvrjtDRKa7634WkdPd8Y+/8DnuNREZ4j7eICJPiMhs4FoRGSoiC9zjP3EH4kFE6onIFHf9UhHp5q4/4PO8D7vHLhORv7rrqorI/9xjVojI9WE/YcbzyuydiSamDQS+UtU1IpIuIp1w7swcCJytqoeyB3IH3gOeV9UpIlIJp/HSOPDT5jiiqt0BRKS2qr7hPn4G58Phn8CrwHeqeqXb8j7F9wlEpA/OIE5dcW49nyoiFwB1gW2qepm7X43inQoTCyxRm1g0CGcUQXDG/B2Ek4D/T1UPAahquohUAxqq6hR33RGAAKMO5vWhz+P2boKuiZOMv3bXXwTc7D5vJpCR5zn6uD+L3eVTcBL3D8BLIvICzsQEP4T4nk0Ms0RtYoqI1MZJku1FRHHGVFCcMYHzjpeQX0Y+gX+3YKU82w/6PJ4IDFTVpW73SM9QQwWeU9X/nLRBpDPQD3hORFJU9akQn9PEKOujNrHmGuBtVW2qqs1UtTHwG85IZrf59CEnqDNU6RYRGeiuq+hu3wi0dZdr4IyFnJ9qwHYRKQ8M9lk/A/iT+7xxIlI9z3Ffu/Gc4u7TUEROFZEGwCFVfRd4CehUnJNhYoMlahNrBgFT8qz7BGgATAUWisgS4CF32x+A+0VkGc5QmfVVdTPwEc6Ibe+R2z0RyOPAPOAbYLXP+j8DF4rIcmAR4DsDDKqaAryPMxD+cuBjnKTfAZjvxvgY8Ezob93EKhs9zxhjPM5a1MYY43GWqI0xxuMsURtjjMdZojbGGI+zRG2MMR5nidoYYzzOErUxxnjc/wcbHxctzxzLFgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(accuracies, mean_difference, lw=3,color='orange', label=\"Accuracies vs. Mean Bias Difference\")\n", + "axe.set_xlabel(\"Accuracies\")\n", + "axe.set_ylabel(\"Mean Bias Difference\")\n", + "axe.set_title(\"Accuracies vs. Mean Bias Difference\")\n", + "axe.plot([0.4, 0.7], [-0.1, 0.1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Interpretation\n", + "\n", + "The graph shows that, for almost all the Tao values, the accuracy goes down as the tao goes up. The mean bia difference is loves when tao is 0.88, coming all the way down to -0.015 from -0.19 biased model. We can clearly observe a trade-off between the accuracy score and bias reduction, as expected." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Other Metrices to Use\n", + "1. FDR (False Discovery Rate): is a method of conceptualizing the rate of type I errors in null hypothesis testing when conducting multiple comparisons. (from Wikipedia) The idea of false discovery rate is to make the total number of FALSE discoveries (false positive + false negative) LOWER as a percentage of the overall discoveries." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### FDR Code Implementation\n", + "\n", + "We implement the FDR metric to assess the \"fairness\" of the model from another bias related metric. Below we compare the best model from the iterations above, with the unconstrained model. This proves the reduction in bias. " + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy : 3254 14653 0.777929434245547\n", + "SR tau : 0.24060062422550277\n", + "FPR tau : 0.43600739741090616\n", + "FNR tau : 0.8609189723320158\n", + "TPR tau : 0.31004901960784315\n", + "TNR tau : 0.980340417865866\n", + "AR tau : 0.8224361467717171\n", + "FDR tau : 0.4288194444444444\n", + "FOR tau : 0.3872744450835204\n", + "PPR tau : 0.4588815789473684\n", + "NPR tau : 0.8136439105306991\n", + "0.4288194444444444\n" + ] + } + ], + "source": [ + "from aif360.algorithms.inprocessing.celisMeta.utils import getStats\n", + "\n", + "#Unconstrained Model\n", + "mfc = MetaFairClassifier(tau=0, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)\n", + "dataset_bias_test = mfc.predict(dataset_orig_test)\n", + "\n", + "\n", + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in list(dataset_bias_test.labels)]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "x_control_test = pd.DataFrame(data=dataset_orig_test.features, columns=dataset_orig_test.feature_names)[\"sex\"]\n", + "\n", + "acc, sr, unconstrainedFDR = getStats(y_test, predictions, x_control_test)\n", + "print(unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy : 6660 14653 0.5454855660956801\n", + "SR tau : 0.9683859899039216\n", + "FPR tau : 0.9120522139618913\n", + "FNR tau : 0.6842461122379986\n", + "TPR tau : 0.9711585968379447\n", + "TNR tau : 0.8803995524643298\n", + "AR tau : 0.7673932778424438\n", + "FDR tau : 0.6863407379682039\n", + "FOR tau : 0.2292871682218505\n", + "PPR tau : 0.3786416244779234\n", + "NPR tau : 0.9379752173348904\n", + "0.6863407379682039 0.4288194444444444\n" + ] + } + ], + "source": [ + "# the best model from the iterations above had a TAO of 0.88.\n", + "mfc = MetaFairClassifier(tau=0.88, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)\n", + "dataset_bias_test = mfc.predict(dataset_orig_test)\n", + "\n", + "predictions = list(dataset_bias_test.labels)\n", + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in predictions]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "x_control_test = pd.DataFrame(data=dataset_orig_test.features, columns=dataset_orig_test.feature_names)[\"sex\"]\n", + "\n", + "acc, sr, fdr = getStats(y_test, predictions, x_control_test)\n", + "print(fdr, unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "assert(fdr >= unconstrainedFDR)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### The statement above checks that the fdr with the balancing TAO gives better results than the unconstrained model in terms of fairness. " + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tau': 0.88, 'sensitive_attr': 'sex'}" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mfc.__dict__['_params']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Comparison with a Logistic Regression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### The comparison of the model at hand with the logistic regression comes from their similarity. Similar to Logistic Regression Classifier, Meta Fair Classfier calculates loss by trying to minimize a term (bias). And therefore, the algorithms work very similarly. We do not preprocess the data for the model to be similar to the Meta Fair Classifier example.\n", + "\n", + "#### In the cells below you will see the logistic regression with the sensitive vairable \"sex\" and without the variable (variable was deleted). " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## With sensitive variable \"sex\"" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score: 0.793\n" + ] + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "\n", + "df_test = pd.DataFrame.from_records(dataset_orig_test.features)\n", + "df_test[\"labels\"] = np.concatenate(dataset_orig_test.labels, axis=0)\n", + "y_test = df_test[\"labels\"]\n", + "X_test = df_test.drop([\"labels\"],axis=1) \n", + "\n", + "df_train = pd.DataFrame.from_records(dataset_orig_train.features)\n", + "df_train[\"labels\"] = np.concatenate(dataset_orig_train.labels, axis=0)\n", + "y_train = df_train[\"labels\"]\n", + "X_train = df_train.drop([\"labels\"],axis=1)\n", + "\n", + "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n", + "preds = clf.predict(X_test)\n", + "print(\"Accuracy Score:\", round(accuracy_score(preds, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Feature Importance (Permutation Test)" + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
WeightFeature
\n", + " 0.0649\n", + " \n", + " ± 0.0030\n", + " \n", + " \n", + " x1\n", + "
\n", + " 0.0242\n", + " \n", + " ± 0.0015\n", + " \n", + " \n", + " x0\n", + "
\n", + " 0.0199\n", + " \n", + " ± 0.0011\n", + " \n", + " \n", + " x2\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 94, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import eli5\n", + "from eli5.sklearn import PermutationImportance\n", + "\n", + "perm = PermutationImportance(clf, random_state=0).fit(X_train, y_train)\n", + "eli5.show_weights(perm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Without sensitive variable \"sex\"" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score without the sensitive variable: 0.778\n" + ] + } + ], + "source": [ + "y_train = df_train[\"labels\"]\n", + "X_train = df_train.drop([\"labels\", 2],axis=1)\n", + "y_test = df_test[\"labels\"]\n", + "X_test = df_test.drop([\"labels\", 2],axis=1) \n", + "\n", + "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n", + "preds = clf.predict(X_test)\n", + "print(\"Accuracy Score without the sensitive variable:\", round(accuracy_score(preds, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Feature Importance (Permutation Test)" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
WeightFeature
\n", + " 0.0584\n", + " \n", + " ± 0.0031\n", + " \n", + " \n", + " x1\n", + "
\n", + " 0.0219\n", + " \n", + " ± 0.0027\n", + " \n", + " \n", + " x0\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 96, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "perm = PermutationImportance(clf, random_state=0).fit(X_train, y_train)\n", + "eli5.show_weights(perm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Correlation Among Variables\n", + "#### \"2\" is the SEX variable" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012labels
01.0000.0320.0910.231
10.0321.0000.0120.335
20.0910.0121.0000.214
labels0.2310.3350.2141.000
\n", + "
" + ], + "text/plain": [ + " 0 1 2 labels\n", + "0 1.000 0.032 0.091 0.231\n", + "1 0.032 1.000 0.012 0.335\n", + "2 0.091 0.012 1.000 0.214\n", + "labels 0.231 0.335 0.214 1.000" + ] + }, + "execution_count": 97, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "round(df.corr(),3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpretation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### From the analysis and comparison above we come to a very important conclusion, that serves as the krux of our bias/fairness discussion. As we can see above, the logistic regression model, that works similar to Meta Fair Classifier (through minimizing an error term), achieves a much higher accuracy than MFC. Accuracy of the unconstrained model in MFC is almost the same as the accuracy of the Logistic Regression without the SEX variable. We can also see this in the Global Feature Importance graphs of the two logistic regression models. The reason both of these models have higher and very similar accuracies is because SEX is actually not a very preditive variable. This is proven by the global feature importance diagrams (0.0199) concoted through a Permuttion test.\n", + "\n", + "### While the imputation of the sensitive variable serves as a loss of data, we can see that it also erases bias. However, erasing sensitive variables is not enough to erase bias in most of the cases. To make sure that there is no inter variable dependency such that the deleted variable is highly corraleted with another variable, which vauses a indirect bias, we created a correlation graph above. On it, we see that none of the other two predictors are strongly correlated with the sensitive variable \"Sex\". \n", + "\n", + "### Overall it is important to keep these strategies in mind and decide WHEN and HOW to use IBM's algorithms. In some cases such as the one I just walked you through, using a logistic regression and imputing the SEX variale yields less bias and higher accuracy than any IBM algorithm." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/IBM_AIF360_Module.ipynb b/examples/IBM_AIF360_Module.ipynb new file mode 100644 index 00000000..e5c71fa4 --- /dev/null +++ b/examples/IBM_AIF360_Module.ipynb @@ -0,0 +1,767 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Understanding Fairness Through Analyzing IBM's AIF360" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Welcome to a introductory module to IBM's AIF360. AIF360 is a AI Fairness package that allows users to evaluate their models for bias during preprocessing, during processing and post-processing. Furthermore, the package allows interference with the model in the three stages mentioned afore, and also serves as a tool to create fairness in your own AI or ML model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Importing Packages and Data" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: aif360 in /opt/conda/lib/python3.7/site-packages (0.2.3)\n", + "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from aif360) (0.21.3)\n", + "Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from aif360) (1.3.2)\n", + "Requirement already satisfied: pandas>=0.23.3 in /opt/conda/lib/python3.7/site-packages (from aif360) (0.25.3)\n", + "Requirement already satisfied: numpy>=1.16 in /opt/conda/lib/python3.7/site-packages (from aif360) (1.17.3)\n", + "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->aif360) (0.14.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23.3->aif360) (2019.3)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23.3->aif360) (2.8.1)\n", + "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas>=0.23.3->aif360) (1.13.0)\n" + ] + } + ], + "source": [ + "%%bash\n", + "pip install aif360" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.metrics import accuracy_score\n", + "# setting random seed\n", + "np.random.seed(218)\n", + "from aif360.datasets import AdultDataset\n", + "from aif360.metrics import BinaryLabelDatasetMetric\n", + "from aif360.metrics import DatasetMetric\n", + "from aif360.algorithms.preprocessing import Reweighing\n", + "from IPython.display import Markdown, display" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### We will be using the AdultDataSet which serves as a CENSUS income dataset for adults with demographic information. The sensitive variable, or in other words, the variable that creates the biased \"priviledge\" is male." + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.7/site-packages/aif360/datasets/standard_dataset.py:121: FutureWarning: outer method for ufunc is not implemented on pandas objects. Returning an ndarray, but in the future this will raise a 'NotImplementedError'. Consider explicitly converting the Series to an array with '.array' first.\n", + " priv = np.logical_or.reduce(np.equal.outer(vals, df[attr]))\n", + "/opt/conda/lib/python3.7/site-packages/aif360/datasets/standard_dataset.py:142: FutureWarning: outer method for ufunc is not implemented on pandas objects. Returning an ndarray, but in the future this will raise a 'NotImplementedError'. Consider explicitly converting the Series to an array with '.array' first.\n", + " df[label_name]))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['age', 'education-num', 'sex']\n", + "['income-per-year']\n" + ] + } + ], + "source": [ + "single_protected = ['sex'] \n", + "single_privileged = [['Male']]\n", + "ad = AdultDataset(protected_attribute_names=single_protected,\n", + " privileged_classes = single_privileged, \n", + " categorical_features=[], \n", + " # even if the priviledged features are not specified, they are kept\n", + " features_to_keep=['age', 'education-num'])\n", + "print(ad.feature_names)\n", + "print(ad.label_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_orig_train, dataset_orig_test = ad.split([0.7], shuffle=True)\n", + "privileged_groups = [{'sex': 1}]\n", + "unprivileged_groups = [{'sex': 0}]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Explore the Data" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Training Dataset shape" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(34189, 3)\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Protected attribute names" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['sex']\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Privileged and unprivileged protected attribute values" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[array([1.])] [array([0.])]\n" + ] + }, + { + "data": { + "text/markdown": [ + "#### Dataset feature names" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['age', 'education-num', 'sex']\n" + ] + } + ], + "source": [ + "display(Markdown(\"#### Training Dataset shape\"))\n", + "print(dataset_orig_train.features.shape)\n", + "display(Markdown(\"#### Protected attribute names\"))\n", + "print(dataset_orig_train.protected_attribute_names)\n", + "display(Markdown(\"#### Privileged and unprivileged protected attribute values\"))\n", + "print(dataset_orig_train.privileged_protected_attributes, \n", + " dataset_orig_train.unprivileged_protected_attributes)\n", + "display(Markdown(\"#### Dataset feature names\"))\n", + "print(dataset_orig_train.feature_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### After we assign the priviledged and under-priviledged groups, we calculate the baseline fairness." + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Original training dataset" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference in mean outcomes between unprivileged and privileged groups = -0.192812\n" + ] + } + ], + "source": [ + "metric_orig_train = BinaryLabelDatasetMetric(dataset_orig_train, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + "display(Markdown(\"#### Original training dataset\"))\n", + "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % \n", + " metric_orig_train.mean_difference())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### This value shows that there is a mean difference between priviledged and underpriviledged groups, and the biad towards the privildeged group." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction of Fainess Algorithm: Inprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### I will be introducing, executing and walking you through one of the inprocessing techniques that IBM AI360 has created to introduce fairness to the model. The interesting aspect of inprocess algorithm is the intervention the algorithm makes in the model, not post or pre analysis. \n", + "\n", + "#### Meta Fair Classifier takes the fairness metric, BinaryLabelDatasetMetric, as an imput and optimizes the algorithm for it. It is essentially passing the fairness metric as a loss function, and trianing the model, while sacrificing from some of the accuracy, to be more fair. The example code for this introductory modelu is inspired by https://github.com/IBM/AIF360/blob/master/aif360/algorithms/inprocessing/meta_fair_classifier.py.\n", + "\n", + "#### Below we create the MetaFairClassifier class, and then create the fit and predict functions for the model. Here, we train the biased model, where algorithm has a tau of 0, motivating it to not be fair:" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "import copy\n", + "from aif360.algorithms import Transformer\n", + "from aif360.algorithms.inprocessing.celisMeta.FalseDiscovery import FalseDiscovery\n", + "from aif360.algorithms.inprocessing.celisMeta.StatisticalRate import StatisticalRate\n", + "from aif360.algorithms.inprocessing.meta_fair_classifier import MetaFairClassifier\n", + "\n", + "mfc = MetaFairClassifier(tau=0, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Make Predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_bias_test = mfc.predict(dataset_orig_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Assess Bias" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "#### Original training dataset" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference in mean outcomes between unprivileged and privileged groups = -0.062442\n", + "We can see that the mean outcome is still biased, towards the variable as before - sex: Male.\n" + ] + } + ], + "source": [ + "metric_orig_train = BinaryLabelDatasetMetric(dataset_bias_test, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + "display(Markdown(\"#### Original training dataset\"))\n", + "print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % \n", + " metric_orig_train.mean_difference())\n", + "print(\"We can see that the mean outcome is still biased, towards the variable as before - sex: Male.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Accuracy of the Unconstrained Model" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Score: 0.778\n" + ] + } + ], + "source": [ + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in list(dataset_bias_test.labels)]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "print(\"Accuracy Score:\", round(accuracy_score(predictions, y_test),3))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Now we will change the tau values in the algorithm to introduce fairness into the model, to see how the accuracy score and the fairness metric changes with it." + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#########################################\n", + "Tau: 0.01\n", + "Training Accuracy: 0.6112784813829009 , Training gamma: 0.6925396834753267\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.119845\n", + "Test Accuracy Score: 0.613\n", + "#########################################\n", + "Tau: 0.12\n", + "Training Accuracy: 0.605487144988154 , Training gamma: 0.6898865741357078\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.101610\n", + "Test Accuracy Score: 0.608\n", + "#########################################\n", + "Tau: 0.23\n", + "Training Accuracy: 0.6044341747345637 , Training gamma: 0.6901179588784879\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.101990\n", + "Test Accuracy Score: 0.606\n", + "#########################################\n", + "Tau: 0.34\n", + "Training Accuracy: 0.5963321536166604 , Training gamma: 0.6939808846675622\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.104616\n", + "Test Accuracy Score: 0.6\n", + "#########################################\n", + "Tau: 0.45\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.55\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.66\n", + "Training Accuracy: 0.5866799262920823 , Training gamma: 0.6775038942815195\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.022175\n", + "Test Accuracy Score: 0.59\n", + "#########################################\n", + "Tau: 0.77\n", + "Training Accuracy: 0.7902249261458364 , Training gamma: 0.5760910552624355\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.125206\n", + "Test Accuracy Score: 0.785\n", + "#########################################\n", + "Tau: 0.88\n", + "Training Accuracy: 0.5477785252566615 , Training gamma: 0.6917244552102103\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.015778\n", + "Test Accuracy Score: 0.554\n", + "#########################################\n", + "Tau: 0.99\n", + "Training Accuracy: 0 , Training gamma: 0\n" + ] + }, + { + "ename": "IndexError", + "evalue": "list index out of range", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mmfc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMetaFairClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msensitive_attr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"sex\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mmfc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_orig_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mpredictions_mfc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmfc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_orig_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0mpred_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredictions_mfc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mdataset_orig_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfavorable_label\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpred_labels\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/aif360/algorithms/transformer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mnew_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mnew_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetadata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/aif360/algorithms/inprocessing/meta_fair_classifier.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 78\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/aif360/algorithms/inprocessing/celisMeta/General.py\u001b[0m in \u001b[0;36mmodel\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training Accuracy: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxAcc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\", Training gamma: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxGamma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetValueForX\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdist_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparamsOpt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/aif360/algorithms/inprocessing/celisMeta/FalseDiscovery.py\u001b[0m in \u001b[0;36mgetValueForX\u001b[0;34m(self, dist_params, a, b, params, samples, z_0, z_1, x, flag)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetValueForX\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdist_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m \u001b[0mu_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mu_2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ml_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ml_2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 67\u001b[0m \u001b[0;31m#print (params)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mIndexError\u001b[0m: list index out of range" + ] + } + ], + "source": [ + "tau_list = np.linspace(0.01, 0.90, 10)\n", + "accuracies = []\n", + "mean_difference = []\n", + "for tau in tau_list:\n", + " print(\"#########################################\")\n", + " print(\"Tau: %.2f\" % tau)\n", + " mfc = MetaFairClassifier(tau=tau, sensitive_attr=\"sex\")\n", + " mfc.fit(dataset_orig_train)\n", + " predictions_mfc = mfc.predict(dataset_orig_test)\n", + " pred_labels = predictions_mfc.labels\n", + " predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in pred_labels]\n", + " metric_orig_train = BinaryLabelDatasetMetric(predictions_mfc, \n", + " unprivileged_groups=unprivileged_groups,\n", + " privileged_groups=privileged_groups)\n", + " mean_diff = metric_orig_train.mean_difference()\n", + " print(\"Difference in mean outcomes between unprivileged and privileged groups = %f\" % mean_diff)\n", + " acc = accuracy_score(predictions, y_test)\n", + " print(\"Test Accuracy Score:\", round(acc,3))\n", + " accuracies.append(acc)\n", + " mean_difference.append(mean_diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Results and Analysis" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "BEST TAU AND IT'S PRINT OUT FROM ABOVE\n", + "'''\n", + "Tau: 0.88\n", + "Training Accuracy: 0.5477785252566615 , Training gamma: 0.6917244552102103\n", + "Difference in mean outcomes between unprivileged and privileged groups = -0.015778\n", + "Test Accuracy Score: 0.554\n", + "'''" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Graphing (with unbiased algorithm)" + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAEWCAYAAACTwaluAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgV5Rn38e+dhBD2JSyyLwIKIgkYRKyACiKgBVGKWC2CCxWlYu0idauvVt/at9ZqiyCKUluFuhSKCi6IERFFQKmyCLIJkR0FkmASknO/f8wkHA4nyWQ5a+7PdZ3LM8uZuRNzfsw888wzoqoYY4wXCZEuwBgTOywwjDGeWWAYYzyzwDDGeGaBYYzxzALDGOOZBYYxxjMLjBggIpki8r2I1I50LbFCRHL8Xj4R+cFv+tpI1xerLDCinIh0BAYACowM876Twrm/6qSq9YtfwE7gx37zXox0fbHKAiP6jQc+AeYA1/svEJE6IvKYiHwjIkdEZLmI1HGXXSAiK0TksIjsEpEJ7vxMEbnJbxsTRGS537SKyG0i8jXwtTvvCXcbR0VkjYgM8Fs/UUTuFpGtIpLtLm8nItNF5LGAel8XkTsCf0ARmSkifw6Y918RudN9f5eIfOtuf5OIDK7cr/Kk7f9IRFa6v7fdIvK4f0CKyCAR+cxd/omI9K3qPuOCqtoril/AFuBW4BzgONDSb9l0IBNoAyQC5wO1gfZANnANUAtIBdLdz2QCN/ltYwKw3G9agXeBpkAdd9517jaSgF8Be4EUd9lvgC+BMwAB0tx1zwV2Awnues2AY/71++1zILALEHe6CfAD0Nrd7i6gtbusI3B6BX+HO4AhAfPOBfq6v7fT3d/zLe6yFsBRYKz7M08ADgCNIv33EOlXxAuwVxn/c+ACNySaudNfAb903ye4X6q0IJ/7HTC/lG16CYyLy6nr++L9ApuAUaWstxG4xH0/BVhUynqCc9ow0J2+GVjqvu8C7AeGALUq+Xs8JTCCrDMNmOu3/2UByz8HxkX6byLSLzsliW7XA++o6kF3+iVOnJY0A1KArUE+166U+V7t8p8QkV+JyEb38Pww0Mjdf3n7+gfO0Qnuf/8ZbCV1vpHzcI6IAH4KvOgu2wLcATwA7BeReSLSujI/lD8R6SEii0Vkn4gcBe7nxM/UGvgm4CPf4BzJ1WgWGFHKbYsYCwwSkb0ishf4JZAmImnAQSAP53A60K5S5gPkAnX9pk8Lsk7JLcxue8Vdbi1NVLUxcATnqKC8ff0LGOXW2x1YUMp6AHOBMSLSAegHvFZSjOpLqnoB0MGt7dEytuPVM8BnOKc3DYEHOfEz7Xb35a898G017DemWWBEryuAIqAHkO6+ugMfAuNV1Qc8B/xFRFq7jY/93UuvLwJDRGSsiCSJSKqIpLvbXQtcKSJ1RaQLcGM5dTQACnHO4ZNE5H6god/yZ4GHRKSrOHqJSCqAqmYBq3COLF5T1R9K24mqfu7u41ngbVU9DCAiZ4jIxe7PlYdzGlZU/q+vXA2AI6qaIyJn4ZyGFFsI9BaRMe7vbzxOYLxVDfuNbZE+J7JX8BfOH+djQeaPxWl0TALqAH/F+ZfvCLCMEw2VA4CVOI13u4Dr3fnNgHdwGkU/wjnUD2zD6OI3nQjMdrezB/gtfm0C7vJ7ge3uNlcBbf0+f527zYs8/Mz3uev+xG9eL+BTd9vfAW9wogH0WmC9h+2W1Os3bzCwGcjBadd5BFjit/winHaLI+7+z4v030Q0vIpbpY0JCREZiHNq0lGdoyITw+yUxISMiNQCpgLPWljEBwsMExIi0h04DLTCOW0yccBOSYwxntkRhjHGs5i7uahZs2basWPHSJdhTFxbs2bNQVVtHjg/5gKjY8eOrF69OtJlGBPXRCSwpytgpyTGmAqwwDDGeGaBYYzxLObaMII5fvw4WVlZ5OXlRboUEwIpKSm0bduWWrVqRbqUGi8uAiMrK4sGDRrQsWNHRKT8D5iYoaocOnSIrKwsOnXqFOlyaryQnZKIyHMisl9E1pWyXETkSRHZIiJfiEifyu4rLy+P1NRUC4s4JCKkpqba0WOUCGUbxhxgWBnLhwNd3dckYEZVdmZhEb/s/230CFlgqOoynNuRSzMKeEEdnwCNRaRVqOoxpqby+ZRp05awbdv3Vd5WJK+StOHkoeCyKGUINBGZJCKrRWT1gQMHwlJcZcyfPx8R4auvvop0KRHx8MMPk56eTnp6OomJiSXvn3zyyUiXVmP5fMpNNy3k0Uc/4sIL51Q5NCIZGMGOM4PeCaeqs1Q1Q1Uzmjc/pbdq1Jg7dy4XXHAB8+bNC+l+ioqqY8Cp6nfPPfewdu1a1q5dS506dUre33777ZEurUYqDovnn18LwK5dR5k1a02VthnJwMjCGUC2WFucsRRjUk5ODh999BGzZ88+JTD+9Kc/cfbZZ5OWlsa0adMA2LJlC0OGDCEtLY0+ffqwdetWMjMzufzyy0s+N2XKFObMmQM4XeIffPBBLrjgAl555RWeeeYZ+vbtS1paGldddRXHjh0DYN++fYwePZq0tDTS0tJYsWIF9913H0888UTJdu+5555T/tW/6667eOqpp0qmH3jgAR577DH27NnDwIEDSU9Pp2fPnnz44YeV+v3897//pV+/fvTu3ZuhQ4eyf/9+AA4ePMjIkSPp1asX559/PuvWBW0jNxUUGBYAEyem88gjVXukSyQvqy4EpojIPJxBX4+o6p6qbrTjtDerXFhpdvzxslKXLViwgGHDhtGtWzeaNm3KZ599Rp8+fVi8eDELFixg5cqV1K1bl+++c5p1rr32WqZNm8bo0aPJy8vD5/Oxa9euUrcPTn+E5cudZw4dOnSIm292hqG89957mT17Nr/4xS+4/fbbGTRoEPPnz6eoqIicnBxat27NlVdeydSpU/H5fMybN49PP/30pG2PGzeOO+64g1tvvRWAl19+mbfeeouXXnqJSy+9lHvuuYeioqKSYKqogQMHMnLkSESEmTNn8thjj/Hoo49y33330a9fPxYuXMg777zDhAkT7F6hKiotLJ59diQJCVVrQA5ZYIjIXOBCoJmIZAG/x3moDqo6E1gEjMB5gMwxYGKoagmHuXPncscdzkO9xo0bx9y5c+nTpw9Llixh4sSJ1K3rDNTdtGlTsrOz+fbbbxk9ejTgBIEXV199dcn7devWce+993L48GFycnK49NJLAVi6dCkvvPACAImJiTRq1IhGjRqRmprK559/zr59++jduzepqaknbbt3797s37+f3bt3c+DAAZo0aUL79u3p27cvN9xwA8ePH+eKK64gPT2dyti5cydjx45l79695Ofn061bNwCWL1/Om286IT906FAmTJhAbm4u9erVq9R+arpQhgWEMDBU9ZpylitwW6j2H06HDh1i6dKlrFu3DhGhqKgIEeFPf/qTM3BqwGXB0gYtSkpKwuc7MZJdYN8D/y/RhAkTWLBgAWlpacyZM4fMzMwya7zpppuYM2cOe/fu5YYbbgi6zpgxY3j11VfZu3cv48aNA5wjg2XLlvHmm2/ys5/9jN/85jeMHz++zH0Fc9ttt3H33XczYsQIlixZwh//+Efg1N+FDehUNb/97bshCwsg9kYNP+ecczTQhg0bTpkXTjNnztRJkyadNG/gwIG6bNkyXbx4sfbv319zc3NVVfXQoUOqqtqvXz+dP3++qqrm5eVpbm6u7ty5Uzt06KB5eXl6+PBh7dixoz7//POqqtqhQwc9cOBAyfZTU1N13759WlBQoEOGDNHrr79eVVWvvvpqffzxx1VVtbCwUI8cOaKqqvn5+dqtWzft1KmTFhYWBv051q1bp/3799euXbvq7t27VVV1x44devz4cVVVffzxx3Xq1Kmefif16tU7abpnz566du1a9fl8et111+ngwYNVVXXy5Mn6yCOPqKrqu+++qxkZGUG3F+n/x7Fi3bp92qLF/1N4QCdOXKBFRb5KbQdYrUG+f3bzWTWYO3duyelFsauuuoqXXnqJYcOGMXLkSDIyMkhPT+fPf3aeOfzPf/6TJ598sqSxb+/evbRr146xY8fSq1cvrr32Wnr37l3qPh966CH69evHJZdcwplnnlky/4knnuD999/n7LPP5pxzzmH9+vUAJCcnc9FFFzF27FgSExODbvOss84iOzubNm3a0KqV0yUmMzOT9PR0evfuzWuvvcbUqVMB54ilIm0NDzzwAKNHj2bQoEG0bNmyZP6DDz7IihUr6NWrF/fffz/PP/+8522aU511VguWLh3Pr37Vv3qPLFwxN6ZnRkaGBv6hbty4ke7du0eootjg8/no06cPr7zyCl27do10ORVm/4/DS0TWqGpG4Hw7wqgBNmzYQJcuXRg8eHBMhoUJzudTpk5dzKefhu8JjnFxt6opW48ePdi2bVukyzDVyP9qyD/+8T/eeednnHtu6J8VHTdHGLF2amW8s/+3Jwu8dHrkSD4vvvhFWPYdF4GRkpLCoUOH7A8rDqk7HobXvirxrrR+Fo8/XtaN4dUnLk5J2rZtS1ZWFtF8Y5qpvOIRt2q6UHfK8iIuAqNWrVo2GpOJa9EQFhAnpyTGxLNoCQuwwDAmqkVTWIAFhjFRTQQaNapdMh3JsIA4acMwJl6JCH/5i3Mn8pEj+RENC7DAMCbqFYeGKhENC7BTEmOiis+nzJ79GUVFvpPmi0jEwwIsMIyJGsUNnDfd9Drjxy84JTSigQWGMVEg8GrISy99yZw5a8v5VPhZYBgTYaVdOp04sfTxUCLFAsOYCIq2fhblscAwJkJiLSzAAsOYiIjFsAALDGPCLlbDAiwwjAm7PXuyeeutLSXTsRIWYIFhTNi1adOQzMwJtGpVP6bCAqxruDER0a1bKqtW3UyrVg1iJizAjjCMCTmfT9m8+dAp89u0aRhTYQEWGMaEVHEDZ0bGLFasKPth27HAAsOYEPG/GpKdXcCwYf8KeqQRSywwjAmBYJdOx4zpQZcuTSNYVdVZYBhTzWK5n0V5LDCMqUbxHBYQ4sAQkWEisklEtojItCDLG4nI6yLyPxFZLyITQ1mPMaEU72EBIQwMEUkEpgPDgR7ANSLSI2C124ANqpoGXAg8JiLJoarJmFCpCWEBoT3COBfYoqrbVLUAmAeMClhHgQYiIkB94DugMIQ1GRMS8+dvjPuwgNAGRhvA/8JzljvP39+B7sBu4EtgqqqeMi6ZiEwSkdUistoeh2ii0ZVXdmfatB8B8RsWENrACPbbCnxa8qXAWqA1kA78XUQanvIh1VmqmqGqGc2bN6/+So2pgI17jnL9c5/yxJKvSx4ALiI88shgXn31J3EbFhDawMgC2vlNt8U5kvA3EfiPOrYA24EzQ1iTMVWSX1jEpH+uJnPTfv7y9le8s2FfyTIR4aqresRtWEBoA2MV0FVEOrkNmeOAhQHr7AQGA4hIS+AMYFsIazKmSl5Y8Q07Dx3j0OIdHHx9O9PfO3GUUROELDBUtRCYArwNbAReVtX1InKLiNzirvYQcL6IfAm8B9ylqgdDVZMxVfF9bgFPvreZQ4t3kPvlIY5t+p73nvqc5ZtqTrtaSG9vV9VFwKKAeTP93u8GhoayBmOqy1+XbGb7/C3kfnnifhBJTmDW8u0MOLNFBCsLH+vpaYwHW/fn8Pj9H5wUFvXPTiV1eEc+3HKQdd8eiWB14WOBYUw5fD7lsjEvk/3FibPlCRPS+dlv++N0IYIZH2yNVHlhZYFhTBl8PmXU1a+w6cMTXYpGje3B7NkjufWiLiXzFn+5h+0HcyNRYlhZYBhTCp9PufHGhbzx6saSeT0GteM/c8eQkCD0bNOIgd2cfkE+hVnL4v8owwLDmFLcffd7Jz3ftGGvZix+7eqT+llMHnR6yfvX1nzLvqN5Ya0x3CwwjCnF9RPTSW5UG4B6Z6dy1yMX0z613knrnNe5Kb3bNwagoMjH7OXbw15nOFlgGFOKD/Yeodm4bjTsdxpdr+rGlMFdTllHRE46ynjxk284cux4OMsMKwsMY4L4LreA6Uu3UKtxbZpc2JZfDu1Gw5RaQdcd0r0lXVvUByC3oIgXPt4RvkLDzALDGJwGzilTFvHBBzsAeGLJZrLznZEWOjevxzXnti/1swkJwi1+RxnPr9jBDwVFIa03UiwwTI1XPPjN9OmrGDHiJV7670ZeXLmzZPnvhnenVmLZX5WR6a1p07gO4BydvLw69h8pEIwFhqnRAkfKOnbsOA//bSWFPueGsvM6N2VI9/K7fddKTODmAZ1Kpmct28bxolOGdol5Fhimxgo2rN5lY7qTc06zkul7RvQo6c1Znqv7tqdpPWeEyW8P/8Dr/wsczSH2WWCYGilYWEyYkA4XtCoJiCt7t+Hsto08b7NOciITz+9YMj0jcys+X3zd+m6BYWqc0gbsvXxKH9btOQpA7aQEfn3pGRXe9vj+HamXnAjA1/tzeO+r/dVTdJSwwDA1Smlh8fcZl/HYO5tL5t00oBOt3UbMimhUtxbXntehZPqpzC1xNcCOBYapMVRLfxTA8yt2sPuI0627Wf1kJl94aictr268oBPJ7lWVz3ceZuX276pWeBSxwDA1Srt2J8aYLg6L744VMCPzxI1jdwzpRv3alR9bqmXDFK7sc2KAfP9tx7qQjrhlTDQRER544EIAdu06WjK691+XbCbH7aTVpUV9xvVtV8ZWvPn5oNP59+pdqMIHmw+w7tsj9GzjvQE1WtkRhqlRikNj9mwnLLbsz2bupyc6Wd094kySyumk5UWnZvUY0bNVyfTMOBlgxwLDxC2fT5k+/VMKArppi0jJpdP/u+gritxLnz/qkspFZ1Tf2JyTLzzRXXzRl3vYEQcD7FhgmLhUfDVkypTF/OQnr5wSGgArthwsuewpAneP6O65k5YXPds0YkBXpxOYT+HpZbH/BA0LDBN3Ai+dLly4iWeeWXPKOg8vOjGS1lV92nJW6+pvY7jV72rLa2uy2B/jA+xYYJi4Ulo/i8mT+5603vzPv2X9bqeTVkqtBH49tOKdtLw4r3NT0tvFzwA7FhgmbpQWFoHPOv2hoIj/9/amkulJAzpzWqOUkNQkIie1ZfwrxgfYscAwccFrWAA8++E29h4t7qRVm0l+Y1mEwiXdW9LFb4Cdf638JqT7CyULDBPzKhIW+7PzTnqGyK+GVq2TlheBA+w8t3x7zA6wY4FhYlpFwgLg8Xe/5pj7Ze3Wsj4/OadtWOocld6a1u5pz6HcAl5ZE5sD7FhgmJh28OAx3n9/R8l0WWGxeV82/151YiStu0d0r5ZOWl7USkzg5oGdS6af/iA2B9ixwDAxrUWLemRmXk/Hjo3LDAuARxZtpHh4igFdmzHIfQhRuFzdtx1N6joDCX97+Afe+CL2BtiJ23tJ3l6/l6c/2Eq92knUr51E3eQk6tdOpF7tJOeV7Pe+tt+y5OJ5iSQnJlRrRx4TGh06NGblypto1qxuqWHx4dcHyNx0AAhNJy0v6iYnMfFHnfjLu85t9DMytzIqrU2pNUejkAaGiAwDngASgWdV9Y9B1rkQ+CtQCzioqoOqY9+7D//AZzsPV2kbSQniFziJJ71PShQSREhMEBLdrsaJCZCY4L53l4lQ8j4hQUhwp5337nxx5icnJdAgpRYNUpJoUDvpxPsU531ykh0Q+nzKhg0H6Nnz5C7cLVrUK+UTUORTHn7zRCetn5zTlu6tGpa6fiiN79+Bpz/YSm5BEZv35bD0q/0M6dEyIrVURsgCQ0QSgenAJUAWsEpEFqrqBr91GgNPAcNUdaeIVFtH/lz37sOqKPQpR344zpEfouO6eXJSAg1TTg6S+icFSy03aE5ep1n92jRvUJuUWomR/hGqpLiBc+7cdbzxxjUMHty5/A8Br32WxVd7swGoUyuRX4Wok5YXjesm89N+7XnmQ6cD11OZWxjcvUXMHMmG8gjjXGCLqm4DEJF5wChgg986PwX+o6o7AVS12sYz+0lGO/p1TiU3v5Dc/CLnvwWF5OYXkpNfxLGCQnLyneljBUUl73Pzi0rWO14UXSMlFRT6OJhTwMGcgkp9vkHtJJo3qE2zBrVp0cAJkeYNatPcDZQWDVJo3qA2Teslkxhlh8mBV0Muv3wuq1bdfMqRRqBjBYX82b+T1sDOtGwYmk5aXt14QWfmrNjB8SLls52H+XT7d/TrnBrRmrwqNzBEZArwoqp+X8FttwH8rx1lAf0C1ukG1BKRTKAB8ISqvhCkhknAJID27Ut/oIy/lg1TqvyHUVDocwPm5FA5VlBIoU/xqfOHXORTfOq8inxQpIqqM//EMufQ2OdTitTvs+rM86mSd9xHTn4h2XnHOZpXSHZeITn5x8l23xffVVlZ2fmFZOcXsq2cuyYTBFLr+4VK/RPhUhwqjerUIlz/KPp8yj13vsN/5q0vmTfiijNISk1h877sMj/72pos9mfnA9CiQW1+PsjbUUkondYohav6tGXeKufrMeODrfETGMBpOKcTnwHPAW+rt0EKg/05BX4uCTgHGAzUAT4WkU9UdfNJH1KdBcwCyMjICNs/+8lJCSQnJdPEHTo+klSVH44XkZNX6IbJcTdQTrwvnp/jBkx2/nGO/lDIoZx8DuTkez5i8ikcyM7ngPtFiyRV5dDiHeR+eahkXr2zU1ndPoVhT3xYoW39eugZ1E2Ojnb+SQM7lwywk7npABt2H6VH68i0q1REub89Vb1XRO4DhgITgb+LyMvAbFUta1SQLMB/6KK2QOB1pCychs5cIFdElgFpwGbMSUSEusnO1Z4Wlfi78rntMQdynCDYn51XEgrOtPs+J5/DUXKvQ2lhkTq8Y4XP+c88rQFXhamTlhedm9dneM/TWPTlXsA5yvjbNb0jXFX5PMWtqqqI7AX2AoVAE+BVEXlXVX9bysdWAV1FpBPwLTAOp83C339xAigJSMY5ZXm84j+GKU9CgtCknnO01K1lgzLXzS8s4mBOwUmBclLI5OSTk1f1RuWyqE/Z8PKmk8KiVd/T6DH2DKSC7Sup9ZP5PyN7Rl27zORBXUoC480vdvProd3okFr61Z5o4KUN43bgeuAg8CzwG1U9LiIJwNdA0MBQ1UK3/eNtnMuqz6nqehG5xV0+U1U3ishbwBeAD+fS67rq+MFM5dVOSqRN4zolzwoNt+IGzvdW7S2ZV16nrFh0dltngJ0Pvz5YMsDOI6PPjnRZZZLymiNE5EGc049TbrETke6qujHIx0ImIyNDV69eHc5dmjB7883NXH753JLpeAyLYiu2HuSnz6wEIDkxgeV3XUSLCF/FARCRNaqaETjfS0+gRUDJgxVEpIGI9AMId1iYmuGyy7rx8MMXA/EdFgD9O6eS5jfAznMf7YhsQeXwEhgzgBy/6Vx3njEhc/fdA3jjjWviOizAacy+NXCAnSjpKBiMl8AQ/8uoquojju9BMeHn8yn5QXrmXnZZt7gOi2KXdG/J6c2dxs6c/EL+9Un0DrDjJTC2icjtIlLLfU0FYn/4YxMVihs4R4/+N3khvvISrQIH2Hn+o+3kHY/OAXa8BMYtwPk4l0aLe2tOCmVRpmbw7+69ePEWrrzy30GPNGqCUeltaN0ohcZ1a3Ftvw5V7tUbKl46bu3H6UNhTLUJNlLWaafVp1aM3yBXWclJCcwan0GnZvWoF+IhA6vCSz+MFOBG4Cyg5HqPqt4QwrpMHKvosHo1RSw8e9XLKck/ce4nuRT4AKeLd9l3/BhTCguL2OYlMLqo6n1Arqr+A7gMiO7uaCYqWVjEPi+BUXxR+LCI9AQaAR1DVpGJSxYW8cFL68osEWkC3AssBOoD94W0KhN3fv/79y0s4kCZRxjuDWZHVfV7VV2mqp1VtYWqPh2m+kycuOWWDLp0aQpYWMSyMgPD7dU5JUy1mDjWpk1DMjOv5957B1hYxDAvbRjvisivRaSdiDQtfoW8MhN32rRpyEMPXWxhEcO8BMYNwG3AMmCN+7L7y02pfD5l8uQ3WLz460iXYqpZuYGhqp2CvCI/kqqJSsVXQ2bOXMMVV/zbQiPOeOnpOT7Y/GCje5uaLfDSaUFBEQsXbmL48K4RrsxUFy+XVfv6vU/BGeH7M8ACw5QorZ/F9OmXRbAqU9283Hz2C/9pEWmE013cGMA6ZdUklXlY5zHAjjENYGFR03hpw3idEw8gSgB6AC+HsigTGywsah4vbRh/9ntfCHyjqlkhqsfECFXl5pstLGoaL4GxE9ijqnkAIlJHRDqq6o6QVmaimohw1lknHoRsYVEzeAmMV3CG6CtW5M7rG3x1U1PceWd/ANav388zz1hY1AReAiNJVQuKJ1S1QEQi/3RiExXuvLM/qlrhZ52a2OTlKskBERlZPCEio3Aem2hqEJ9PefzxjzkW5EHNFhY1h9dRw+8WkZ0ishO4C/h5aMsy0aT4asidd77DyJFzg4aGqRm83EuyVVXPw7mcepaqnq+qW0JfmokGgZdO33tvO08/bfce1lTlBoaIPCIijVU1R1WzRaSJiPwhHMWZyCqtn8XUqedFsCoTSV5OSYar6uHiCVX9HhgRupJMNLBOWSYYL4GRKCK1iydEpA5Qu4z1TYyzsDCl8RIY/wLeE5EbReRG4F3gH142LiLDRGSTiGwRkWllrNdXRIpEZIy3sk2oWFiYsni5W/VPIvIFMAQQ4C2gQ3mfE5FEYDpwCc4zWVeJyEJV3RBkvUeBtytevqlOFhamPF7vVt0L+ICrcMbD2OjhM+cCW1R1m9vxax4wKsh6vwBeA/Z7rMWEyOHDeXz88YnbhCwsTKBSjzBEpBvOQ5ivAQ4B/wZEVS/yuO02wC6/6eInv/vvow0wGriYMrqai8gk3CfGt2/f3uPuTUU1bVqHpUvHc/HFL9C/f1sLC3OKsk5JvgI+BH5c3O9CRH5ZgW0H+0sLfIb9X4G7VLWorN6CqjoLmAWQkZERuA1TjVq1asBHH91A48YpFhbmFGWdklyFcyryvog8IyKDCR4CpckC2vlNtwV2B6yTAcwTkR3AGOApEbmiAvswVeDzKZ99tueU+U2b1rGwMEGVGhiqOl9VrwbOBDKBXwItRWSGiAz1sO1VQFcR6eTerDYO51GL/vvopKodVbUj8Cpwq6ouqNyPYiqiuNLQomUAAAq7SURBVIHzvPOeZeHCTZEux8QIL13Dc1X1RVW9HOcoYS1Q6iVSv88V4jw17W2cRtKXVXW9iNwiIrdUsW5TBf5XQ44f9zFmzMtBjzSMCeTl9vYSqvod8LT78rL+ImBRwLyZpaw7oSK1mMoJdun0uut6kZ5+WgSrMrGiMoMAmxhl/SxMVVlg1BAWFqY6WGDUABYWprpYYMQ5CwtTnSww4tzSpdstLEy1scCIc0OGdOZvfxsOWFiYqqvQZVUTm6ZMOZczz2zGxRd3srAwVWJHGHHG59Ogg/QOGdLZwsJUmQVGHClu4Bw+/EVycwvK/4AxFWSBESf8r4YsW/YNI0a8ZI8DMNXOAiMOBLt0evrpTUhJsSYqU70sMGKc9bMw4WSBEcMsLEy4WWDEKAsLEwkWGDHIwsJEigVGDPrDH5ZZWJiIsMCIQZMnZ9CzZwvAwsKEl113i0HNm9dj6dLxzJixmnvvHWhhYcLGAiNGNW9ej/vvHxTpMkwNY6ckUc7nU37+89d59dUN5a9sTIhZYESx4qshs2Z9xrhxr1pomIizwIhSgZdOi4qUd9/dGuGqTE1ngRGFSutnMWPG5RGsyhgLjKhjnbJMNLPAiCIWFibaWWBECQsLEwssMKKAqjJp0usWFibqWWBEARGhX782JdMWFiZaWU/PKHHzzecAsHLlt8ya9WMLCxOVRFUjXUOFZGRk6OrVqyNdRsioKiIWFiayRGSNqmYEzg/pKYmIDBORTSKyRUSmBVl+rYh84b5WiEhaKOuJFj6f8uijyzl6NP+UZRYWJpqFLDBEJBGYDgwHegDXiEiPgNW2A4NUtRfwEDArVPVEi+KrIdOmvcewYf8KGhrGRKtQHmGcC2xR1W2qWgDMA0b5r6CqK1T1e3fyE6BtCOuJuMBLpx9/nMWMGasiXJUx3oUyMNoAu/yms9x5pbkRWBxsgYhMEpHVIrL6wIED1Vhi+JTWz+I3v/lRBKsypmJCGRjBTsaDtrCKyEU4gXFXsOWqOktVM1Q1o3nz5tVYYnhYpywTL0J5WTULaOc33RbYHbiSiPQCngWGq+qhENYTERYWJp6E8ghjFdBVRDqJSDIwDljov4KItAf+A/xMVTeHsJaIsLAw8SZkRxiqWigiU4C3gUTgOVVdLyK3uMtnAvcDqcBT7uXEwmDXfmORhYWJRyHt6amqi4BFAfNm+r2/CbgplDVESk5OAWvX7i2ZtrAw8cC6hodIw4a1WbJkPEOGvEB6+mkWFiYuWGCEUNOmdcjMnED9+skWFiYu2N2q1cTnU1as2HXK/IYNa1tYmLhhgVENihs4Bwx4nhdf/CLS5RgTMhYYVeR/NcTnU8aPXxD0SMOYeGCBUQXBLp1ef30a550X17fEmBrMAqOSrJ+FqYksMCrBwsLUVBYYFWRhYWoyC4wKsLAwNZ0FRgV89NFO5syxsDA1lwVGBQwY0IHnnhuFiIWFqZmsa3gFTZiQzumnN+FHP2pvYWFqHDvCKIPPp2RnnzpI74ABHSwsTI1kgVGK4gbOwYNf4PDhvEiXY0xUsMAIwv9qyKpVuxk69J9BjzSMqWksMAIEu3Tas2cL6tVLjmBVxkQHCww/1s/CmLJZYLgsLIwpnwUGFhbGeFXjA8PCwhjvanxgPProcgsLYzyq8YExeXJf+vZtDVhYGFOeGt81vHHjFN5552fMmLGKu+66wMLCmDLU+MAAJzR+97sBkS7DmKhXo05JfD7l5psXnnSLujHGuxpzhOF/NWT27M8B585TY4x3NeIII/DSqaozGI4xpmLiPjBK62fx9NM/jmBVxsSmuA4M65RlTPWK28CwsDCm+oU0MERkmIhsEpEtIjItyHIRkSfd5V+ISJ/q2K+FhTGhEbLAEJFEYDowHOgBXCMiPQJWGw50dV+TgBnVse9bbnnDwsKYEAjlEca5wBZV3aaqBcA8YFTAOqOAF9TxCdBYRFpVdccXXdSxJBwsLIypPqHsh9EG8H+MeRbQz8M6bYA9/iuJyCScIxDat29f7o6vueZsAN5/fwczZ15uYWFMNQnlEUawb6lWYh1UdZaqZqhqRvPmzT3t/JprzmbWrB9bWBhTjUIZGFlAO7/ptsDuSqxjjIkSoQyMVUBXEekkIsnAOGBhwDoLgfHu1ZLzgCOquidwQ8aY6BCyNgxVLRSRKcDbQCLwnKquF5Fb3OUzgUXACGALcAyYGKp6jDFVF9Kbz1R1EU4o+M+b6fdegdtCWYMxpvrEbU9PY0z1s8AwxnhmgWGM8cwCwxjjmTjtjrFDRA4A33hYtRlwMMTlVEU01xfNtUF01xfNtYH3+jqo6im9JGMuMLwSkdWqmhHpOkoTzfVFc20Q3fVFc21Q9frslMQY45kFhjHGs3gOjFmRLqAc0VxfNNcG0V1fNNcGVawvbtswjDHVL56PMIwx1cwCwxjjWcwHRqQGGq6m2q51a/pCRFaISFq4avNSn996fUWkSETGRFNtInKhiKwVkfUi8kG4avNSn4g0EpHXReR/bn1huxNbRJ4Tkf0isq6U5ZX/TqhqzL5wbpvfCnQGkoH/AT0C1hkBLMYZ3es8YGUU1XY+0MR9PzxctXmtz2+9pTh3HY+JltqAxsAGoL073SKafnfA3cCj7vvmwHdAcpjqGwj0AdaVsrzS34lYP8KI2EDD1VGbqq5Q1e/dyU9wRhwLFy+/O4BfAK8B+6Ostp8C/1HVnQCqGm31KdBARASojxMYheEoTlWXufsrTaW/E7EeGKUNIlzRdUKhovu9ESf1w6Xc+kSkDTAamEl4efnddQOaiEimiKwRkfFhq85bfX8HuuMMOfklMFVVfeEpr1yV/k7E+tPbq22g4RDwvF8RuQgnMC4IaUUBuw0yL7C+vwJ3qWqR8w9l2HipLQk4BxgM1AE+FpFPVHVzqIvDW32XAmuBi4HTgXdF5ENVPRrq4jyo9Hci1gMjmgca9rRfEekFPAsMV9VDYairmJf6MoB5blg0A0aISKGqLoiC2rKAg6qaC+SKyDIgDQhHYHipbyLwR3UaDbaIyHbgTODTMNRXnsp/J8LVUBSixp0kYBvQiRONT2cFrHMZJzfwfBpFtbXHGc/0/Gj83QWsP4fwNXp6+d11B95z160LrAN6RlF9M4AH3PctgW+BZmH8/9uR0hs9K/2diOkjDI3igYY91nY/kAo85f4rXqhhutPRY30R4aU2Vd0oIm8BXwA+4FlVDXoZMRL1AQ8Bc0TkS5wv5l2qGpbb3kVkLnAh0ExEsoDfA7X8aqv0d8K6hhtjPIv1qyTGmDCywDDGeGaBYYzxzALDGOOZBYYxxrOYvqxqIk9EUnH6QwCcBhQBB9zpc9W518LECbusaqqNiDwA5KjqnyNdiwkNOyUxIeOOB7HGHQ/iJr/514nIlyKyTkQeiWSNpmLslMSE0vWq+p2I1AVWi8hrQD3gDzj3qRwBlojI5ar6RiQLNd7YEYYJpV+KyP+Aj3FucDod6AcsVdWDqnoceAlnwBcTA+wIw4SEiAzBCYLzVPUHEVkOpBD81moTI+wIw4RKI+A7NyzOAvq68z8BLhKRVBFJAsYBYR2P01SeHWGYUHkTmOSeknwFrARQ1SwRuR/IxDnaeF1V34xYlaZC7LKqMcYzOyUxxnhmgWGM8cwCwxjjmQWGMcYzCwxjjGcWGMYYzywwjDGe/X+8nyK/8u4uWwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(tau_list[:-1], accuracies, lw=3, label=\"Accuracy vs. Tao\")\n", + "axe.set_xlabel(\"Tao\")\n", + "axe.set_ylabel(\"Accuracy\")\n", + "axe.set_title(\"Accuracy vs. Tao\")\n", + "axe.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPQAAAEWCAYAAABR1LrDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deXwV1dnHvw9hFQggUJBEBBVkUTYJICqyKAhatIItuKGoFEFr9ZUKvr6C1lpbbbVqFXHHVhG0Lrgiq1JENgEBFRFZIqCsAVkSkjzvH2cSLpd7bybLzV3yfD+f+8nMmTMzvzuZ3z1nzpzzHFFVDMNIDirFWoBhGGWHGdowkggztGEkEWZow0gizNCGkUSYoQ0jiTBDxxEi8rOInByD804Ukf8LWL9JRH709NQXkbNF5Ftv/dLy1mf4R5L1PbSIbACaAE1UdUdA+nKgPdBcVTeUo56ewGzggJeUBTyrquOjfN4NQCMgF8gD1gCTgUmqmh8ifxVgL9BNVVd4abOAd1T1H9HUmkiIyETgKm+1KiBAtrf+qar2j4WuZC+hvweGFqyIyBlAjdjJYYuq1lLVWsA5wPXlVOL9UlVrAycBDwJ3As+FydsIqA6sDkg7KWjdNyJSuST7xTuqOjLgf/kA8FrBeqzMDMlv6JeBawLWh+FKp0JEpJqIPCwim7xq5kQRqeFtqyci74rIdhHZ7S2nB+w7V0T+KCL/FZF9IjJDRBr4Eaaq3wMLgDYBx1MROdVbvkhEvhCRvSKyWUQmBOSrLiL/EpGdIrJHRBaLSCMf58xS1XeA3wDDROR073gvisj9ItIS+MbLvkdEZovId8DJwHSvyl1NROqIyHMislVEfvD2TfGOda13PR4RkV3ABC99uIh85V3Hj0TkpKDvPdKr1u8WkX+KiARsv9Hbd5+IrBGRTl56ExF5w/v/fC8ivwv1vUWkm4hsK9Dopf1KRFZ6y11EZIl3rX8Ukb8XdS2LQkQqe9p+9P5Hc0TktIDtx4vIKwHa/xD4nUuMqiblB9gAnI+7QVsDKcBmXGmjQDMv36PAO8DxQG1gOvBnb1t9YBBwnLdtGvBWwDnmAt8BLXEl/1zgwTB6egKZAestgB+A3gFpCpwakP8M3I9uO+BH4FJv2289ncd53+tMIDXSdQiRvgm4yVt+EbjfW27m6agc7hjAW8DTQE3gF8Ai4Lfetmtx1ftbgMredbkUWOf9HyoDdwMLgr73u0BdoCmwHbjQ23a5d50ycNXaU73/YSVgKXAPrsp7MrAe6BfmOnwHXBCwPg0Y6y1/BlztLdfCPW4U516bAPwrKK0yrgCphavxPAUsDNg+1dNQy/tO3wNXlvq+j7XxovXhiKHvBv4MXAh87F1o9W5cAfYDpwTsdxbwfZhjdgB2B6zPBe4OWB8FfBhm355APrAH94yqwH+AqkE39qlh9n8UeMRbHo4r3dv5vQ4h0hcC/+stv4hPQ+Oq5NlAjYDtQ4E53vK1wKagc30AXB+wXgnXlnBSwPc+J+hmLzDbR8CtIfR3DXGeccALYa7D/cDz3nJt7/9ecP5PgHuBBiW81yYQZOgQeRp7///qQDVce8bJAdtvDXfvFOeT7FVucNXuK3A32uSgbQ1xpdxSr1q0B/jQS0dEjhORp0Vko4jsxf3j6wZW3YBtAcsHcL+44diiqnVVNRVXGh0EXgqVUUS6etW07SKSBYwECqrzL+Nu9CkiskVE/uo1ZhWHNGBXMfcBVzpWAbYGXLOncSV1AZtD7POPgPy7cD+maQF5wl3HE3GlaygdTQqO6R33LtwPTiheAS4TkWrAZcAyVd3obbseV8v62nt8uTjcl/eLV+V+WETWe/fO17jvXB9n7kq4WlIBGzn6epSIpDe090/7HhiAKxED2YEzVVvPaHVVtY66hg6A/wFOA7p6JuzhpZf6WUdVs3A32S/DZHkF9yhwoqrWASYWnFdVD6vqvaraBugOXMzRbQUREZEM3M0zvwTSN+NK6AYB1yxVVdsG5Al+dbIZVyWvG/CpoaoLfJ7vlDDp3wcds7aqDgh1EFVdgzNNf9wP/CsB275V1aG4H6W/AK+LSE0f2iJxHdAX6AXUAVp56YL78crHPV4U0BT3aFEqkt7QHtfjnlX3Byaqe23zDPCIiPwCQETSRKSfl6U2zvB7ROR4oMxeMYlILWAI4VuPawO7VPWQiHTB3YQF+/YSkTO8msJe4DCuClfUOVO90mcKror4ZXF1q+pWYAbwN+94lUTkFBE5L8JuE4FxItLW01FHRC73ecpngTtE5ExxnOo1qC0C9orInSJSQ0RSROR078cqHK8Av8P9ME8rSBSRq0SkoXc/7PGSi7yeRVAbOATsxLU13F+wQVWzgTeBB0Skpoicgqty/6uU56wYhlbV71R1SZjNd+IabBZ6VaOZuFIZ3HNrDVxJvhBXHS8NTbyW4p9xpcXxwJVh8o4C7hORfbiGn6kB2xoDr+PM/BUwj8g3w3TvOJuB/wX+jitBSso1uIaoNcBuT8sJ4TKr6pu4km+Kd41X4UrKIlHVacCfcGbch2uQO15V83C1mw64GtgOnPnrRDjcq7i2jNka0DcB176y2vu//AMYoqqHoLCzz7l+tAbxHK5xbxvwJcfWhn7r/d2I65/wLPDvEpznKJK2Y4lhVEQqRAltGBUFM7RhJBFmaMNIIszQhpFEJFzH+QYNGmizZs1iLcMwYsbSpUt3qGrDUNsSztDNmjVjyZJwb6AMI/kRkY3htlmV2zCSCDO0YSQRZmjDSCLM0IaRRJihDSOJiJqhReR5EflJRFaF2S4i8piIrBORlQVhZQzDKDnRLKFfxI1iCUd/XBieFsAIXIgWwzBKQdQMraqfEDkixiXAZHUsxEUCCTsEzzCSmezsXG677UN+/PHnUh0nls/QaRwdqiaTMCFYRGSEF5Vxyfbt28tFnGGUF9nZuVx++TQeffRzeveeXCpTx9LQocL4hBycraqTVLWzqnZu2DBkjzfDSEgKzDx9+loA1qzZzquvhmx28kUsDZ2JCwBXQDqwJUZaDKPcCTYzwNixZ3PrrV1LfMxYGvod4BqvtbsbkOXFqzKMpCecmR94oA+libcftcEZIlIQv6mBiGTiAuxVAVDVicD7uEic63BhW0sT48owEoZomRmiaGgvLGqk7QqMjtb5DSMeiaaZwXqKGUa5sm7dLubNOzL6sSzNDGZowyhX2rb9BR9/fDWpqdXK3MyQgAEODCPR6dIljVWrbiI9PbVMzQxWQhtGVMnOzmX9+t3HpJ94Yp0yNzOYoQ0jahQ0gHXv/hxr1pRPD0cztGFEgcDW7B9/3E/v3i+Vup+2H8zQhlHGhHo1NXx4R37xi9JOaFk0ZmjDKENCmXncuHP40596R+WZORgztGGUEbE2M5ihDaNMiAczgxnaMEpNvJgZzNCGUWomTVoaF2YGM7RhlJpRozIYNqw9EFszg3X9NIxSk5JSieeeG8jFF7dk0KDWMTMzWAltGMXm8OE83OjfI6SkVGLw4DYxNTOYoQ2jWGRn5zJo0FRuueWDY0wdD1iV2zB8Eqo1+/HH+8e8VA7ESmjD8EEoM6emVouhotCYoQ2jCOLpPXNRmKENIwKJZGYwQxtGWBLNzGCGNoyQJKKZoRiGFpEaIlIjmmIMI14YMeLdhDMz+DC0iLQVkcXAt8B3IvK5iLSJvjTDiB233tqVevWqA4ljZvD3Hvpp4C5V/RhARM4HJgHnRFOYYcSSTp1OYObMa3jvvbXcfXePhDAz+DN07QIzA6jqTBH5WxQ1GUZc0KnTCXTqlFhTlvt5ht4gIuNEJN37jAU2FrkXICIXisg3IrLO2y94ex0RmS4iK0RktYjY/FZGuZOdnct1173NypU/xlpKqfFj6OG4aV/fBz7wlos0noikAP8E+gNtgKEhnr1HA2tUtT1uYru/iUhV3+oNo5QUtGa/+OJy+vSZnPCmLrLKrao7gVElOHYXYJ2qrgcQkSnAJcCawMMDtcU9oNQCdgG5JTiXYRSb4FdTO3YcYPr0b2jXrlGMlZWcIg0tIg2A/wHaAtUL0lW1bxG7pgGbA9YzgeCZrJ/AzRO9BagN/EZV80NoGAGMAGjatGlRkg2jSMK9Z77rrnNjqKr0+Kly/wvYALQE/gJsA5b72C9Us2DweLN+3rGaAB2AJ0Qk9ZidVCepamdV7dywYUMfpzaM8CRqpxE/+DF0Q1V9GshR1VnAMFx1uigycc/bBaTjSuJArgP+o451wPdAKx/HNowSkcxmBn+GPuz93SYi/YDTOdqo4VgMtBCR5l5D1xBc9TqQTUAfABFpBJwGrPcj3DCKS7KbGfy9h35AROoAd+BarVOBMUXtpKq5InIz8BGQAjyvqqtFZKS3fSLwR+BFEfkSV0W/U1V3lOyrGEZ4cnLykt7MEMHQInKzqj6hqgWl6kqgWC0Gqvo+7nVXYNrEgOUtQFGNa4ZRaipXrkTjxrUK15PRzBC5hB6Oa4U2jISnUiVh4sSLAWjQ4LikNDNYTDGjAlFgahGS0swQuVGsnYjsDfHZJyJ7y02hYZSAnJw8nnpqMfn5R78prVRJktbMELmE/lJVO5abEsMoI3Jy8hg8eCrTp6/liy+2MXHixVSqlLwmDsQilhhJRaCZAZ55Zhlvv/11jFWVH5EMPa3cVBhGGRBsZoCxY8/m0ksrTl+lsIZW1QfKU4hhlIZwZn7ggT5J/cwcjFW5jYTHzHwEM7SR0JiZj6ZEhhaRTmUtxDCKi5n5WEpaQt9UpioMowRs2pTFZ59lFq5XdDNDCQ2tqjeWtRDDKC6nnno8s2dfQ4MGx5mZPfxELDkbWK6q+0XkKqAT8A9V9RUo0DCiyRlnNGLlypE0blyrwpsZ/JXQTwEHRKQ98AdcxM/JUVVlGCHIzs7lm2+OHV17wgm1zcwefgydq26q+ktwJfM/cPG/DKPcKAhOcNZZz7Fs2dZYy4lb/Bh6n4iMA64C3vPC81aJrizDOEJgpJHduw9x/vmT+eEHGx8UCj+G/g2QDVyvqttw0Twfiqoqw/AIFTZo5MjONGlilcRQ+BkPPRSYqqqZAKq6CXuGNsqBihADrKzxU0KnAh+JyKciMtoL5mcYUcXMXDKKNLSq3quqbXHT1jQB5onIzKgrMyosZuaSU5yOJT/hguzvBH4RHTlGRcfMXDr8TPh+k4jMBWYBDYAbVbVdtIUZFZPJk1eYmUuBn0axk4Dfq6qf6W8Mo1TccEMnVqz4kX/+c7GZuQT4mX3ymHmdDSNaiAiPP96ffv1O4eKLW5qZi4mNhzZiSk5OHnl5R084KiL88penmZlLgBnaiBnZ2bkMHjyV4cPfOcbURsnwFWhfRE4CWqjqTBGpAVRW1X0+9rsQ+AdubqtnVfXBEHl6Ao/iupPuUNXziqHfSFBCtWa/8MIlFSbcbrTw08p9I/A68LSXlA685WO/FNzkdv2BNsBQEWkTlKcu8CQw0HvXfXmx1BsJSSgzp6XVxmrYpcdPlXs0cDawF0BVv8Xfe+guwDpVXa+qOcAU3IitQK7AzQ+9yTv2T36FG4mJvWeOLn4Mne0ZEgARqQxohPwFpAGbA9YzvbRAWgL1RGSuiCwVkWt8HNdIUMzM0cfPM/Q8EbkLqCEiFwCjgOk+9gv1Hwr+IagMnImb9L0G8JmILFTVtYGZRGQEMAKgadOmPk5txBtm5vLBTwk9FtgOfAn8Fjff890+9ssETgxYTwe2hMjzoaru9yZ6/wRoH3wgVZ2kqp1VtXPDhg19nNqIJ8zM5YefEroG8LyqPgOFjV01gANF7LcYaCEizYEfgCG4Z+ZA3gae8KrxVYGuwCP+5RuJwKhR75mZywk/JfQsnIELqAEUOdpKVXOBm4GPgK9wY6pXi8hIERnp5fkK+BBYCSzCvdpaVbyvYMQ7d9zRnUaNagJm5mgjLlxYhAwiy1W1Q1Fp5UXnzp11yZIlsTi1UQq++mo7b731NWPHnmNmLiUislRVO4fa5qfKvV9EOqnqMu9gZwIHy1Kgkfy0bt2Q1q2t/SPa+Kly/x6Y5kUs+RR4DVeVNoxjyM7O5aqr/sOiRT/EWkqFxM9oq8Ui0go4Dfcq6mtVPRx1ZUbCEdiaPX36Wj7++Gq6dAnuemBEE7+DMzKAdkBHXBdO6wBiHEXwq6m9e7P56KN1MVZV8fAzFc7LwCnAciDPS1Ys8qfhEe49891394ihqoqJn0axzkAbLao53KiQWKeR+MJPlXsV0DjaQozEw8wcf/gpoRsAa0RkEW4GDQBUdWDUVBlxj5k5PvFj6AnRFmEkFocP55mZ4xQ/r63mBUUsOQ4XgcSooFSuXIkWLY4vXDczxw9+WrlvxA1dPB7X2p0GTMQNeTQqICLCww/3BaBatcpm5jjCT5V7NC76yOfgIpaIiM2cUcEJNLWZOX6IZsQSI0nIzs7lkUc+Izf32HC7Zub4wo+hgyOWTMNfxBIjCShozb799hlcffWbx5jaiC/8DJ+sBFwP9MX15f4IN245JqV0pOGThw8fJjMzk0OHDpWzquREVdm+/QAHDx7put+gwXHUrFk1hqoqDtWrVyc9PZ0qVaoclV7i4ZNedJKXVPUq4JkyUxolMjMzqV27Ns2aNbOqYCnJz1e++24XNWtmU9PFJqBx41peuF27ttFGVdm5cyeZmZk0b97c934Rq9yqmgc0FJGE+Ek+dOgQ9evXtxuulBSYOSursB+RmbmcERHq169f7Nqmn1buDcB/ReQdYH9Boqr+vVhnKifshisdZub4oSTX24+ht3ifSkDtYp/BSBjMzIlPka3cqnqvqt4LPFyw7K0bIRARrr766sL13NxcGjZsyMUXXxzV81577bU0b96cDh060KpVK+6998i/6IYbbmDNmjUR9y/KzC+++CINGzakY8eOtGjRgn79+rFgwYLCvPfccw8zZ7rYkZ9++ilt27alQ4cOHDx4kDFjxtC2bVvGjBlTxt86fvjTn/5Ehw4d6NChAykpKYXLjz32WPkKUdWIH+AsYA2wyVtvDzxZ1H7R+px55pkajjVr1oTdVl7UrFlTO3TooAcOHFBV1ffff1/bt2+vF110UVTPO2zYMJ02bZqqqh48eFCbN2+u69ev973/oUOHdfnyrbp48Q+6ePEPunlzlubn5xduf+GFF3T06NGF67Nnz9ZGjRqFvOa//e1v9fnnny9cr127th46dMi3lsOHD/vOG4/UrFmzzI4V6voCSzSMP/y8h34U6Afs9H4AVgA2cj0C/fv357333gPg1VdfZejQoYXb9u/fz/Dhw8nIyKBjx468/fbbAGzYsIFzzz2XTp060alTp8LSb+7cufTs2ZPBgwfTqlUrrrzyyoIf2rAUNKTU9Jqne/bsScGrvptuuonOnTvTtm1bxo8fX7jP+PF3M2RIb6644nyeeebPRVaze/XqxYgRI5g0aRLgagivv/46zz77LFOnTuW+++7jyiuvZODAgezfv5+uXbvy2muvsX37dgYNGkRGRgYZGRn897//BWDChAmMGDGCvn37cs0115CXl8eYMWPIyMigXbt2PP3000Vej8WLF9O9e3fat29Ply5d2LdvX9jjBHLnnXfy5JNPFq5PmDCBv/3tb2zdupUePXrQoUMHTj/9dD799NOI1z0cb7/9Nl27dqVjx4707duXn35yU7jt2LGDgQMH0q5dO7p3786qVWUQwTqc0ws+wOfe3y8C0lYUtV+0Pr5L6H8TvU8EatasqStWrNBBgwbpwYMHtX379jpnzpzCEnrcuHH68ssvq6rq7t27tUWLFvrzzz/r/v379eDBg6qqunbtWi34nnPmzNHU1FTdvHmz5uXlabdu3fTTTz895rzDhg3TZs2aafv27bVmzZo6bty4wm3nnXeeLl68WFVVd+7cqaqqubm5et555+mKFSt0586d2rJlS83Pz9ecnFzdtWvXMccPLqFVVd9880298MILC89fUEMIXC64JgUMHTq0UP/GjRu1VatWqqo6fvx47dSpU2HN5umnn9Y//vGPqqp66NAhPfPMM3X9+vVhr0d2drY2b95cFy1apKqqWVlZevjw4bDHCWTZsmXao0ePwvXWrVvrxo0b9eGHH9b777+/8Hrt3bv3mOsSiuASeteuXYW1naeeekr/8Ic/qKrqyJEjC4//0Ucfaah7u7gltJ9Gsc0i0h1Q7/XV73CB840wtGvXjg0bNvDqq68yYMCAo7bNmDGDd955h4cffhhwpemmTZto0qQJN998M8uXLyclJYW1a48MTezSpQvp6ekAdOjQgQ0bNnDOOeccc96HHnqIwYMH8/PPP9OnTx8WLFhA9+7dj8ozdepUJk2aRG5uLlu3bmXNmjW0adOG6tWrc8MNN3DRRRf5ft7XEvQtmjlz5lHP83v37mXfPjfV+MCBA6lRw83pMGPGDFauXMnrr78OQFZWFt9++y1Vq1YNeT3q1KnDCSecQEZGBgCpqakRjxP4brdjx4789NNPbNmyhe3bt1OvXj2aNm1KRkYGw4cP5/Dhw1x66aV06FCyUPSbNm3i17/+Ndu2bSM7O5uWLVsCMH/+/MKaXN++fbn22mvZv39/Yc2qJPgx9EjcpO1puLmoZuAGbBgRGDhwIHfccQdz585l586dhemqyhtvvMFpp512VP4JEybQqFEjVqxYQX5+PtWrVy/cVq1atcLllJQUcnNzI567Vq1a9OzZk/nz5x9l6O+//56HH36Yzz9fxK5dyu23j2LPnn1UrlyZRYsWMWvWLKZMmcITTzzB7Nmzi/yOX3zxBa1bty4yXyD5+fl89tlnhcYNJPBGVlUef/xx+vXrd1SeuXPnhrweqhryESHccYIZPHgwr7/+Otu2bWPIkCEA9OjRg08++YT33nuPq6++mjFjxnDNNcWPjzl69GjuuusuBgwYwMyZM3nwwQcLtQVrLS1hDS0if1HVO4Feqnplqc9U3lwR2/Ejw4cPp06dOpxxxhnMnTu3ML1fv348/vjjPP7444gIX3zxBR07diQrK4v09HQqVarESy+9RF5eXviDF0Fubi6ff/45t9xyy1Hpe/fupWbNmuzYkceGDVtYsGA2Z555Fjt37iEvL4cBAwbQrVs3Tj311CLPMW/ePCZNmsScOXOKpa1v37488cQThS3ey5cvD1ny9evXj6eeeorevXtTpUoV1q5dS1pa+JDArVq1YsuWLSxevJiMjAz27dtHjRo1wh4nuBQcMmQIN954Izt27GDevHkAbNy4kbS0NG688Ub279/PsmXLSmTorKws0tLSUFVeeumlwvQePXrw73//m3HjxjFz5kzS09NLVTpD5BJ6gIjcDYzDDcgoNiJyIa50T8H1/34wTL4MYCHwG1V9vSTnijfS09O59dZbj0n/v//7P37/+9/Trl07VJVmzZrx7rvvMmrUKAYNGsS0adPo1atXif6xY8aM4f777ycnJ4c+ffpw2WWXHbX9jDPaccopbbjwwu6kpTWlXbsMUlOrkZ19gEsvvZRDhw6hqjzySOj5Al977TXmz5/PgQMHaN68OW+88UaxS+jHHnuM0aNH065dO3Jzc+nRowcTJ048Jt8NN9zAhg0b6NSpE6pKw4YNeeutt8Iet2rVqrz22mvccsstHDx4kBo1ajBz5kzfx2nbti379u0jLS2NE044AXC1gYceeogqVapQq1YtJk+eXKht5MiRdO4csjv1MUyYMIFf/epXpKen06VLF7Zu3QrAfffdx3XXXUe7du2oVasWL7zwgq/jRSLs4AwReQgX2KAmbqZJwQ2bFEBVNTXigV0/8LXABbiq+mJgqKquCZHvY+AQbpbLiIaONDjjq6++KvYNVlGwTiOJSah7OtLgjEivre5W1TrAe6qaqqq1A//60NIFWKeq69WNp54CXBIi3y3AG8BPPo5plAAzc8UhkqE/8/7uLeGx04DNAeuZXlohIpIG/AoX0siIAmbmikWkZ+iqIjIM6C4ilwVvVNX/FHHsUHdLcP3+UeBOVc2LdHOJyAhc9Z+mTZtGPGm41s6KiJk5sSlJq3ckQ48ErgTqAr8MPhdQlKEzgRMD1tNxgzwC6QxM8W6uBriGuFxVParVQlUnAZPAPUOHO2H16tXZuXOnDaH02L37oJk5QVFvPHTg60s/hDW0qs4H5ovIElV9rgSaFgMtRKQ58AMwBLgi6ByFb/dF5EXg3WAzF4f09HQyMzPZvn17SQ+RdBw+fIisrEOkplZn374svv461ooMvxRELCkOkd5D91bV2cDuklS5VTVXRG7GhSxKwbVgrxaRkd72Mn9urlKlSrGiO1QEVJWPP17PWWedbCVzBSBSlfs8YDbHVrfBX5UbVX0feD8oLaSRVfXaoo5nRCYnJw8RqFLlyDwIIkLfvqfEUJVRnkSqco/3/l5XfnKMklIQnbNKlRSmTBl0lKmNikNRQQJPw7Uut/KSvgImqera8HsZ5U3wxHFDhrzB1KmDSUnxMzrWSCbC/sdF5CxgLvAzroX5GVxMsbki0q1c1BlFEmoWyNNOq0+lSva8XBGJVELfg+uqOTcg7S0RmQ2MB/pHU5hRNDalqxFMpDrZKUFmBtxslMDJUVNk+MLMbIQikqH3Rdi2P8I2I8qYmY1wRKpynygioUIWCkF9so3yw8xsRCKSoSPFXA09ftGIOrfe+qGZ2QhLpPfQL4XbZsSOP/zhbD74YB2bNmWZmY1j8BNTzIgjTj65HnPmDOONN9Zwxx3dzczGUZihE5CTT67HmDFnx1qGEYdYV6I4Jjs7l6FD32Du3A2xlmIkCEUaWkT+KiKpIlJFRGaJyA4Ruao8xFVkClqzp0xZxUUXvWKmNnzhp4Tuq6p7gYtxQQtaErkF3Cglwa+mDhw4bIY2fOHH0FW8vwOAV1V1VxT1VHjCvWceP/68GKoyEgU/jWLTReRr4CAwSkQa4kLuGmWMdRoxSouf+aHH4qaU7ayqh3HdPkOF4zVKgZnZKAv8vrZKAy4QkcCIZZOjoKdCYmY2yooiDS0i44GeQBtcOKH+wHzM0GVCXl6+mdkoM/w0ig0G+gDbvHBE7YFqkXcx/JKSUokzzzyhcN3MbJQGP1Xug6qaLyK5IpKKm7LGxkOXIePH9wQgOzvPzGyUCj+GXiIidXEhiJbiQgUgRVQAABAVSURBVBItiqqqCsj48T1t1g+j1Php5R6lqnu88LsXAMMsEmjJyc7O5cEH55Odfeyk7WZmo7RECrTfSlW/FpFOIbZ1UtVl0ZWWfAS2Zi9YsJlp0y6nWjUbH2OUHZHupttxIXz/FmKbAr2joihJCX41NX36WqZOXc3VV7ePsTIjmYgU4GCE97dX+clJTsK9Z77qqnYxVGUkI0UF2j8J2K+qO7xY3OfgJnEv8YRyFQ3rNGKUJ5EC7d+Dm9tqoYjcj5vLuQFwq4g86ufgInKhiHwjIutEZGyI7VeKyErvs0BEkqr+aWY2yptIJfQQoDVwHLAJaKyqB0SkMrC8qAOLSArwT1zLeCawWETeUdU1Adm+B85T1d0i0h83Q0fXkn2V+MLMbMSCSK+tDqlqjqruAb5T1QPgpokFcnwcuwuuer5eVXOAKQQN6lDVBaq621tdiJsUPuExMxuxIlIJXdebF1qA1IA5ogWo4+PYacDmgPVMIpe+1wMfhNogIiNwLe40bdrUx6ljy86dB1m16qfCdTOzUV5EMvQ8jswN/QlHzxP9iY9jh7p7NWRGkV44Q58TaruqTsJVx+ncuXPIY8QTTZrUZu7ca+nZ80WGDDndzGyUG5FeW5W2N1gmcGLAejqwJTiTiLQDngX6q+rOUp4zbmjatA5Ll46gbt3qZmaj3Ihm1M/FQAsRaS4iVXGNbO8EZhCRpsB/gKsTec7p7OxcFi/+4Zj0evVqmJmNciVqhvYaz24GPsJNFD9VVVeLyEgRGelluweoDzwpIstFJOGm2CloAOvR40VmzPgu1nKMCo6oxv0j6VF07txZlyyJD98Ht2ZXr16ZlStH0qJF/RgrM5IZEVmqqp1DbfM1MkBEugPNAvOraoWOWBLq1dRtt3Xj1FOPj6Eqo6LjJwTRy8ApuM4keV6yUoFDENl7ZiNe8VNCdwbaaKLVzaOEmdmIZ/w0iq0CGkdbSCJgZjbiHT8ldANgjYgsArILElV1YNRUxSFmZiMR8GPoCdEWkQhMn77WzGzEPUUaWlXnlYeQeGfw4Db8+c99GDdulpnZiFv8tHJ3Ax7HDaWsCqTggh6kRllb3DF27DmcdVY6PXqcZGY24hI/jWJPAEOBb4EawA1eWlKTk5PHoUPHRuY877xmZmYjbvHV9VNV1wEpqpqnqi/gpsZJWnJy8hg8eCqXXjolpKkNI17x0yh2wBtcsVxE/gpsBWpGV1bsKDBzQQPYpZdOYfr0oVSpkhJjZYZRNH5K6Ku9fDfjppI9ERgUTVGxItjMAB07NqZy5WgOSjOMssNPK/dGEakBnKCq95aDppgQysxjx57NAw/0sWdmI2EosugRkV/i+nF/6K13EJF3Iu+VWJiZjWTBT11yAi7g3x4AVV2OG3mVFJiZjWTCj6FzVTUr6kpigJnZSDb8tHKvEpErgBQRaQH8DlgQXVnlwx13zDAzG0mFnxL6FqAtbmDGq8Be4PfRFFVejB17Di1aHO8tm5mNxKfChyDasmUfU6as4rbbupmZjYSgRCGIimrJTpbhk02a1Ob228+KtQzDKBMiPUOfhZv54lXgc0IHzk8YsrNzueqqN7n22vZcdFHLWMsxjKgQ6Rm6MXAXcDrwD9ykcztUdV6iDaksCE7w+utruOyyqbz3XsKGADeMiIQ1tDcQ40NVHQZ0A9YBc0XklnJTVwYERxrJyclj4cLMGKsyjOhQ1ITv1YCLcMMnmwGP4Wa6SAjChQ26775eMVRlGNEjUqPYS7jq9gfAvaq6qtxUlQEWA8yoiEQqoa/Gja5qCfwuwAQCaDxHLDEzGxWVSM/QlVS1tvdJDfjU9mtmEblQRL4RkXUiMjbEdhGRx7ztK0WkU2m+DJiZjYpN1Ab6ikgK8E+gP9AGGCoibYKy9QdaeJ8RwFOlOWd+vpqZjQpNNEfudwHWqep6Vc0BpgCXBOW5BJisjoVAXRE5oaQnrFRJ6NWrWeG6mdmoaPiarK6EpOE6phSQCXT1kScNF+aoEBEZgSvBadq0acST3nab6/W1ffsBM7MRXfashtotIKVqrJUUEk1Dh3JScMdxP3lQ1UnAJHB9uYs6cYGpDSNq7FgIs/tC4z5w9mtxY+poGjoTF3+sgHRgSwnyGNEmPw++ewa2fQyaH2s1x1K5JpxyAzTqGWsljh2fw5x+kLsPMt+CJaOg67OxVgVE19CLgRYi0hz4ARgCXBGU5x3gZhGZgquOZ6nqVozy49BPsOBK2DYz1kois/FV6PIsnHJdbHXsWARz+sLhvW69WkM4LX5GE0fN0KqaKyI3Ax/hZtt4XlVXi8hIb/tE4H1gAK5b6QEgxv+tCsZPn8B/h8DBBPgN1Xz4fLgrFU/7XWw07FwcZOYG0Gc21D09NnpCEM0SGlV9H2fawLSJAcsKjI6mBiMEmg9fPQQr/hc0z0sUaD0GGnSLqbRjUVj1R9i93K0uvRUO74O2d0F5NnjuXAyzL4DDXjSuavXjzswQZUMbcUj2TvjsGtgS8DtbrQGc9S9o0i92uiLRqDfMHQA7PnPrK+92pWSHB8vH1DuXHGvm3rOh7hnRP3cxsQjyFYkdC+GDjkebueHZ0P+L+DUzQNW60GsGNOpzJO2rv8KS0dFvxNu1NISZZ0G9dtE9bwkxQ1cEVOHrR+Djc+FAwGv/1mOgzxw4Lj122vxSpRb0fBfSAgLlfPsUfHYt5Edp/rFdyzwz73HrVY/3zNw+OucrA8zQyU7OHvh0ECy7HdS78avUhR5vQ8e/QqUqsdVXHFKqw7mvw0kBL0s2vAzzfw152WV7rl3LYPb5kLPbrVetB33i28xghk5udi2FDzpB5ptH0o7PcFXs9AQNCVepCpw1GU4dcSQt802YNxBy95fNOXZ9cayZe8+Ceh3K5vhRxAydjKi66uiM7rD/+yPpLW+BCz6FWs1iJq1MqJQCGROh1f8cSds2w3X2yCnlnBC7l4cw80w4vmPpjltOmKGTjcP7YMEVsHgU5Oe4tMq14Zyp0PkxSKkWW31lhQh0fAjOCJg/cft/YVZvOLSjZMfcvQJm9YGcXW69Sl3PzKUe1VtumKGTid0r4cPOsHHKkbS67eHCpdD08tjpihYicMY90OnvR9J2L4NZ58GBYvYg3r0CZgeZuU9imRnM0MmBKnz3PMzoCvsCIpqeOgL6fgapLWKnrTxodRt0eYbCsT5Za2DmufDz9xF3K2T3Smfm7J1uvUod6P0xHH9mVORGEzN0opO7HxZeB59fD3mHXFrKcXDWy9DlaahcI7b6yotTb4Dur4B4faV+Xu9e02V9HXm/PV+GNnP9kBNTxD3WUywUeYdg79duvGvWatj7FeT+zJHRnuL1UJLwy5G2SwpUqgaVqh75pFQ9er1SVfe8G5wW+MnPgWW3OY0F1GkL50yDOq3L8YLFCc2GuJFZ8y+H/Gw4+IMrqXvNCN2otWeVe+bO9p65q6S6vPUzyld3GVKxDZ13CPZ+4wxR8NmzGvavj89hhEXR/BrIeNLd1BWV9F9Cz/fhE+81VvYOmNXLpTXsfiRfSDN/DA26xEZ3GVExDJ2Xfaxxs1bDz98lpnGDSakOnf8JJ19XvgMW4pXGvV3r9Jz+rpfX4SzX4+u8t6Hx+e5He1ZvyN7u8leu7UrmBDczJKuhN051z0ZZq2HvGti3LmBUkR8Eap3sqq8Fn+oNXeMT3qdwmaD1gmVC5PWWNc9Vlws+edlHrxcnvWZTaHt33PYtjhkNusH5c1xUkeztkHcA5l4EHf4Ca/58tJl7z4AGwdGxEpPkNPSKca5RpEgEajU/2rh12kJqq4rTmJTM1OvgOtLMPh8OZB5pcyigcm3o9VEcDhktOclp6NQ2xxq6pmfcusHGPS42Go3yIfU0ON8z9c/fHUmvXAt6fQgNkyv+XHIauulgqNPG+7R1Lb4VuaGoolOrmVdSX+AewwrN3L3IXRON5DT0ycNircCIN2qcAP0Wuz7f9btBjUaxVhQVktPQhhGKyjUgPXiuh+TCeooZRhJhhjaMJMIMbRhJhBnaMJIIM7RhJBFmaMNIIszQhpFEiGqRs7PGFSKyHdhYRLYGQAkDS5U5piU08aQF4ktPUVpOUtWGoTYknKH9ICJLVDUuQk6YltDEkxaILz2l0WJVbsNIIszQhpFEJKuhJ8VaQACmJTTxpAXiS0+JtSTlM7RhVFSStYQ2jAqJGdowkoiENrSIXCgi34jIOhEZG2K7iMhj3vaVIhK1eU18aLnS07BSRBaISNTmJS1KS0C+DBHJE5HBsdQiIj1FZLmIrBaRebHSIiJ1RGS6iKzwtFwXRS3Pi8hPIrIqzPaS3buqmpAfIAX4DjgZqAqsANoE5RkAfICLct8N+DyGWroD9bzl/rHUEpBvNvA+MDiG16UusAZo6q3/IoZa7gL+4i03BHYBVaOkpwfQCVgVZnuJ7t1ELqG7AOtUdb2q5gBTgOBwFJcAk9WxEKgrIifEQouqLlBVb45SFgLpUdDhS4vHLcAbwE9R0uFXyxXAf1R1E4CqRkuPHy0K1BYRAWrhDJ0bDTGq+ol3/HCU6N5NZEOnAZsD1jO9tOLmKS8tgVyP+/WNBkVqEZE04FfAxChp8K0FaAnUE5G5IrJURK6JoZYngNbAFuBL4FbVmM3EUKJ7N5FjioWaIiL4HZyfPOWlxWUU6YUz9DlR0OFXy6PAnaqaJ9GdacOPlsrAmUAfoAbwmYgsVNW1x+wZfS39gOVAb+AU4GMR+VRV95axFj+U6N5NZENnAicGrKfjflmLm6e8tCAi7YBngf6qujMKOvxq6QxM8czcABggIrmq+lYMtGQCO1R1P7BfRD4B2gNlbWg/Wq4DHlT3ELtORL4HWgGLyliLH0p270bjgb88Prgfo/VAc440crQNynMRRzcsLIqhlqbAOqB7rK9LUP4XiV6jmJ/r0hqY5eU9DlgFnB4jLU8BE7zlRsAPQIMo/q+aEb5RrET3bsKW0KqaKyI3Ax/hWjCfV9XVIjLS2z4R14I7AGekA7hf4FhpuQeoDzzplYy5GoXRPT61lAt+tKjqVyLyIbASyAeeVdWQr3KirQX4I/CiiHyJM9KdqhqVIZUi8irQE2ggIpnAeKBKgJYS3bvW9dMwkohEbuU2DCMIM7RhJBFmaMNIIszQhpFEmKENI4lI2NdWRnQQkfq498IAjYE8YLu33kVdP2gjTrHXVkZYRGQC8LOqPhxrLYY/rMpt+MYbK7zUGyt8Q0D6VSLypYisEpEHYqmxomNVbqM4DFPVXSJyHLBERN4AagL34/qHZwEzReRiVX03lkIrKlZCG8XhNhFZAXyGGyxwCtAVmK2qO1T1MPAKbvC+EQOshDZ8ISLn44zaTVUPish8oDqhh/kZMcJKaMMvdYBdnpnbAhle+kKgl4jUF5HKwBAganHBjMhYCW345T1ghFfl/hr4HEBVM0XkHmAurrSerqrvxUxlBcdeWxlGEmFVbsNIIszQhpFEmKENI4kwQxtGEmGGNowkwgxtGEmEGdowkoj/B1NsvlaMJCEYAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plot\n", + "figure = plot.figure()\n", + "axe = figure.gca()\n", + "axe.plot(tau_list[:-1], mean_difference, lw=3,color='orange', label=\"Mean Bias Difference vs. Tao\")\n", + "axe.set_xlabel(\"Tao\")\n", + "axe.set_ylabel(\"Mean Bias Difference vs. Tao\")\n", + "axe.set_title(\"Mean Bias Difference vs. Tao\")\n", + "axe.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n", + "axe.legend()\n", + "axe.set_aspect('equal')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Interpretation\n", + "\n", + "The graph shows that, for almost all the Tao values, the accuracy goes down as the tao goes up. The mean bia difference is loves when tao is 0.88, coming all the way down to -0.015 from -0.19 biased model. We can clearly observe a trade-off between the accuracy score and bias reduction, as expected." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Other Metrices to Use\n", + "1. FDR (False Discovery Rate): is a method of conceptualizing the rate of type I errors in null hypothesis testing when conducting multiple comparisons. (from Wikipedia) The idea of false discovery rate is to make the total number of FALSE discoveries (false positive + false negative) LOWER as a percentage of the overall discoveries." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### FDR Code Implementation\n", + "\n", + "We implement the FDR metric to assess the \"accuracy\" of the model from another bia related metric. Below we compare the best model from the iterations above, with the unconstrained model. This proves the reduction in bias. " + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy : 3254 14653 0.777929434245547\n", + "SR tau : 0.24060062422550277\n", + "FPR tau : 0.43600739741090616\n", + "FNR tau : 0.8609189723320158\n", + "TPR tau : 0.31004901960784315\n", + "TNR tau : 0.980340417865866\n", + "AR tau : 0.8224361467717171\n", + "FDR tau : 0.4288194444444444\n", + "FOR tau : 0.3872744450835204\n", + "PPR tau : 0.4588815789473684\n", + "NPR tau : 0.8136439105306991\n", + "0.4288194444444444\n" + ] + } + ], + "source": [ + "from aif360.algorithms.inprocessing.celisMeta.utils import getStats\n", + "\n", + "#Unconstrained Model\n", + "mfc = MetaFairClassifier(tau=0, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)\n", + "dataset_bias_test = mfc.predict(dataset_orig_test)\n", + "\n", + "\n", + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in list(dataset_bias_test.labels)]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "x_control_test = pd.DataFrame(data=dataset_orig_test.features, columns=dataset_orig_test.feature_names)[\"sex\"]\n", + "\n", + "acc, sr, unconstrainedFDR = getStats(y_test, predictions, x_control_test)\n", + "print(unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": 130, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training Accuracy: 0.5383017929743485 , Training gamma: 0.6980020035235538\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'biased_model' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mmfc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMetaFairClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.88\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msensitive_attr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"sex\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mmfc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_orig_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mdataset_bias_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbiased_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_orig_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_debiasing_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'biased_model' is not defined" + ] + } + ], + "source": [ + "# the best model from the iterations above had a TAO of 0.88.\n", + "mfc = MetaFairClassifier(tau=0.88, sensitive_attr=\"sex\")\n", + "mfc.fit(dataset_orig_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy : 5672 14653 0.6129120316658705\n", + "SR tau : 0.8000013123449397\n", + "FPR tau : 0.9130276132539338\n", + "FNR tau : 0.6943346508563901\n", + "TPR tau : 0.9308788818438468\n", + "TNR tau : 0.9253680086167871\n", + "AR tau : 0.93946485365946\n", + "FDR tau : 0.6808922089217645\n", + "FOR tau : 0.39480000000000004\n", + "PPR tau : 0.41435204806001147\n", + "NPR tau : 0.9341294990910202\n", + "0.6808922089217645 0.4288194444444444\n" + ] + } + ], + "source": [ + "dataset_bias_test = mfc.predict(dataset_orig_test)\n", + "\n", + "predictions = list(dataset_debiasing_test.labels)\n", + "predictions = [1 if y == dataset_orig_train.favorable_label else -1 for y in predictions]\n", + "y_test = np.array([1 if y == [dataset_orig_train.favorable_label] else -1 for y in dataset_orig_test.labels])\n", + "x_control_test = pd.DataFrame(data=dataset_orig_test.features, columns=dataset_orig_test.feature_names)[\"sex\"]\n", + "\n", + "acc, sr, fdr = getStats(y_test, predictions, x_control_test)\n", + "print(fdr, unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": 132, + "metadata": {}, + "outputs": [], + "source": [ + "assert(fdr >= unconstrainedFDR)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}