From ed5ef205bca90ed8077eaead2cb6836a006f7074 Mon Sep 17 00:00:00 2001 From: vielleicht-07 Date: Thu, 14 Nov 2024 18:37:16 +0100 Subject: [PATCH] Create main.ipynb --- main.ipynb | 1898 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1898 insertions(+) create mode 100644 main.ipynb diff --git a/main.ipynb b/main.ipynb new file mode 100644 index 0000000..906ea66 --- /dev/null +++ b/main.ipynb @@ -0,0 +1,1898 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Before your start:\n", + "- Read the README.md file\n", + "- Comment as much as you can and use the resources (README.md file)\n", + "- Happy learning!\n", + "\n", + "- **Consider a significance level of 5% for all tests.**" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# import numpy and pandas\n", + "from scipy import stats #remenver stats.\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 1 - Independent Sample T-tests\n", + "\n", + "In this challenge, we will be using the Pokemon dataset. Before applying statistical methods to this data, let's first examine the data.\n", + "\n", + "To load the data, run the code below." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Run this code:\n", + "\n", + "pokemon = pd.read_csv('pokemon.csv')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start off by looking at the `head` function in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
207192SunfloraGrassNaN42575755510585302False
240222CorsolaWaterRock3805555856585352False
\n", + "
" + ], + "text/plain": [ + " # Name Type 1 Type 2 Total HP Attack Defense Sp. Atk \\\n", + "207 192 Sunflora Grass NaN 425 75 75 55 105 \n", + "240 222 Corsola Water Rock 380 55 55 85 65 \n", + "\n", + " Sp. Def Speed Generation Legendary \n", + "207 85 30 2 False \n", + "240 85 35 2 False " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "\n", + "pokemon.sample(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Legendary\n", + "False 735\n", + "True 65\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pokemon[\"Legendary\"].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(800, 13)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pokemon.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 800 entries, 0 to 799\n", + "Data columns (total 13 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 # 800 non-null int64 \n", + " 1 Name 800 non-null object\n", + " 2 Type 1 800 non-null object\n", + " 3 Type 2 414 non-null object\n", + " 4 Total 800 non-null int64 \n", + " 5 HP 800 non-null int64 \n", + " 6 Attack 800 non-null int64 \n", + " 7 Defense 800 non-null int64 \n", + " 8 Sp. Atk 800 non-null int64 \n", + " 9 Sp. Def 800 non-null int64 \n", + " 10 Speed 800 non-null int64 \n", + " 11 Generation 800 non-null int64 \n", + " 12 Legendary 800 non-null bool \n", + "dtypes: bool(1), int64(9), object(3)\n", + "memory usage: 75.9+ KB\n" + ] + } + ], + "source": [ + "pokemon.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first thing we would like to do is compare the legendary Pokemon to the regular Pokemon. To do this, we should examine the data further. What is the count of legendary vs. non legendary Pokemons?" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
670609ChandelureGhostFire52060559014590805False
\n", + "
" + ], + "text/plain": [ + " # Name Type 1 Type 2 Total HP Attack Defense Sp. Atk \\\n", + "670 609 Chandelure Ghost Fire 520 60 55 90 145 \n", + "\n", + " Sp. Def Speed Generation Legendary \n", + "670 90 80 5 False " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "\n", + "df = pokemon.copy()\n", + "df.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Legendary\n", + "False 735\n", + "True 65\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"Legendary\"].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "regular_pokemons = df[df[\"Legendary\"] == True]\n", + "#regular_pokemons.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Legendary\n", + "True 65\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "regular_pokemons.Legendary.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "legendary_pokemons = df[df[\"Legendary\"] == False]\n", + "# legendary_pokemons.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Legendary\n", + "False 735\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "legendary_pokemons.Legendary.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Compute the mean and standard deviation of the total points for both legendary and non-legendary Pokemon." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 361, + "metadata": {}, + "outputs": [], + "source": [ + "#df.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 355, + "metadata": {}, + "outputs": [], + "source": [ + "#regular_pokemons.describe() # 60.94" + ] + }, + { + "cell_type": "code", + "execution_count": 357, + "metadata": {}, + "outputs": [], + "source": [ + "#legendary_pokemons.describe() #106.76 " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The computation of the mean might give us a clue regarding how the statistical test may turn out; However, it certainly does not prove whether there is a significant difference between the two groups.\n", + "\n", + "In the cell below, use the `ttest_ind` function in `scipy.stats` to compare the the total points for legendary and non-legendary Pokemon. Since we do not have any information about the population, assume the variances are not equal." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "from scipy import stats #ttest_ind()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "#out nans\n", + "df_legen_test = regular_pokemons.copy()\n", + "df_regular_test = legendary_pokemons.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# df_numeric_Data = df.select_dtypes(include=[\"number\"])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
552493ArceusNormalNaN7201201201201201201204True
\n", + "
" + ], + "text/plain": [ + " # Name Type 1 Type 2 Total HP Attack Defense Sp. Atk \\\n", + "552 493 Arceus Normal NaN 720 120 120 120 120 \n", + "\n", + " Sp. Def Speed Generation Legendary \n", + "552 120 120 4 True " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_legen_test.sample(1)\n", + "#df_regular_test.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "361" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_legen_test[\"Type 2\"].isnull().sum() # 25\n", + "df_regular_test[\"Type 2\"].isnull().sum() # 361" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + " df_legen_test.drop([\"Type 1\",\"Type 2\",\"Name\" , \"Generation\",\"Legendary\", \"#\"], axis=1, inplace=True)\n", + " df_regular_test.drop([\"Type 1\",\"Type 2\",\"Name\", \"Generation\",\"Legendary\", \"#\"], axis=1, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "# df_legen_test.sample(1)\n", + "# df_regular_test.sample(1)\n", + "# Total\tHP\tAttack\tDefense\tSp. Atk\tSp. Def\tSpeed" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "t_stat, p_value = stats.ttest_ind(df_regular_test[\"Total\"], df_legen_test[\"Total\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-16.386116965872425" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t_stat" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3e-52" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "p_value.round(52)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What do you conclude from this test? Write your conclusions below." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "# Your conclusions here: we have a significance difference between legenday pokemons and regular pokemons, so we can reject Ho and work with H1\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "How about we try to compare the different types of pokemon? In the cell below, list the types of Pokemon from column `Type 1` and the count of each type." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n", + "df_types = pokemon.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
218203GirafarigNormalPsychic4557080659065852False
\n", + "
" + ], + "text/plain": [ + " # Name Type 1 Type 2 Total HP Attack Defense Sp. Atk \\\n", + "218 203 Girafarig Normal Psychic 455 70 80 65 90 \n", + "\n", + " Sp. Def Speed Generation Legendary \n", + "218 65 85 2 False " + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_types.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Grass', 'Fire', 'Water', 'Bug', 'Normal', 'Poison', 'Electric',\n", + " 'Ground', 'Fairy', 'Fighting', 'Psychic', 'Rock', 'Ghost', 'Ice',\n", + " 'Dragon', 'Dark', 'Steel', 'Flying'], dtype=object)" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_types[\"Type 1\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 199, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "count 800\n", + "unique 18\n", + "top Water\n", + "freq 112\n", + "Name: Type 1, dtype: object" + ] + }, + "execution_count": 199, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_types[\"Type 1\"].describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 203, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Type 1\n", + "Water 112\n", + "Normal 98\n", + "Grass 70\n", + "Bug 69\n", + "Psychic 57\n", + "Fire 52\n", + "Electric 44\n", + "Rock 44\n", + "Dragon 32\n", + "Ground 32\n", + "Ghost 32\n", + "Dark 31\n", + "Poison 28\n", + "Steel 27\n", + "Fighting 27\n", + "Ice 24\n", + "Fairy 17\n", + "Flying 4\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 203, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_types[\"Type 1\"].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since water is the largest group of Pokemon, compare the mean and standard deviation of water Pokemon to all other Pokemon." + ] + }, + { + "cell_type": "code", + "execution_count": 294, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "df_type_water = df_types[df_types[\"Type 1\"] == \"Water\"]\n", + "df_no_water = df_types[df_types[\"Type 1\"] != \"Water\"]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 302, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
353323CameruptFireGround460701007010575403False
\n", + "
" + ], + "text/plain": [ + " # Name Type 1 Type 2 Total HP Attack Defense Sp. Atk \\\n", + "353 323 Camerupt Fire Ground 460 70 100 70 105 \n", + "\n", + " Sp. Def Speed Generation Legendary \n", + "353 75 40 3 False " + ] + }, + "execution_count": 302, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_no_water.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 304, + "metadata": {}, + "outputs": [], + "source": [ + "#df_no_water[\"Type 1\"].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 307, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean for water type: 430.45535714285717\n", + "Std for water type: 113.18826606431458\n" + ] + } + ], + "source": [ + "water_pokes_mean = df_type_water[\"Total\"].mean()\n", + "water_pokes_std = df_type_water[\"Total\"].std()\n", + "\n", + "print(\"Mean for water type:\", water_pokes_mean)\n", + "print(\"Std for water type:\",water_pokes_std)" + ] + }, + { + "cell_type": "code", + "execution_count": 313, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "For water type pokemon, mean: 435.85901162790697\n", + "For water type pokemon, std: 121.09168230208066\n" + ] + } + ], + "source": [ + "no_water_all_mean = df_no_water[\"Total\"].mean()\n", + "no_water_all_std = df_no_water[\"Total\"].std()\n", + "\n", + "print(\"For water type pokemon, mean:\", no_water_all_mean)\n", + "print(\"For water type pokemon, std:\",no_water_all_std)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform a hypothesis test comparing the mean of total points for water Pokemon to all non-water Pokemon. Assume the variances are equal. " + ] + }, + { + "cell_type": "code", + "execution_count": 320, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n", + "#h0 Water type Pokemon and other types are equally. \n", + "#h1 water type Pokemon are stronger than other types of Pokemon." + ] + }, + { + "cell_type": "code", + "execution_count": 344, + "metadata": {}, + "outputs": [], + "source": [ + "# df_no_water.sample(1)\n", + "# df_type_water.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 347, + "metadata": {}, + "outputs": [], + "source": [ + "t_stat, p_value = stats.ttest_ind(df_type_water[\"Total\"], df_no_water[\"Total\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 349, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "t_stat -0.4418547448849676\n", + "p_value 0.6587140317488793\n" + ] + } + ], + "source": [ + "print(\"t_stat\",t_stat)\n", + "print(\"p_value\",p_value)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Write your conclusion below." + ] + }, + { + "cell_type": "code", + "execution_count": 353, + "metadata": {}, + "outputs": [], + "source": [ + "# Your conclusions here: #we get a p-value 0.66, greater than 0.05 \"alpha\", we cannot reject the null hypothesis. \n", + "#we dont have enough evidence to say water type Pokémon are stronger than other types.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 2 - Matched Pairs Test\n", + "\n", + "In this challenge we will compare dependent samples of data describing our Pokemon. Our goal is to see whether there is a significant difference between each Pokemon's defense and attack scores. Our hypothesis is that the defense and attack scores are equal. In the cell below, import the `ttest_rel` function from `scipy.stats` and compare the two columns to see if there is a statistically significant difference between them." + ] + }, + { + "cell_type": "code", + "execution_count": 419, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n", + "from scipy import stats #ttest_rel()\n", + "\n", + "# Ho is that the defense and attack scores are equal" + ] + }, + { + "cell_type": "code", + "execution_count": 421, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [#, Name, Type 1, Type 2, Total, HP, Attack, Defense, Sp. Atk, Sp. Def, Speed, Generation, Legendary]\n", + "Index: []" + ] + }, + "execution_count": 421, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_defense = pokemon.copy()\n", + "df_attack = pokemon.copy()\n", + "df.sample(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 423, + "metadata": {}, + "outputs": [], + "source": [ + "df_defense.drop(\"Attack\", axis=1, inplace=True)\n", + "df_attack.drop(\"Defense\", axis=1, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 427, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackSp. AtkSp. DefSpeedGenerationLegendary
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [#, Name, Type 1, Type 2, Total, HP, Attack, Sp. Atk, Sp. Def, Speed, Generation, Legendary]\n", + "Index: []" + ] + }, + "execution_count": 427, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_attack.head(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 432, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "t_stat -3.241764074042312\n", + "p_value 0.0012123980547321484\n" + ] + } + ], + "source": [ + "t_stat, p_value = stats.ttest_ind(df_defense[\"Defense\"], df_attack[\"Attack\"])\n", + "print(\"t_stat\",t_stat)\n", + "print(\"p_value\",p_value)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Describe the results of the test in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": 444, + "metadata": {}, + "outputs": [], + "source": [ + "# Your conclusions here: Defense and attack are not equal, p-value is 0,0012, lower than 0,05, so we reject the Ho.\n", + "\n", + "# Ho, defense and attack scores are equal" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are also curious about whether therer is a significant difference between the mean of special defense and the mean of special attack. Perform the hypothesis test in the cell below. " + ] + }, + { + "cell_type": "code", + "execution_count": 468, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "# Ho, sp.defense and sp.attack scores are equal\n", + "# " + ] + }, + { + "cell_type": "code", + "execution_count": 462, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [#, Name, Type 1, Type 2, Total, HP, Attack, Defense, Sp. Atk, Sp. Def, Speed, Generation, Legendary]\n", + "Index: []" + ] + }, + "execution_count": 462, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 464, + "metadata": {}, + "outputs": [], + "source": [ + "df_sp_atk_def = df.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": 466, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "t_stat 0.6041290031014401\n", + "p_value 0.5458436328840358\n" + ] + } + ], + "source": [ + "t_stat, p_value = stats.ttest_ind(df_sp_atk_def[\"Sp. Atk\"], df_sp_atk_def[\"Sp. Def\"])\n", + "print(\"t_stat\",t_stat)\n", + "print(\"p_value\",p_value)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Describe the results of the test in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": 474, + "metadata": {}, + "outputs": [], + "source": [ + "# Your conclusions here: P-value 0,54 more than 0,05 so, we can not reject Ho\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you may recall, a two sample matched pairs test can also be expressed as a one sample test of the difference between the two dependent columns.\n", + "\n", + "Import the `ttest_1samp` function and perform a one sample t-test of the difference between defense and attack. Test the hypothesis that the difference between the means is zero. Confirm that the results of the test are the same." + ] + }, + { + "cell_type": "code", + "execution_count": 650, + "metadata": {}, + "outputs": [], + "source": [ + "###IDK :c" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 653, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 608, + "metadata": {}, + "outputs": [], + "source": [ + "df_test_attak_deff = df.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": 610, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#NameType 1Type 2TotalHPAttackDefenseSp. AtkSp. DefSpeedGenerationLegendary
413376MetagrossMega MetagrossSteelPsychic700801451501051101103False
\n", + "
" + ], + "text/plain": [ + " # Name Type 1 Type 2 Total HP Attack Defense \\\n", + "413 376 MetagrossMega Metagross Steel Psychic 700 80 145 150 \n", + "\n", + " Sp. Atk Sp. Def Speed Generation Legendary \n", + "413 105 110 110 3 False " + ] + }, + "execution_count": 610, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + " df_test_attak_deff.sample(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 629, + "metadata": {}, + "outputs": [], + "source": [ + "diff= df_test_attak_deff[\"Defense\"] - df_test_attak_deff[\"Attack\"]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 633, + "metadata": {}, + "outputs": [], + "source": [ + "t_stat_deff, p_value = stats.ttest_1samp( diff, 0) #data, popmean\n" + ] + }, + { + "cell_type": "code", + "execution_count": 637, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1.7140303479358558e-05" + ] + }, + "execution_count": 637, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "p_value # 0.000017" + ] + }, + { + "cell_type": "code", + "execution_count": 639, + "metadata": {}, + "outputs": [], + "source": [ + "# popmean_deff = df_test_attak_deff[\"Attack\"].mean()\n", + "# popmean_attak = df_test_attak_deff[\"Defense\"].mean()\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bonus Challenge - The Chi-Square Test\n", + "\n", + "The Chi-Square test is used to determine whether there is a statistically significant difference in frequencies. In other words, we are testing whether there is a relationship between categorical variables or rather when the variables are independent. This test is an alternative to Fisher's exact test and is used in scenarios where sample sizes are larger. However, with a large enough sample size, both tests produce similar results. Read more about the Chi Squared test [here](https://en.wikipedia.org/wiki/Chi-squared_test).\n", + "\n", + "In the cell below, create a contingency table using `pd.crosstab` comparing whether a Pokemon is legenadary or not and whether the Type 1 of a Pokemon is water or not." + ] + }, + { + "cell_type": "code", + "execution_count": 655, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform a chi-squared test using the `chi2_contingency` function in `scipy.stats`. You can read the documentation of the function [here](https://docs.scipy.org/doc/scipy-0.15.1/reference/generated/scipy.stats.chi2_contingency.html)." + ] + }, + { + "cell_type": "code", + "execution_count": 664, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Based on a 95% confidence, should we reject the null hypothesis?" + ] + }, + { + "cell_type": "code", + "execution_count": 661, + "metadata": {}, + "outputs": [], + "source": [ + "# Your answer here:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}