From 7e8e8371189be42f63a5804d5f62d81c70c7d780 Mon Sep 17 00:00:00 2001 From: adris Date: Mon, 20 Nov 2023 08:44:39 +0000 Subject: [PATCH] Lab Done --- 04_lab_hypothesis_testing_1.ipynb | 682 +++++++++++++++++++++++++++++ .gitignore => your-code/.gitignore | 0 README.md => your-code/README.md | 0 3 files changed, 682 insertions(+) create mode 100644 04_lab_hypothesis_testing_1.ipynb rename .gitignore => your-code/.gitignore (100%) rename README.md => your-code/README.md (100%) mode change 100755 => 100644 diff --git a/04_lab_hypothesis_testing_1.ipynb b/04_lab_hypothesis_testing_1.ipynb new file mode 100644 index 0000000..ff08989 --- /dev/null +++ b/04_lab_hypothesis_testing_1.ipynb @@ -0,0 +1,682 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "MuMJueZDc-Ko" + }, + "source": [ + "# Before your start:\n", + "- Read the README.md file\n", + "- Comment as much as you can and use the resources (README.md file)\n", + "- Happy learning!" + ] + }, + { + "cell_type": "code", + "source": [ + "# From README.md file\n", + "\n", + "# [T Test](http://b.link/test50)\n", + "# [Hypothesis Tests in SciPy](http://b.link/scipy65)\n", + "# [Standard Error in SciPy](http://b.link/scipy86)\n", + "# [One Sample Tests of Proportions](http://b.link/categorical45)" + ], + "metadata": { + "id": "67W6N1tXdVRF" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "xK4mIEOlc-Kr" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_GLjpxJVc-Ks" + }, + "source": [ + "# Challenge 1 - Exploring the Data\n", + "\n", + "In this challenge, we will examine all salaries of employees of the City of Chicago. We will start by loading the dataset and examining its contents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pPDgTMt1c-Kt" + }, + "outputs": [], + "source": [ + "df = pd.read_csv(\"/content/Current_Employee_Names__Salaries__and_Position_Titles.csv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lKbImXXDc-Kt" + }, + "source": [ + "Examine the `salaries` dataset using the `head` function below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jQoiexh7c-Kt" + }, + "outputs": [], + "source": [ + "print(df.columns)\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8QhwSAeWc-Ku" + }, + "source": [ + "We see from looking at the `head` function that there is quite a bit of missing data. Let's examine how much missing data is in each column. Produce this output in the cell below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pXQLj1TPc-Ku" + }, + "outputs": [], + "source": [ + "print(f\"The current shape of our dataframe is {df.shape} \\nNumber of nulls is:\")\n", + "df.isnull().sum()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I-XBo0d9c-Ku" + }, + "source": [ + "Let's also look at the count of hourly vs. salaried employees. Write the code in the cell below" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "gdpUw2pfc-Kv", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "0049e4c5-37b3-45c7-b3d6-3b3b4efac62e" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Total number of employees is 33183\n", + "Total number of employees paid with salary is 25161 so the proportion is 75.82497061748485 %\n", + "Total number of employees paid hourly is 8022 so the proportion is 24.17502938251514 %\n" + ] + } + ], + "source": [ + "# count of salary or hourly (officers that were paid by salary ou by hour) then use that number to get te proportion of officers\n", + "\n", + "count_employees = df[\"Salary or Hourly\"].count() # no nulls here, so I know i have all officers, the 33.183 is accurate\n", + "print(\"Total number of employees is\", count_employees)\n", + "\n", + "s_employees = df[df[\"Salary or Hourly\"] == \"Salary\"].shape[0]\n", + "print(\"Total number of employees paid with salary is\", s_employees,\"so the proportion is\", (s_employees/count_employees)*100,\"%\")\n", + "\n", + "h_employees = df[df[\"Salary or Hourly\"] == \"Hourly\"].shape[0]\n", + "print(\"Total number of employees paid hourly is\", h_employees,\"so the proportion is\", (h_employees/count_employees)*100,\"%\")" + ] + }, + { + "cell_type": "code", + "source": [ + "nulls_in_column = df[\"Annual Salary\"].isnull().sum()\n", + "nulls_in_column ### That answers the nulls in this column (number of officers paid hourly and NOT Salary, won't have annual salary)" + ], + "metadata": { + "id": "8qawfcuScDSA", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "76faf429-8981-439b-c2c9-ebe5fa9f0001" + }, + "execution_count": 12, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "8022" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CqL98vbYc-Kv" + }, + "source": [ + "What this information indicates is that the table contains information about two types of employees - salaried and hourly. Some columns apply only to one type of employee while other columns only apply to another kind. This is why there are so many missing values. Therefore, we will not do anything to handle the missing values." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1QpAPj8Hc-Kv" + }, + "source": [ + "There are different departments in the city. List all departments and the count of employees in each department." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "P66B-rfWc-Kv", + "outputId": "6338d4c3-b5a8-4387-ebab-6c04154bd62d" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "You have a total of 35 departments named:\n", + " ['POLICE' 'GENERAL SERVICES' 'WATER MGMNT' 'OEMC' 'CITY COUNCIL'\n", + " 'AVIATION' 'STREETS & SAN' 'FIRE' 'FAMILY & SUPPORT' 'PUBLIC LIBRARY'\n", + " 'TRANSPORTN' \"MAYOR'S OFFICE\" 'HEALTH' 'BUSINESS AFFAIRS' 'LAW' 'FINANCE'\n", + " 'CULTURAL AFFAIRS' 'COMMUNITY DEVELOPMENT' 'PROCUREMENT' 'BUILDINGS'\n", + " 'ANIMAL CONTRL' 'CITY CLERK' 'BOARD OF ELECTION' 'DISABILITIES'\n", + " 'HUMAN RESOURCES' 'DoIT' 'BUDGET & MGMT' 'TREASURER' 'INSPECTOR GEN'\n", + " 'HUMAN RELATIONS' 'COPA' 'BOARD OF ETHICS' 'POLICE BOARD' 'ADMIN HEARNG'\n", + " 'LICENSE APPL COMM']\n" + ] + } + ], + "source": [ + "departments_count = df[\"Department\"].nunique() # number of different departments\n", + "departments = df[\"Department\"].unique() # describing the different departments\n", + "print(\"You have a total of\",departments_count,\" departments named:\\n\",departments)" + ] + }, + { + "cell_type": "code", + "source": [ + "employees_by_depart = df[\"Department\"].value_counts() # employees by department\n", + "print(\"You have this number of by department:\\n\")\n", + "employees_by_depart" + ], + "metadata": { + "id": "OoJ4YztHeIHu" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "q3xb0g_Fc-Kw" + }, + "source": [ + "# Challenge 2 - Hypothesis Tests\n", + "\n", + "In this section of the lab, we will test whether the hourly wage of all hourly workers is significantly different from $30/hr. Import the correct one sample test function from scipy and perform the hypothesis test for a 95% two sided confidence interval." + ] + }, + { + "cell_type": "code", + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import scipy.stats as st" + ], + "metadata": { + "id": "wbceCL5LRVwD" + }, + "execution_count": 16, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TNKmukrfc-Kw" + }, + "outputs": [], + "source": [ + "df[df[\"Salary or Hourly\"] == \"Hourly\"][\"Hourly Rate\"].mean() # just out of curiosity" + ] + }, + { + "cell_type": "code", + "source": [ + "# 1. Set the hypothesis\n", + "\n", + "### H0: hour rate = 30 dls/hr\n", + "### H1: hour rate != 30 dls/hr\n", + "\n", + "# 2. Significance level\n", + "alpha = 0.05\n", + "\n", + "# 3. Define the sample\n", + "sample_hour = df[df[\"Salary or Hourly\"] == \"Hourly\"][\"Hourly Rate\"] # values of the hour/ should I sample this ?\n", + "\n", + "# 4.\n", + "print(st.ttest_1samp(sample_hour, 30)) # value of null hypothesis\n", + "p_value = st.ttest_1samp(sample_hour, 30)[1]\n", + "# this is the closest to zero I have ever seen, so I am actually not sure :)\n", + "\n", + "if p_value > 0.05:\n", + " print(\"I can not reject the null hypothesis\")\n", + "else:\n", + " print(\"We can reject the null hypothesis\")" + ], + "metadata": { + "id": "aM6WyhmJQVlV", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "c73db5f0-938e-42ed-d1e9-4fb4fec47653" + }, + "execution_count": 18, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "TtestResult(statistic=20.6198057854942, pvalue=4.3230240486229894e-92, df=8021)\n", + "We can reject the null hypothesis\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Thinking out loud: by just checking the sample, we can see how much the value can differ and if the null is low, it has to goooooo!\n", + "\n", + "# boolean mask to check if I have someone that earns 30 dls an hour.. and it's a big NO NO\n", + "count_30dls_hour = df[(df[\"Salary or Hourly\"] == \"Hourly\") & (df[\"Hourly Rate\"] == 30)]\n", + "count_30dls_hour" + ], + "metadata": { + "id": "U1wm3aLBWW2H" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Yz3Z39AZc-Kw" + }, + "source": [ + "We are also curious about salaries in the police force. The chief of police in Chicago claimed in a press briefing that salaries this year are higher than last year's mean of $86000/year a year for all salaried employees. Test this one sided hypothesis using a 95% confidence interval.\n", + "\n", + "Hint: A one tailed test has a p-value that is half of the two tailed p-value. If our hypothesis is greater than, then to reject, the test statistic must also be positive." + ] + }, + { + "cell_type": "code", + "source": [ + "df[df[\"Salary or Hourly\"] == \"Salary\"][\"Annual Salary\"].mean() # just checking" + ], + "metadata": { + "id": "a-Sa1lB1aYjG" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "id": "O8IwIO4mc-Kw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "e8c5d211-683e-4f0c-cd86-fc47b1c5780c" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "TtestResult(statistic=1.686027692770258, pvalue=0.9525301346031133, df=99)\n", + "I can not reject the null hypothesis\n" + ] + } + ], + "source": [ + "# Testing Fixed Values\n", + "\n", + "# 1. Set the hypothesis\n", + "# H0: mu salaries >= 86000\n", + "# H1: mu salaries < 86000\n", + "\n", + "#2. significance level\n", + "alpha = 0.05\n", + "\n", + "# sample\n", + "sample_salary = df[df[\"Salary or Hourly\"] == \"Salary\"][\"Annual Salary\"].sample(100) ### sample ?\n", + "\n", + "print(st.ttest_1samp(sample_salary, 86000, alternative = \"less\"))\n", + "# Bottom line, if is going right(+ sign) we cannot reject the null, and if p > 0,05\n", + "\n", + "p_value = st.ttest_1samp(sample_salary, 86000, alternative = \"less\")[1]\n", + "p_value\n", + "stats = st.ttest_1samp(sample_salary, 86000, alternative = \"less\")[0]\n", + "stats\n", + "\n", + "if p_value > 0.05 and stats > 0:\n", + " print(\"I can not reject the null hypothesis\")\n", + "else:\n", + " print(\"We can reject the null hypothesis\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XBhlH9zUc-Kw" + }, + "source": [ + "Using the `crosstab` function, find the department that has the most hourly workers." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "id": "lU_kacHvc-Kw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "a5f399ec-1a66-4250-df77-aa40d6927c63" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "The department with the most hourly workers is STREETS & SAN with 1862 employees\n" + ] + } + ], + "source": [ + "# This \"cross_tab\" thing I asked \"a friend\"\n", + "crosstab = pd.crosstab(df[\"Department\"], df[\"Salary or Hourly\"])\n", + "hourly_workers_dep = crosstab[\"Hourly\"].idxmax()\n", + "\n", + "# now me:\n", + "employees_on_depart = df[(df[\"Department\"] == \"STREETS & SAN\") & (df[\"Salary or Hourly\"] == \"Hourly\")].shape[0]\n", + "print(\"The department with the most hourly workers is\",hourly_workers_dep, \"with\",employees_on_depart,\"employees\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QR2fXavlc-Kx" + }, + "source": [ + "The workers from the department with the most hourly workers have complained that their hourly wage is less than $35/hour. Using a one sample t-test, test this one-sided hypothesis at the 95% confidence level." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "id": "QORVwz04c-Kx", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "3d6d48d2-3d34-4d5e-dd59-01cdeb8320d2" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "TtestResult(statistic=-3.499344374335869, pvalue=0.00035029734988668436, df=99)\n", + "We can reject the null hypothesis\n" + ] + } + ], + "source": [ + "# Testing Fixed Values\n", + "\n", + "# 1. Set the hypothesis - dunno if i am defining this well\n", + "# H0: hourly wage > 35\n", + "# H1: hourly wage < 35\n", + "\n", + "#2. significance level\n", + "alpha = 0.05\n", + "\n", + "# sample\n", + "sample_35 = df[(df[\"Department\"] == \"STREETS & SAN\") & (df[\"Salary or Hourly\"] == \"Hourly\")][\"Hourly Rate\"].sample(100)\n", + "\n", + "print(st.ttest_1samp(sample_35, 35, alternative = \"less\"))\n", + "# checking the sign of the stats, is going left so actually is going favorable to H1 - workers claim.\n", + "\n", + "p_value = st.ttest_1samp(sample_35, 35, alternative = \"less\")[1]\n", + "p_value\n", + "stats = st.ttest_1samp(sample_35, 35, alternative = \"less\")[0]\n", + "stats\n", + "\n", + "# Bottom line, if stats value is going left (- sign) it's in favor of H0 so just need to check p > 0,05\n", + "if p_value > 0.05 and stats > 0:\n", + " print(\"I can not reject the null hypothesis\")\n", + "else:\n", + " print(\"We can reject the null hypothesis\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2lB-0sCAc-Kx" + }, + "source": [ + "# Challenge 3: To practice - Constructing Confidence Intervals\n", + "\n", + "While testing our hypothesis is a great way to gather empirical evidence for accepting or rejecting the hypothesis, another way to gather evidence is by creating a confidence interval. A confidence interval gives us information about the true mean of the population. So for a 95% confidence interval, we are 95% sure that the mean of the population is within the confidence interval.\n", + ").\n", + "\n", + "To read more about confidence intervals, click [here](https://en.wikipedia.org/wiki/Confidence_interval).\n", + "\n", + "\n", + "In the cell below, we will construct a 95% confidence interval for the mean hourly wage of all hourly workers.\n", + "\n", + "The confidence interval is computed in SciPy using the `t.interval` function. You can read more about this function [here](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.t.html).\n", + "\n", + "To compute the confidence interval of the hourly wage, use the 0.95 for the confidence level, number of rows - 1 for degrees of freedom, the mean of the sample for the location parameter and the standard error for the scale. The standard error can be computed using [this](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.sem.html) function in SciPy." + ] + }, + { + "cell_type": "code", + "source": [ + "### Are we sampling or use all \"population\" ?\n", + "# like all df[df[\"Salary or Hourly\"] == \"Hourly\"][\"Hourly Rate\"] --- > all hourly rates or sample(30)?" + ], + "metadata": { + "id": "Q-hFIiPnYtwu" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# In order to create our interval at 95% confidence we need\n", + "hourly_wage = df[df[\"Salary or Hourly\"] == \"Hourly\"][\"Hourly Rate\"]\n", + "\n", + "# 1. We need a sample\n", + "s_hourly_wage = hourly_wage.sample(30) # sample???\n", + "\n", + "# 2. the mean of the sample\n", + "mean_s = s_hourly_wage.mean()\n", + "\n", + "# 3. number observations = sample_hourly_wage.count()\n", + "n = 30\n", + "\n", + "#4. std sample\n", + "s = s_hourly_wage.std(ddof = 1)\n", + "\n", + "interval = st.t.interval(0.95, 30, loc = mean_s, scale = s/np.sqrt(30-1)) # ddof = 1 ? is it right here?\n", + "interval\n", + "\n", + "print(\"left end:\", interval[0])\n", + "print(\"right end:\", interval[1])" + ], + "metadata": { + "id": "QH53Z9xeXCF3", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f27b2305-07a3-41f0-9cf4-910ad10fa4aa" + }, + "execution_count": 36, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "left end: 27.865124166757177\n", + "right end: 37.74154249990949\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WYYWeq7fc-Kx" + }, + "source": [ + "Now construct the 95% confidence interval for all salaried employeed in the police in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uaXq7dxRc-Kx" + }, + "outputs": [], + "source": [ + "salaried = df[df[\"Salary or Hourly\"] == \"Salary\"][\"Annual Salary\"]\n", + "\n", + "# 1. We need a sample\n", + "s_salaried = salaried.sample(30) #### sample?\n", + "\n", + "# 2. the mean of the sample\n", + "mean_s = s_salaried.mean()\n", + "\n", + "# 3. number observations = s_salaried.count()\n", + "n = 30\n", + "\n", + "#4. std sample\n", + "s = s_salaried.std(ddof = 1)\n", + "\n", + "interval = st.t.interval(0.95, 30-1, loc = mean_s, scale = s/np.sqrt(30))\n", + "interval\n", + "\n", + "print(\"left end:\", interval[0])\n", + "print(\"right end:\", interval[1])" + ] + }, + { + "cell_type": "code", + "source": [ + "# For TA: by now i am really not sure for the standard error, I talked with teacher about that and used it for proportions, buthere\n", + "# I am getting confused so I just used same example as class :\n", + "\n", + "# houses_close_sample = houses_close.sample(50)\n", + "# mean = houses_close_sample[\"median_house_value\"].mean()\n", + "# n = 50\n", + "# s = houses_close_sample[\"median_house_value\"].std(ddof = 1)\n", + "# st.t.interval(0.955, 50-1, loc = mean, scale = s/np.sqrt(n))" + ], + "metadata": { + "id": "wVKDhtuoY9Z7" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vYBszaBoc-Ky" + }, + "source": [ + "# Bonus Challenge - Hypothesis Tests of Proportions\n", + "\n", + "Another type of one sample test is a hypothesis test of proportions. In this test, we examine whether the proportion of a group in our sample is significantly different than a fraction.\n", + "\n", + "You can read more about one sample proportion tests [here](http://sphweb.bumc.bu.edu/otlt/MPH-Modules/BS/SAS/SAS6-CategoricalData/SAS6-CategoricalData2.html).\n", + "\n", + "In the cell below, use the `proportions_ztest` function from `statsmodels` to perform a hypothesis test that will determine whether the number of hourly workers in the City of Chicago is significantly different from 25% at the 95% confidence level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8Mc0Ktidc-Ky" + }, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + }, + "colab": { + "provenance": [] + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/.gitignore b/your-code/.gitignore similarity index 100% rename from .gitignore rename to your-code/.gitignore diff --git a/README.md b/your-code/README.md old mode 100755 new mode 100644 similarity index 100% rename from README.md rename to your-code/README.md