From 9f3b39d570620cc1b09d5b5df153a1bc8efe32da Mon Sep 17 00:00:00 2001 From: marioc94 Date: Sat, 2 Sep 2023 16:39:32 +0200 Subject: [PATCH] MarioCarmona --- .../.ipynb_checkpoints/main-checkpoint.ipynb | 1012 +++++++++++++ your-code/main.ipynb | 1291 +++++++++++++---- 2 files changed, 2024 insertions(+), 279 deletions(-) create mode 100644 your-code/.ipynb_checkpoints/main-checkpoint.ipynb diff --git a/your-code/.ipynb_checkpoints/main-checkpoint.ipynb b/your-code/.ipynb_checkpoints/main-checkpoint.ipynb new file mode 100644 index 0000000..b607ce1 --- /dev/null +++ b/your-code/.ipynb_checkpoints/main-checkpoint.ipynb @@ -0,0 +1,1012 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Before your start:\n", + "- Read the README.md file\n", + "- Comment as much as you can and use the resources (README.md file)\n", + "- Happy learning!" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# import numpy and pandas\n", + "import pandas as pd\n", + "import numpy as np\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 1 - Exploring the Data\n", + "\n", + "In this challenge, we will examine all salaries of employees of the City of Chicago. We will start by loading the dataset and examining its contents." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "data = pd.read_csv('C:\\\\Users\\\\carmo\\\\Desktop\\\\IronHack\\\\Week 13\\\\Day 2\\\\lab-hypothesis-testing-1\\\\your-code\\\\Current_Employee_Names__Salaries__and_Position_Titles.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Examine the `salaries` dataset using the `head` function below." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameJob TitlesDepartmentFull or Part-TimeSalary or HourlyTypical HoursAnnual SalaryHourly Rate
0AARON, JEFFERY MSERGEANTPOLICEFSalaryNaN101442.0NaN
1AARON, KARINAPOLICE OFFICER (ASSIGNED AS DETECTIVE)POLICEFSalaryNaN94122.0NaN
2AARON, KIMBERLEI RCHIEF CONTRACT EXPEDITERGENERAL SERVICESFSalaryNaN101592.0NaN
3ABAD JR, VICENTE MCIVIL ENGINEER IVWATER MGMNTFSalaryNaN110064.0NaN
4ABASCAL, REECE ETRAFFIC CONTROL AIDE-HOURLYOEMCPHourly20.0NaN19.86
5ABBASI, CHRISTOPHERSTAFF ASST TO THE ALDERMANCITY COUNCILFSalaryNaN50436.0NaN
6ABBATACOLA, ROBERT JELECTRICAL MECHANICAVIATIONFHourly40.0NaN46.10
7ABBATE, JOSEPH LPOOL MOTOR TRUCK DRIVERSTREETS & SANFHourly40.0NaN35.60
8ABBATEMARCO, JAMES JFIRE ENGINEER-EMTFIREFSalaryNaN103350.0NaN
9ABBATE, TERRY MPOLICE OFFICERPOLICEFSalaryNaN93354.0NaN
10ABBOTT, BETTY LFOSTER GRANDPARENTFAMILY & SUPPORTPHourly20.0NaN2.65
11ABDALLAH, ZAIDPOLICE OFFICERPOLICEFSalaryNaN84054.0NaN
12ABDELHADI, ABDALMAHDPOLICE OFFICERPOLICEFSalaryNaN87006.0NaN
13ABDELLATIF, AREF RFIREFIGHTER (PER ARBITRATORS AWARD)-PARAMEDICFIREFSalaryNaN102228.0NaN
14ABDELMAJEID, AZIZPOLICE OFFICERPOLICEFSalaryNaN84054.0NaN
15ABDOLLAHZADEH, ALIFIREFIGHTER/PARAMEDICFIREFSalaryNaN91272.0NaN
16ABDUL-KARIM, MUHAMMAD AENGINEERING TECHNICIAN VIWATER MGMNTFSalaryNaN111492.0NaN
17ABDULLAH, DANIEL NFIREFIGHTER-EMTFIREFSalaryNaN95484.0NaN
18ABDULLAH, LAKENYA NCROSSING GUARDOEMCPHourly20.0NaN17.68
19ABDULLAH, RASHADELECTRICAL MECHANIC (AUTOMOTIVE)GENERAL SERVICESFHourly40.0NaN46.10
20ABDULSATTAR, MUDHARCIVIL ENGINEER IIWATER MGMNTFSalaryNaN65448.0NaN
21ABDUL-SHAKUR, TAHIRGENERAL LABORER - DSSSTREETS & SANFHourly40.0NaN21.43
22ABEJERO, JASON VPOLICE OFFICERPOLICEFSalaryNaN90024.0NaN
23ABERCROMBIE IV, EARL SPARAMEDIC I/CFIREFSalaryNaN82614.0NaN
24ABERCROMBIE, TIMOTHYMOTOR TRUCK DRIVERSTREETS & SANFHourly40.0NaN35.60
25ABFALL, RICHARD CPOLICE OFFICERPOLICEFSalaryNaN48078.0NaN
26ABIOYE, ADEWOLE ALIBRARY ASSOCIATE - HOURLYPUBLIC LIBRARYPHourly20.0NaN25.10
27ABNEY, PATRICKPOLICE OFFICERPOLICEFSalaryNaN76266.0NaN
28ABOUASSI, CHADIPOLICE OFFICERPOLICEFSalaryNaN48078.0NaN
29ABOUELKHEIR, HASSAN ASENIOR PROGRAMMER/ANALYSTFAMILY & SUPPORTFSalaryNaN110064.0NaN
\n", + "
" + ], + "text/plain": [ + " Name Job Titles \\\n", + "0 AARON, JEFFERY M SERGEANT \n", + "1 AARON, KARINA POLICE OFFICER (ASSIGNED AS DETECTIVE) \n", + "2 AARON, KIMBERLEI R CHIEF CONTRACT EXPEDITER \n", + "3 ABAD JR, VICENTE M CIVIL ENGINEER IV \n", + "4 ABASCAL, REECE E TRAFFIC CONTROL AIDE-HOURLY \n", + "5 ABBASI, CHRISTOPHER STAFF ASST TO THE ALDERMAN \n", + "6 ABBATACOLA, ROBERT J ELECTRICAL MECHANIC \n", + "7 ABBATE, JOSEPH L POOL MOTOR TRUCK DRIVER \n", + "8 ABBATEMARCO, JAMES J FIRE ENGINEER-EMT \n", + "9 ABBATE, TERRY M POLICE OFFICER \n", + "10 ABBOTT, BETTY L FOSTER GRANDPARENT \n", + "11 ABDALLAH, ZAID POLICE OFFICER \n", + "12 ABDELHADI, ABDALMAHD POLICE OFFICER \n", + "13 ABDELLATIF, AREF R FIREFIGHTER (PER ARBITRATORS AWARD)-PARAMEDIC \n", + "14 ABDELMAJEID, AZIZ POLICE OFFICER \n", + "15 ABDOLLAHZADEH, ALI FIREFIGHTER/PARAMEDIC \n", + "16 ABDUL-KARIM, MUHAMMAD A ENGINEERING TECHNICIAN VI \n", + "17 ABDULLAH, DANIEL N FIREFIGHTER-EMT \n", + "18 ABDULLAH, LAKENYA N CROSSING GUARD \n", + "19 ABDULLAH, RASHAD ELECTRICAL MECHANIC (AUTOMOTIVE) \n", + "20 ABDULSATTAR, MUDHAR CIVIL ENGINEER II \n", + "21 ABDUL-SHAKUR, TAHIR GENERAL LABORER - DSS \n", + "22 ABEJERO, JASON V POLICE OFFICER \n", + "23 ABERCROMBIE IV, EARL S PARAMEDIC I/C \n", + "24 ABERCROMBIE, TIMOTHY MOTOR TRUCK DRIVER \n", + "25 ABFALL, RICHARD C POLICE OFFICER \n", + "26 ABIOYE, ADEWOLE A LIBRARY ASSOCIATE - HOURLY \n", + "27 ABNEY, PATRICK POLICE OFFICER \n", + "28 ABOUASSI, CHADI POLICE OFFICER \n", + "29 ABOUELKHEIR, HASSAN A SENIOR PROGRAMMER/ANALYST \n", + "\n", + " Department Full or Part-Time Salary or Hourly Typical Hours \\\n", + "0 POLICE F Salary NaN \n", + "1 POLICE F Salary NaN \n", + "2 GENERAL SERVICES F Salary NaN \n", + "3 WATER MGMNT F Salary NaN \n", + "4 OEMC P Hourly 20.0 \n", + "5 CITY COUNCIL F Salary NaN \n", + "6 AVIATION F Hourly 40.0 \n", + "7 STREETS & SAN F Hourly 40.0 \n", + "8 FIRE F Salary NaN \n", + "9 POLICE F Salary NaN \n", + "10 FAMILY & SUPPORT P Hourly 20.0 \n", + "11 POLICE F Salary NaN \n", + "12 POLICE F Salary NaN \n", + "13 FIRE F Salary NaN \n", + "14 POLICE F Salary NaN \n", + "15 FIRE F Salary NaN \n", + "16 WATER MGMNT F Salary NaN \n", + "17 FIRE F Salary NaN \n", + "18 OEMC P Hourly 20.0 \n", + "19 GENERAL SERVICES F Hourly 40.0 \n", + "20 WATER MGMNT F Salary NaN \n", + "21 STREETS & SAN F Hourly 40.0 \n", + "22 POLICE F Salary NaN \n", + "23 FIRE F Salary NaN \n", + "24 STREETS & SAN F Hourly 40.0 \n", + "25 POLICE F Salary NaN \n", + "26 PUBLIC LIBRARY P Hourly 20.0 \n", + "27 POLICE F Salary NaN \n", + "28 POLICE F Salary NaN \n", + "29 FAMILY & SUPPORT F Salary NaN \n", + "\n", + " Annual Salary Hourly Rate \n", + "0 101442.0 NaN \n", + "1 94122.0 NaN \n", + "2 101592.0 NaN \n", + "3 110064.0 NaN \n", + "4 NaN 19.86 \n", + "5 50436.0 NaN \n", + "6 NaN 46.10 \n", + "7 NaN 35.60 \n", + "8 103350.0 NaN \n", + "9 93354.0 NaN \n", + "10 NaN 2.65 \n", + "11 84054.0 NaN \n", + "12 87006.0 NaN \n", + "13 102228.0 NaN \n", + "14 84054.0 NaN \n", + "15 91272.0 NaN \n", + "16 111492.0 NaN \n", + "17 95484.0 NaN \n", + "18 NaN 17.68 \n", + "19 NaN 46.10 \n", + "20 65448.0 NaN \n", + "21 NaN 21.43 \n", + "22 90024.0 NaN \n", + "23 82614.0 NaN \n", + "24 NaN 35.60 \n", + "25 48078.0 NaN \n", + "26 NaN 25.10 \n", + "27 76266.0 NaN \n", + "28 48078.0 NaN \n", + "29 110064.0 NaN " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "data.head(30)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see from looking at the `head` function that there is quite a bit of missing data. Let's examine how much missing data is in each column. Produce this output in the cell below" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Name 0\n", + "Job Titles 0\n", + "Department 0\n", + "Full or Part-Time 0\n", + "Salary or Hourly 0\n", + "Typical Hours 25161\n", + "Annual Salary 8022\n", + "Hourly Rate 25161\n", + "dtype: int64" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "data_null = data.isnull().sum()\n", + "data_null\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's also look at the count of hourly vs. salaried employees. Write the code in the cell below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What this information indicates is that the table contains information about two types of employees - salaried and hourly. Some columns apply only to one type of employee while other columns only apply to another kind. This is why there are so many missing values. Therefore, we will not do anything to handle the missing values." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are different departments in the city. List all departments and the count of employees in each department." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['GENERAL SERVICES',\n", + " 'BUILDINGS',\n", + " 'FIRE',\n", + " 'COPA',\n", + " 'OEMC',\n", + " 'POLICE',\n", + " 'PUBLIC LIBRARY',\n", + " 'BUSINESS AFFAIRS',\n", + " 'TRANSPORTN',\n", + " 'HUMAN RELATIONS',\n", + " 'HEALTH',\n", + " 'BUDGET & MGMT',\n", + " 'BOARD OF ELECTION',\n", + " 'LAW',\n", + " 'CITY CLERK',\n", + " 'FAMILY & SUPPORT',\n", + " 'WATER MGMNT',\n", + " 'TREASURER',\n", + " \"MAYOR'S OFFICE\",\n", + " 'CULTURAL AFFAIRS',\n", + " 'ANIMAL CONTRL',\n", + " 'STREETS & SAN',\n", + " 'HUMAN RESOURCES',\n", + " 'ADMIN HEARNG',\n", + " 'DoIT',\n", + " 'DISABILITIES',\n", + " 'LICENSE APPL COMM',\n", + " 'FINANCE',\n", + " 'AVIATION',\n", + " 'INSPECTOR GEN',\n", + " 'POLICE BOARD',\n", + " 'PROCUREMENT',\n", + " 'COMMUNITY DEVELOPMENT',\n", + " 'CITY COUNCIL',\n", + " 'BOARD OF ETHICS']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "#First I'll build a list with all the unique items for department\n", + "departments = list(set(data['Department']))\n", + "departments" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "POLICE 13414\n", + "FIRE 4641\n", + "STREETS & SAN 2198\n", + "OEMC 2102\n", + "WATER MGMNT 1879\n", + "AVIATION 1629\n", + "TRANSPORTN 1140\n", + "PUBLIC LIBRARY 1015\n", + "GENERAL SERVICES 980\n", + "FAMILY & SUPPORT 615\n", + "FINANCE 560\n", + "HEALTH 488\n", + "CITY COUNCIL 411\n", + "LAW 407\n", + "BUILDINGS 269\n", + "COMMUNITY DEVELOPMENT 207\n", + "BUSINESS AFFAIRS 171\n", + "COPA 116\n", + "BOARD OF ELECTION 107\n", + "DoIT 99\n", + "PROCUREMENT 92\n", + "INSPECTOR GEN 87\n", + "MAYOR'S OFFICE 85\n", + "CITY CLERK 84\n", + "ANIMAL CONTRL 81\n", + "HUMAN RESOURCES 79\n", + "CULTURAL AFFAIRS 65\n", + "BUDGET & MGMT 46\n", + "ADMIN HEARNG 39\n", + "DISABILITIES 28\n", + "TREASURER 22\n", + "HUMAN RELATIONS 16\n", + "BOARD OF ETHICS 8\n", + "POLICE BOARD 2\n", + "LICENSE APPL COMM 1\n", + "Name: Department, dtype: int64" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Group by the 'Department' column and count the number of employees in each department\n", + "department_counts = data['Department'].value_counts()\n", + "\n", + "department_counts" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 2 - Hypothesis Tests\n", + "\n", + "In this section of the lab, we will test whether the hourly wage of all hourly workers is significantly different from $30/hr. Import the correct one sample test function from scipy and perform the hypothesis test for a 95% two sided confidence interval." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(20.6198057854942, 4.3230240486229894e-92)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "from scipy.stats import ttest_1samp\n", + "\n", + "# Filter out hourly workers and their hourly rates\n", + "hourly_rates = data[data['Salary or Hourly'] == 'Hourly']['Hourly Rate'].dropna()\n", + "\n", + "# Conduct the one-sample t-test\n", + "t_stat, p_value = ttest_1samp(hourly_rates, 30)\n", + "\n", + "t_stat, p_value\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are also curious about salaries in the police force. The chief of police in Chicago claimed in a press briefing that salaries this year are higher than last year's mean of $86000/year a year for all salaried employees. Test this one sided hypothesis using a 95% confidence interval.\n", + "\n", + "Hint: A one tailed test has a p-value that is half of the two tailed p-value. If our hypothesis is greater than, then to reject, the test statistic must also be positive." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(3.081997005712994, 0.0010301701775482569)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "# Filter out salaried police employees and their annual salaries\n", + "police_salaries = data[(data['Department'] == 'POLICE') & (data['Salary or Hourly'] == 'Salary')]['Annual Salary'].dropna()\n", + "\n", + "# Conduct the one-sample t-test\n", + "t_stat_police, p_value_police = ttest_1samp(police_salaries, 86000)\n", + "\n", + "# Since it's a one-tailed test, we'll halve the p-value\n", + "p_value_one_tailed = p_value_police / 2\n", + "\n", + "t_stat_police, p_value_one_tailed\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the `crosstab` function, find the department that has the most hourly workers. " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Department\n", + "STREETS & SAN 1862\n", + "Name: Hourly, dtype: int64" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "# Create a crosstab of the 'Department' and 'Salary or Hourly' columns to count the number of hourly and salaried workers in each department\n", + "department_hourly_crosstab = pd.crosstab(data['Department'], data['Salary or Hourly'])\n", + "\n", + "# Sort the departments based on the number of hourly workers in descending order\n", + "sorted_departments = department_hourly_crosstab['Hourly'].sort_values(ascending=False)\n", + "\n", + "# Get the department with the most hourly workers\n", + "top_department = sorted_departments.head(1)\n", + "\n", + "top_department\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The workers from the department with the most hourly workers have complained that their hourly wage is less than $35/hour. Using a one sample t-test, test this one-sided hypothesis at the 95% confidence level." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(-9.567447887848152, 1.6689265282353859e-21)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "streets_salaries = data[(data['Department'] == 'STREETS & SAN') & (data['Salary or Hourly']== 'Hourly')]['Hourly Rate']\n", + "\n", + "\n", + "# Conduct the one-sample t-test\n", + "t_stat_streets, p_value_streets = ttest_1samp(streets_salaries, 35)\n", + "\n", + "# Since it's a one-tailed test, we'll halve the p-value\n", + "p_value_one_tailed = p_value_streets / 2\n", + "\n", + "t_stat_streets, p_value_one_tailed\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 3: To practice - Constructing Confidence Intervals\n", + "\n", + "While testing our hypothesis is a great way to gather empirical evidence for accepting or rejecting the hypothesis, another way to gather evidence is by creating a confidence interval. A confidence interval gives us information about the true mean of the population. So for a 95% confidence interval, we are 95% sure that the mean of the population is within the confidence interval. \n", + ").\n", + "\n", + "To read more about confidence intervals, click [here](https://en.wikipedia.org/wiki/Confidence_interval).\n", + "\n", + "\n", + "In the cell below, we will construct a 95% confidence interval for the mean hourly wage of all hourly workers. \n", + "\n", + "The confidence interval is computed in SciPy using the `t.interval` function. You can read more about this function [here](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.t.html).\n", + "\n", + "To compute the confidence interval of the hourly wage, use the 0.95 for the confidence level, number of rows - 1 for degrees of freedom, the mean of the sample for the location parameter and the standard error for the scale. The standard error can be computed using [this](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.sem.html) function in SciPy." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(32.52345834488425, 33.05365708767623)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "\n", + "from scipy.stats import t\n", + "\n", + "# Parameters for the confidence interval\n", + "confidence_level = 0.95\n", + "degrees_freedom = len(hourly_rates) - 1\n", + "sample_mean = hourly_rates.mean()\n", + "sample_standard_error = hourly_rates.std() / (len(hourly_rates) ** 0.5)\n", + "\n", + "# Compute the 95% confidence interval for the mean hourly wage of all hourly workers\n", + "confidence_interval = t.interval(confidence_level, degrees_freedom, sample_mean, sample_standard_error)\n", + "\n", + "confidence_interval" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 95% confidence interval for the mean hourly wage of all hourly workers is between approximately $32.52 and $33.05. This means we are 95% confident that the true mean hourly wage for the entire population of hourly workers lies within this interval." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now construct the 95% confidence interval for all salaried employeed in the police in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(86177.05631531785, 86795.77269094893)" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "# Parameters for the confidence interval for salaried police employees\n", + "degrees_freedom_police = len(police_salaries) - 1\n", + "sample_mean_police = police_salaries.mean()\n", + "sample_standard_error_police = police_salaries.std() / (len(police_salaries) ** 0.5)\n", + "\n", + "# Compute the 95% confidence interval for the mean salary of all salaried police employees\n", + "confidence_interval_police = t.interval(confidence_level, degrees_freedom_police, sample_mean_police, sample_standard_error_police)\n", + "\n", + "confidence_interval_police\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 95% confidence interval for the mean annual salary of all salaried police employees is between approximately $86,177.06 and $86,795.77. This means we are 95% confident that the true mean annual salary for the entire population of salaried police employees lies within this interval." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bonus Challenge - Hypothesis Tests of Proportions\n", + "\n", + "Another type of one sample test is a hypothesis test of proportions. In this test, we examine whether the proportion of a group in our sample is significantly different than a fraction. \n", + "\n", + "You can read more about one sample proportion tests [here](http://sphweb.bumc.bu.edu/otlt/MPH-Modules/BS/SAS/SAS6-CategoricalData/SAS6-CategoricalData2.html).\n", + "\n", + "In the cell below, use the `proportions_ztest` function from `statsmodels` to perform a hypothesis test that will determine whether the number of hourly workers in the City of Chicago is significantly different from 25% at the 95% confidence level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/main.ipynb b/your-code/main.ipynb index b2b6f8d..b607ce1 100644 --- a/your-code/main.ipynb +++ b/your-code/main.ipynb @@ -1,279 +1,1012 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Before your start:\n", - "- Read the README.md file\n", - "- Comment as much as you can and use the resources (README.md file)\n", - "- Happy learning!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import numpy and pandas\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Challenge 1 - Exploring the Data\n", - "\n", - "In this challenge, we will examine all salaries of employees of the City of Chicago. We will start by loading the dataset and examining its contents." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Examine the `salaries` dataset using the `head` function below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We see from looking at the `head` function that there is quite a bit of missing data. Let's examine how much missing data is in each column. Produce this output in the cell below" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's also look at the count of hourly vs. salaried employees. Write the code in the cell below" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "What this information indicates is that the table contains information about two types of employees - salaried and hourly. Some columns apply only to one type of employee while other columns only apply to another kind. This is why there are so many missing values. Therefore, we will not do anything to handle the missing values." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are different departments in the city. List all departments and the count of employees in each department." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Challenge 2 - Hypothesis Tests\n", - "\n", - "In this section of the lab, we will test whether the hourly wage of all hourly workers is significantly different from $30/hr. Import the correct one sample test function from scipy and perform the hypothesis test for a 95% two sided confidence interval." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We are also curious about salaries in the police force. The chief of police in Chicago claimed in a press briefing that salaries this year are higher than last year's mean of $86000/year a year for all salaried employees. Test this one sided hypothesis using a 95% confidence interval.\n", - "\n", - "Hint: A one tailed test has a p-value that is half of the two tailed p-value. If our hypothesis is greater than, then to reject, the test statistic must also be positive." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using the `crosstab` function, find the department that has the most hourly workers. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The workers from the department with the most hourly workers have complained that their hourly wage is less than $35/hour. Using a one sample t-test, test this one-sided hypothesis at the 95% confidence level." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Challenge 3: To practice - Constructing Confidence Intervals\n", - "\n", - "While testing our hypothesis is a great way to gather empirical evidence for accepting or rejecting the hypothesis, another way to gather evidence is by creating a confidence interval. A confidence interval gives us information about the true mean of the population. So for a 95% confidence interval, we are 95% sure that the mean of the population is within the confidence interval. \n", - ").\n", - "\n", - "To read more about confidence intervals, click [here](https://en.wikipedia.org/wiki/Confidence_interval).\n", - "\n", - "\n", - "In the cell below, we will construct a 95% confidence interval for the mean hourly wage of all hourly workers. \n", - "\n", - "The confidence interval is computed in SciPy using the `t.interval` function. You can read more about this function [here](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.t.html).\n", - "\n", - "To compute the confidence interval of the hourly wage, use the 0.95 for the confidence level, number of rows - 1 for degrees of freedom, the mean of the sample for the location parameter and the standard error for the scale. The standard error can be computed using [this](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.sem.html) function in SciPy." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now construct the 95% confidence interval for all salaried employeed in the police in the cell below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Bonus Challenge - Hypothesis Tests of Proportions\n", - "\n", - "Another type of one sample test is a hypothesis test of proportions. In this test, we examine whether the proportion of a group in our sample is significantly different than a fraction. \n", - "\n", - "You can read more about one sample proportion tests [here](http://sphweb.bumc.bu.edu/otlt/MPH-Modules/BS/SAS/SAS6-CategoricalData/SAS6-CategoricalData2.html).\n", - "\n", - "In the cell below, use the `proportions_ztest` function from `statsmodels` to perform a hypothesis test that will determine whether the number of hourly workers in the City of Chicago is significantly different from 25% at the 95% confidence level." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Your code here:\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Before your start:\n", + "- Read the README.md file\n", + "- Comment as much as you can and use the resources (README.md file)\n", + "- Happy learning!" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# import numpy and pandas\n", + "import pandas as pd\n", + "import numpy as np\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 1 - Exploring the Data\n", + "\n", + "In this challenge, we will examine all salaries of employees of the City of Chicago. We will start by loading the dataset and examining its contents." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "data = pd.read_csv('C:\\\\Users\\\\carmo\\\\Desktop\\\\IronHack\\\\Week 13\\\\Day 2\\\\lab-hypothesis-testing-1\\\\your-code\\\\Current_Employee_Names__Salaries__and_Position_Titles.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Examine the `salaries` dataset using the `head` function below." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameJob TitlesDepartmentFull or Part-TimeSalary or HourlyTypical HoursAnnual SalaryHourly Rate
0AARON, JEFFERY MSERGEANTPOLICEFSalaryNaN101442.0NaN
1AARON, KARINAPOLICE OFFICER (ASSIGNED AS DETECTIVE)POLICEFSalaryNaN94122.0NaN
2AARON, KIMBERLEI RCHIEF CONTRACT EXPEDITERGENERAL SERVICESFSalaryNaN101592.0NaN
3ABAD JR, VICENTE MCIVIL ENGINEER IVWATER MGMNTFSalaryNaN110064.0NaN
4ABASCAL, REECE ETRAFFIC CONTROL AIDE-HOURLYOEMCPHourly20.0NaN19.86
5ABBASI, CHRISTOPHERSTAFF ASST TO THE ALDERMANCITY COUNCILFSalaryNaN50436.0NaN
6ABBATACOLA, ROBERT JELECTRICAL MECHANICAVIATIONFHourly40.0NaN46.10
7ABBATE, JOSEPH LPOOL MOTOR TRUCK DRIVERSTREETS & SANFHourly40.0NaN35.60
8ABBATEMARCO, JAMES JFIRE ENGINEER-EMTFIREFSalaryNaN103350.0NaN
9ABBATE, TERRY MPOLICE OFFICERPOLICEFSalaryNaN93354.0NaN
10ABBOTT, BETTY LFOSTER GRANDPARENTFAMILY & SUPPORTPHourly20.0NaN2.65
11ABDALLAH, ZAIDPOLICE OFFICERPOLICEFSalaryNaN84054.0NaN
12ABDELHADI, ABDALMAHDPOLICE OFFICERPOLICEFSalaryNaN87006.0NaN
13ABDELLATIF, AREF RFIREFIGHTER (PER ARBITRATORS AWARD)-PARAMEDICFIREFSalaryNaN102228.0NaN
14ABDELMAJEID, AZIZPOLICE OFFICERPOLICEFSalaryNaN84054.0NaN
15ABDOLLAHZADEH, ALIFIREFIGHTER/PARAMEDICFIREFSalaryNaN91272.0NaN
16ABDUL-KARIM, MUHAMMAD AENGINEERING TECHNICIAN VIWATER MGMNTFSalaryNaN111492.0NaN
17ABDULLAH, DANIEL NFIREFIGHTER-EMTFIREFSalaryNaN95484.0NaN
18ABDULLAH, LAKENYA NCROSSING GUARDOEMCPHourly20.0NaN17.68
19ABDULLAH, RASHADELECTRICAL MECHANIC (AUTOMOTIVE)GENERAL SERVICESFHourly40.0NaN46.10
20ABDULSATTAR, MUDHARCIVIL ENGINEER IIWATER MGMNTFSalaryNaN65448.0NaN
21ABDUL-SHAKUR, TAHIRGENERAL LABORER - DSSSTREETS & SANFHourly40.0NaN21.43
22ABEJERO, JASON VPOLICE OFFICERPOLICEFSalaryNaN90024.0NaN
23ABERCROMBIE IV, EARL SPARAMEDIC I/CFIREFSalaryNaN82614.0NaN
24ABERCROMBIE, TIMOTHYMOTOR TRUCK DRIVERSTREETS & SANFHourly40.0NaN35.60
25ABFALL, RICHARD CPOLICE OFFICERPOLICEFSalaryNaN48078.0NaN
26ABIOYE, ADEWOLE ALIBRARY ASSOCIATE - HOURLYPUBLIC LIBRARYPHourly20.0NaN25.10
27ABNEY, PATRICKPOLICE OFFICERPOLICEFSalaryNaN76266.0NaN
28ABOUASSI, CHADIPOLICE OFFICERPOLICEFSalaryNaN48078.0NaN
29ABOUELKHEIR, HASSAN ASENIOR PROGRAMMER/ANALYSTFAMILY & SUPPORTFSalaryNaN110064.0NaN
\n", + "
" + ], + "text/plain": [ + " Name Job Titles \\\n", + "0 AARON, JEFFERY M SERGEANT \n", + "1 AARON, KARINA POLICE OFFICER (ASSIGNED AS DETECTIVE) \n", + "2 AARON, KIMBERLEI R CHIEF CONTRACT EXPEDITER \n", + "3 ABAD JR, VICENTE M CIVIL ENGINEER IV \n", + "4 ABASCAL, REECE E TRAFFIC CONTROL AIDE-HOURLY \n", + "5 ABBASI, CHRISTOPHER STAFF ASST TO THE ALDERMAN \n", + "6 ABBATACOLA, ROBERT J ELECTRICAL MECHANIC \n", + "7 ABBATE, JOSEPH L POOL MOTOR TRUCK DRIVER \n", + "8 ABBATEMARCO, JAMES J FIRE ENGINEER-EMT \n", + "9 ABBATE, TERRY M POLICE OFFICER \n", + "10 ABBOTT, BETTY L FOSTER GRANDPARENT \n", + "11 ABDALLAH, ZAID POLICE OFFICER \n", + "12 ABDELHADI, ABDALMAHD POLICE OFFICER \n", + "13 ABDELLATIF, AREF R FIREFIGHTER (PER ARBITRATORS AWARD)-PARAMEDIC \n", + "14 ABDELMAJEID, AZIZ POLICE OFFICER \n", + "15 ABDOLLAHZADEH, ALI FIREFIGHTER/PARAMEDIC \n", + "16 ABDUL-KARIM, MUHAMMAD A ENGINEERING TECHNICIAN VI \n", + "17 ABDULLAH, DANIEL N FIREFIGHTER-EMT \n", + "18 ABDULLAH, LAKENYA N CROSSING GUARD \n", + "19 ABDULLAH, RASHAD ELECTRICAL MECHANIC (AUTOMOTIVE) \n", + "20 ABDULSATTAR, MUDHAR CIVIL ENGINEER II \n", + "21 ABDUL-SHAKUR, TAHIR GENERAL LABORER - DSS \n", + "22 ABEJERO, JASON V POLICE OFFICER \n", + "23 ABERCROMBIE IV, EARL S PARAMEDIC I/C \n", + "24 ABERCROMBIE, TIMOTHY MOTOR TRUCK DRIVER \n", + "25 ABFALL, RICHARD C POLICE OFFICER \n", + "26 ABIOYE, ADEWOLE A LIBRARY ASSOCIATE - HOURLY \n", + "27 ABNEY, PATRICK POLICE OFFICER \n", + "28 ABOUASSI, CHADI POLICE OFFICER \n", + "29 ABOUELKHEIR, HASSAN A SENIOR PROGRAMMER/ANALYST \n", + "\n", + " Department Full or Part-Time Salary or Hourly Typical Hours \\\n", + "0 POLICE F Salary NaN \n", + "1 POLICE F Salary NaN \n", + "2 GENERAL SERVICES F Salary NaN \n", + "3 WATER MGMNT F Salary NaN \n", + "4 OEMC P Hourly 20.0 \n", + "5 CITY COUNCIL F Salary NaN \n", + "6 AVIATION F Hourly 40.0 \n", + "7 STREETS & SAN F Hourly 40.0 \n", + "8 FIRE F Salary NaN \n", + "9 POLICE F Salary NaN \n", + "10 FAMILY & SUPPORT P Hourly 20.0 \n", + "11 POLICE F Salary NaN \n", + "12 POLICE F Salary NaN \n", + "13 FIRE F Salary NaN \n", + "14 POLICE F Salary NaN \n", + "15 FIRE F Salary NaN \n", + "16 WATER MGMNT F Salary NaN \n", + "17 FIRE F Salary NaN \n", + "18 OEMC P Hourly 20.0 \n", + "19 GENERAL SERVICES F Hourly 40.0 \n", + "20 WATER MGMNT F Salary NaN \n", + "21 STREETS & SAN F Hourly 40.0 \n", + "22 POLICE F Salary NaN \n", + "23 FIRE F Salary NaN \n", + "24 STREETS & SAN F Hourly 40.0 \n", + "25 POLICE F Salary NaN \n", + "26 PUBLIC LIBRARY P Hourly 20.0 \n", + "27 POLICE F Salary NaN \n", + "28 POLICE F Salary NaN \n", + "29 FAMILY & SUPPORT F Salary NaN \n", + "\n", + " Annual Salary Hourly Rate \n", + "0 101442.0 NaN \n", + "1 94122.0 NaN \n", + "2 101592.0 NaN \n", + "3 110064.0 NaN \n", + "4 NaN 19.86 \n", + "5 50436.0 NaN \n", + "6 NaN 46.10 \n", + "7 NaN 35.60 \n", + "8 103350.0 NaN \n", + "9 93354.0 NaN \n", + "10 NaN 2.65 \n", + "11 84054.0 NaN \n", + "12 87006.0 NaN \n", + "13 102228.0 NaN \n", + "14 84054.0 NaN \n", + "15 91272.0 NaN \n", + "16 111492.0 NaN \n", + "17 95484.0 NaN \n", + "18 NaN 17.68 \n", + "19 NaN 46.10 \n", + "20 65448.0 NaN \n", + "21 NaN 21.43 \n", + "22 90024.0 NaN \n", + "23 82614.0 NaN \n", + "24 NaN 35.60 \n", + "25 48078.0 NaN \n", + "26 NaN 25.10 \n", + "27 76266.0 NaN \n", + "28 48078.0 NaN \n", + "29 110064.0 NaN " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "data.head(30)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see from looking at the `head` function that there is quite a bit of missing data. Let's examine how much missing data is in each column. Produce this output in the cell below" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Name 0\n", + "Job Titles 0\n", + "Department 0\n", + "Full or Part-Time 0\n", + "Salary or Hourly 0\n", + "Typical Hours 25161\n", + "Annual Salary 8022\n", + "Hourly Rate 25161\n", + "dtype: int64" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "data_null = data.isnull().sum()\n", + "data_null\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's also look at the count of hourly vs. salaried employees. Write the code in the cell below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What this information indicates is that the table contains information about two types of employees - salaried and hourly. Some columns apply only to one type of employee while other columns only apply to another kind. This is why there are so many missing values. Therefore, we will not do anything to handle the missing values." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are different departments in the city. List all departments and the count of employees in each department." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['GENERAL SERVICES',\n", + " 'BUILDINGS',\n", + " 'FIRE',\n", + " 'COPA',\n", + " 'OEMC',\n", + " 'POLICE',\n", + " 'PUBLIC LIBRARY',\n", + " 'BUSINESS AFFAIRS',\n", + " 'TRANSPORTN',\n", + " 'HUMAN RELATIONS',\n", + " 'HEALTH',\n", + " 'BUDGET & MGMT',\n", + " 'BOARD OF ELECTION',\n", + " 'LAW',\n", + " 'CITY CLERK',\n", + " 'FAMILY & SUPPORT',\n", + " 'WATER MGMNT',\n", + " 'TREASURER',\n", + " \"MAYOR'S OFFICE\",\n", + " 'CULTURAL AFFAIRS',\n", + " 'ANIMAL CONTRL',\n", + " 'STREETS & SAN',\n", + " 'HUMAN RESOURCES',\n", + " 'ADMIN HEARNG',\n", + " 'DoIT',\n", + " 'DISABILITIES',\n", + " 'LICENSE APPL COMM',\n", + " 'FINANCE',\n", + " 'AVIATION',\n", + " 'INSPECTOR GEN',\n", + " 'POLICE BOARD',\n", + " 'PROCUREMENT',\n", + " 'COMMUNITY DEVELOPMENT',\n", + " 'CITY COUNCIL',\n", + " 'BOARD OF ETHICS']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "#First I'll build a list with all the unique items for department\n", + "departments = list(set(data['Department']))\n", + "departments" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "POLICE 13414\n", + "FIRE 4641\n", + "STREETS & SAN 2198\n", + "OEMC 2102\n", + "WATER MGMNT 1879\n", + "AVIATION 1629\n", + "TRANSPORTN 1140\n", + "PUBLIC LIBRARY 1015\n", + "GENERAL SERVICES 980\n", + "FAMILY & SUPPORT 615\n", + "FINANCE 560\n", + "HEALTH 488\n", + "CITY COUNCIL 411\n", + "LAW 407\n", + "BUILDINGS 269\n", + "COMMUNITY DEVELOPMENT 207\n", + "BUSINESS AFFAIRS 171\n", + "COPA 116\n", + "BOARD OF ELECTION 107\n", + "DoIT 99\n", + "PROCUREMENT 92\n", + "INSPECTOR GEN 87\n", + "MAYOR'S OFFICE 85\n", + "CITY CLERK 84\n", + "ANIMAL CONTRL 81\n", + "HUMAN RESOURCES 79\n", + "CULTURAL AFFAIRS 65\n", + "BUDGET & MGMT 46\n", + "ADMIN HEARNG 39\n", + "DISABILITIES 28\n", + "TREASURER 22\n", + "HUMAN RELATIONS 16\n", + "BOARD OF ETHICS 8\n", + "POLICE BOARD 2\n", + "LICENSE APPL COMM 1\n", + "Name: Department, dtype: int64" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Group by the 'Department' column and count the number of employees in each department\n", + "department_counts = data['Department'].value_counts()\n", + "\n", + "department_counts" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 2 - Hypothesis Tests\n", + "\n", + "In this section of the lab, we will test whether the hourly wage of all hourly workers is significantly different from $30/hr. Import the correct one sample test function from scipy and perform the hypothesis test for a 95% two sided confidence interval." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(20.6198057854942, 4.3230240486229894e-92)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "from scipy.stats import ttest_1samp\n", + "\n", + "# Filter out hourly workers and their hourly rates\n", + "hourly_rates = data[data['Salary or Hourly'] == 'Hourly']['Hourly Rate'].dropna()\n", + "\n", + "# Conduct the one-sample t-test\n", + "t_stat, p_value = ttest_1samp(hourly_rates, 30)\n", + "\n", + "t_stat, p_value\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are also curious about salaries in the police force. The chief of police in Chicago claimed in a press briefing that salaries this year are higher than last year's mean of $86000/year a year for all salaried employees. Test this one sided hypothesis using a 95% confidence interval.\n", + "\n", + "Hint: A one tailed test has a p-value that is half of the two tailed p-value. If our hypothesis is greater than, then to reject, the test statistic must also be positive." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(3.081997005712994, 0.0010301701775482569)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "# Filter out salaried police employees and their annual salaries\n", + "police_salaries = data[(data['Department'] == 'POLICE') & (data['Salary or Hourly'] == 'Salary')]['Annual Salary'].dropna()\n", + "\n", + "# Conduct the one-sample t-test\n", + "t_stat_police, p_value_police = ttest_1samp(police_salaries, 86000)\n", + "\n", + "# Since it's a one-tailed test, we'll halve the p-value\n", + "p_value_one_tailed = p_value_police / 2\n", + "\n", + "t_stat_police, p_value_one_tailed\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the `crosstab` function, find the department that has the most hourly workers. " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Department\n", + "STREETS & SAN 1862\n", + "Name: Hourly, dtype: int64" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "# Create a crosstab of the 'Department' and 'Salary or Hourly' columns to count the number of hourly and salaried workers in each department\n", + "department_hourly_crosstab = pd.crosstab(data['Department'], data['Salary or Hourly'])\n", + "\n", + "# Sort the departments based on the number of hourly workers in descending order\n", + "sorted_departments = department_hourly_crosstab['Hourly'].sort_values(ascending=False)\n", + "\n", + "# Get the department with the most hourly workers\n", + "top_department = sorted_departments.head(1)\n", + "\n", + "top_department\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The workers from the department with the most hourly workers have complained that their hourly wage is less than $35/hour. Using a one sample t-test, test this one-sided hypothesis at the 95% confidence level." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(-9.567447887848152, 1.6689265282353859e-21)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "streets_salaries = data[(data['Department'] == 'STREETS & SAN') & (data['Salary or Hourly']== 'Hourly')]['Hourly Rate']\n", + "\n", + "\n", + "# Conduct the one-sample t-test\n", + "t_stat_streets, p_value_streets = ttest_1samp(streets_salaries, 35)\n", + "\n", + "# Since it's a one-tailed test, we'll halve the p-value\n", + "p_value_one_tailed = p_value_streets / 2\n", + "\n", + "t_stat_streets, p_value_one_tailed\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 3: To practice - Constructing Confidence Intervals\n", + "\n", + "While testing our hypothesis is a great way to gather empirical evidence for accepting or rejecting the hypothesis, another way to gather evidence is by creating a confidence interval. A confidence interval gives us information about the true mean of the population. So for a 95% confidence interval, we are 95% sure that the mean of the population is within the confidence interval. \n", + ").\n", + "\n", + "To read more about confidence intervals, click [here](https://en.wikipedia.org/wiki/Confidence_interval).\n", + "\n", + "\n", + "In the cell below, we will construct a 95% confidence interval for the mean hourly wage of all hourly workers. \n", + "\n", + "The confidence interval is computed in SciPy using the `t.interval` function. You can read more about this function [here](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.t.html).\n", + "\n", + "To compute the confidence interval of the hourly wage, use the 0.95 for the confidence level, number of rows - 1 for degrees of freedom, the mean of the sample for the location parameter and the standard error for the scale. The standard error can be computed using [this](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.sem.html) function in SciPy." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(32.52345834488425, 33.05365708767623)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "\n", + "from scipy.stats import t\n", + "\n", + "# Parameters for the confidence interval\n", + "confidence_level = 0.95\n", + "degrees_freedom = len(hourly_rates) - 1\n", + "sample_mean = hourly_rates.mean()\n", + "sample_standard_error = hourly_rates.std() / (len(hourly_rates) ** 0.5)\n", + "\n", + "# Compute the 95% confidence interval for the mean hourly wage of all hourly workers\n", + "confidence_interval = t.interval(confidence_level, degrees_freedom, sample_mean, sample_standard_error)\n", + "\n", + "confidence_interval" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 95% confidence interval for the mean hourly wage of all hourly workers is between approximately $32.52 and $33.05. This means we are 95% confident that the true mean hourly wage for the entire population of hourly workers lies within this interval." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now construct the 95% confidence interval for all salaried employeed in the police in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(86177.05631531785, 86795.77269094893)" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Your code here:\n", + "# Parameters for the confidence interval for salaried police employees\n", + "degrees_freedom_police = len(police_salaries) - 1\n", + "sample_mean_police = police_salaries.mean()\n", + "sample_standard_error_police = police_salaries.std() / (len(police_salaries) ** 0.5)\n", + "\n", + "# Compute the 95% confidence interval for the mean salary of all salaried police employees\n", + "confidence_interval_police = t.interval(confidence_level, degrees_freedom_police, sample_mean_police, sample_standard_error_police)\n", + "\n", + "confidence_interval_police\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 95% confidence interval for the mean annual salary of all salaried police employees is between approximately $86,177.06 and $86,795.77. This means we are 95% confident that the true mean annual salary for the entire population of salaried police employees lies within this interval." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Bonus Challenge - Hypothesis Tests of Proportions\n", + "\n", + "Another type of one sample test is a hypothesis test of proportions. In this test, we examine whether the proportion of a group in our sample is significantly different than a fraction. \n", + "\n", + "You can read more about one sample proportion tests [here](http://sphweb.bumc.bu.edu/otlt/MPH-Modules/BS/SAS/SAS6-CategoricalData/SAS6-CategoricalData2.html).\n", + "\n", + "In the cell below, use the `proportions_ztest` function from `statsmodels` to perform a hypothesis test that will determine whether the number of hourly workers in the City of Chicago is significantly different from 25% at the 95% confidence level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Your code here:\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}