machine_learnng.py

# -*- coding: utf-8 -*-
"""Machine_Learnng.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1rOYZcvLNrB4JOJcOKgGbi9C6ILBJ388T
"""

from google.colab import drive
drive.mount('/content/drive')

# Commented out IPython magic to ensure Python compatibility.
# %cd /content/drive/MyDrive/Machine_Learnin/

"""#**Import libraries**

"""

# Commented out IPython magic to ensure Python compatibility.
import numpy as np     
import pandas as pd     

# To plot
import matplotlib.pyplot as plt  
# %matplotlib inline        
import seaborn as sns


from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, balanced_accuracy_score
from xgboost import XGBClassifier
import xgboost as xgb
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import OneHotEncoder

"""## **Read** **files**"""

df_train_set_value = pd.read_csv('TrainingSetValues.csv')
df_train_labels_value = pd.read_csv('TrainingSetLabels.csv')
df_test_set_value =pd.read_csv('TestSetValues.csv')

df_train_set = df_train_set_value.copy()
df_train_labels =df_train_labels_value.copy()
df_test_set =df_test_set_value.copy()

df_train_set.shape

df = pd.merge(df_train_labels, df_train_set, how = 'inner')

"""## **Data cleaning**"""

df.head()

df.info()

df.isna().sum()

df.describe()

df['status_group'].value_counts()

df['payment'].value_counts()

df['payment_type'].value_counts()

df['water_quality'].value_counts()

df['quality_group'].value_counts()

df['source'].value_counts()

df['source_type'].value_counts()

df['source_class'].value_counts()

df['quantity_group'].value_counts()

df['quantity'].value_counts()

df['scheme_management'].value_counts()

df['management'].value_counts()

df['extraction_type'].value_counts()

df['extraction_type_group'].value_counts()

df['extraction_type_class'].value_counts()

df['waterpoint_type'].value_counts()

df['waterpoint_type_group'].value_counts()

"""Drop similler columns"""

df.drop(columns=['management_group','scheme_management','quantity_group','source_class','source_type','quality_group',
                'payment_type','extraction_type_class','extraction_type', 'waterpoint_type_group'],inplace=True )



df['construction_year'].value_counts()

df['construction_year'].replace(to_replace = 0, value = 1986, inplace=True)

df['decade'] = df['construction_year']

def decade_encode(df):
  df['decade'].replace(to_replace = (1960,1961,1962,1963,1964,1965,1966,1967,1968,1969),
                          value ='60s' , inplace=True)
  df['decade'].replace(to_replace = (1970,1971,1972,1973,1974,1975,1976,1977,1978,1979),
                          value ='70s' , inplace=True)
  df['decade'].replace(to_replace = (1980,1981,1982,1983,1984,1985,1986,1987,1988,1989),
                          value ='80s' , inplace=True)
  df['decade'].replace(to_replace = (1990,1991,1992,1993,1994,1995,1996,1997,1998,1999),
                          value ='90s' , inplace=True)
  df['decade'].replace(to_replace = (2000,2001,2002,2003,2004,2005,2006,2007,2008,2009),
                          value ='00s' , inplace=True)
  df['decade'].replace(to_replace = (2010,2011,2012,2013),
                          value ='10s' , inplace=True)
  
  return df

df=decade_encode(df)

df['recorded_by'].value_counts()

df.drop(columns=['recorded_by'],inplace=True )

"""Fill missing values as "unknown""""

df['installer'].fillna(value='Unknown',inplace=True)

df['installer'].value_counts().head(20)



df['installer'].replace(to_replace = '0', value ='Unknown' , inplace=True)



def installer_encode(df):
  df['installer'].replace(to_replace = ('District Water Department', 'District water depar','Distric Water Department'),
                          value ='District water department' , inplace=True)

  df['installer'].replace(to_replace = ('FinW','Fini water','FINI WATER'), value ='Fini Water' , inplace=True)
  df['installer'].replace(to_replace = 'JAICA', value ='Jaica' , inplace=True)

  df['installer'].replace(to_replace = ('COUN', 'District COUNCIL', 'DISTRICT COUNCIL','District Counci', 
                                        'District Council','Council','Counc','District  Council','Distri'),
                                      value ='District council' , inplace=True)

  df['installer'].replace(to_replace = ('RC CHURCH', 'RC Churc', 'RC','RC Ch','RC C', 'RC CH','RC church', 
                                        'RC CATHORIC',) , value ='RC Church' , inplace=True)

  df['installer'].replace(to_replace = ('Central Government','Tanzania Government',
                                        'central government','Cental Government', 'Cebtral Government', 
                                        'Tanzanian Government','Tanzania government', 'Centra Government' ,
                                        'CENTRAL GOVERNMENT', 'TANZANIAN GOVERNMENT','Central govt', 'Centr', 
                                        'Centra govt') , value ='Central government' , inplace=True)

  df['installer'].replace(to_replace = ('World vision', 'World Division','World Vision'),
                                          value ='world vision' , inplace=True)

  df['installer'].replace(to_replace = ('Unisef','UNICEF'),value ='Unicef' , inplace=True)
  df['installer'].replace(to_replace = 'DANID', value ='DANIDA' , inplace=True)

  df['installer'].replace(to_replace = ('villigers', 'villager', 'Villagers', 'Villa', 'Village', 'Villi', 
                                        'Village Council','Village Counil', 'Villages', 'Vill', 'Village community', 
                                        'Villaers', 'Village Community', 'Villag','Villege Council', 'Village council',
                                        'Village  Council','Villagerd', 'Villager', 'Village Technician',
                                        'Village Office','Village community members'),
                                          value ='villagers' , inplace=True)

  df['installer'].replace(to_replace =('Commu','Communit','commu','COMMU', 'COMMUNITY') ,
                                          value ='Community' , inplace=True)

  df['installer'].replace(to_replace = ('GOVERNMENT', 'GOVER', 'GOVERNME', 'GOVERM','GOVERN','Gover','Gove',
                                        'Governme','Governmen' ) ,value ='Government' , inplace=True)

  df['installer'].replace(to_replace = 'Hesawa' ,value ='HESAWA' , inplace=True)
  df['installer'].replace(to_replace = ('Colonial Government') , value ='Colonial government' , inplace=True)
  df['installer'].replace(to_replace = ('Government of Misri') , value ='Misri Government' , inplace=True)
  df['installer'].replace(to_replace = ('Italy government') , value ='Italian government' , inplace=True)
  df['installer'].replace(to_replace = ('British colonial government') , value ='British government' , inplace=True)
  df['installer'].replace(to_replace = ('Concern /government') , value ='Concern/Government' , inplace=True)
  df['installer'].replace(to_replace = ('Village Government') , value ='Village government' , inplace=True)
  df['installer'].replace(to_replace = ('Government and Community') , value ='Government /Community' , inplace=True)
  df['installer'].replace(to_replace = ('Cetral government /RC') , value ='RC church/Central Gover' , inplace=True)
  df['installer'].replace(to_replace = ('Government /TCRS','Government/TCRS') , value ='TCRS /Government' , inplace=True)
  df['installer'].replace(to_replace = ('ADRA /Government') , value ='ADRA/Government' , inplace=True)


  return df

df=installer_encode(df)

df['installer_cat'] = df['installer']

c_ins = ['DWE','Unknown','Government','Community','DANIDA','RWE','District council', 'Central government',
         'KKKT','HESAWA','TCRS', 'world vision', 'Fini Water', 'RC Church','villagers','LGA']


df.loc[~df["installer_cat"].isin(c_ins), "installer_cat"] = "Others"

"""Fill missing values as "unknown""""

df['funder'].fillna(value='Unknown',inplace=True)
df['funder'].replace(to_replace = '0', value ='Unknown' , inplace=True)



df['funder_cat'] = df['funder'] 


c_fund = ['Danida','Unknown','Government Of Tanzania','Hesawa','Rwssp','World Bank','Kkkt', 'World Vision',
         'Unicef','Tasaf','District Council', 'Dhv', 'Private Individual', 'Dwsp','Norad','Germany Republi',
         'Tcrs','Ministry Of Water','Water','Dwe']
    

df.loc[~df["funder_cat"].isin(c_fund), "funder_cat"] = "Others"

df['longitude'].replace(to_replace = 0 , value =35.15, inplace=True)

df['wpt_name'].value_counts()

df['region_code'].value_counts()

df['amount_tsh'].value_counts()

df['population'].value_counts()

df['num_private'].value_counts()

df['subvillage'].value_counts()

df['scheme_name'].value_counts()

df.drop(columns=['wpt_name','scheme_name','region_code','amount_tsh','num_private','subvillage','id'],inplace=True )

df['population'].replace(to_replace = 0 , value =180, inplace=True)

"""Calculate the number of dates since recorded day and add new colum"""

df.date_recorded = pd.datetime(2013, 12, 3) - pd.to_datetime(df.date_recorded)
df.columns = ['days_since_recorded' if x=='date_recorded' else x for x in df.columns]
df.days_since_recorded = df.days_since_recorded.astype('timedelta64[D]').astype(int)
df.days_since_recorded.describe()

"""Fill missing values of public meeting and permit columns"""

df['public_meeting'].fillna(value=True,inplace=True)
df['permit'].fillna(value=True, inplace=True)

"""Drop columns"""

df.drop(columns=['funder','installer','construction_year','lga','ward'],inplace=True )

df1 = df.copy()

df1['permit'] = df1['permit'].astype(bool).astype(int)

df1['public_meeting'] = df1['public_meeting'].astype(bool).astype(int)

cat_col = ['basin','region','extraction_type_group','management','payment','water_quality','quantity',
               'source','waterpoint_type','decade','installer_cat','funder_cat']

num_col = ['gps_height','longitude','latitude','district_code','population','public_meeting','permit']

target='status_group'

def one_hot_encode(df,test,object_cols):
    
    OH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
    OH_cols_train = pd.DataFrame(OH_encoder.fit_transform(df[object_cols]))
    # print("dulaj")
    OH_cols_valid = pd.DataFrame(OH_encoder.transform(test[object_cols]))
  # One-hot encoding removed index; put it back
   
    OH_cols_train.index = df.index
    OH_cols_valid.index = test.index

  # Remove categorical columns (will replace with one-hot encoding)
    num_X_train = df.drop(object_cols, axis=1)
    num_X_valid = test.drop(object_cols, axis=1)

  # Add one-hot encoded columns to numerical features
    df = pd.concat([num_X_train, OH_cols_train], axis=1)
    test = pd.concat([num_X_valid, OH_cols_valid], axis=1)
    return df,test

"""## **To check the accuracy divide df1 data set into two parts**"""

used_cols = [c for c in df1.columns.tolist() if c not in [target]]
X=df1[used_cols]
y=df1[target]

X_train, y_train,= train_test_split(X,  test_size=0.2, random_state=42)
X_test, y_test = train_test_split( y, test_size=0.2, random_state=42)

X_train.info()

columns=['basin','region','extraction_type_group','management','payment','water_quality','quantity',
               'source','waterpoint_type','decade','installer_cat','funder_cat']
train, train_test = one_hot_encode(X_train, y_train, columns)

model =RandomForestClassifier()
model.fit(train, X_test)
predictions = model.predict(train_test)
print(predictions)

accuracy_score(y_test, predictions)

"""# Test data set"""

df_test_set.drop(columns=['management_group','scheme_management','quantity_group','source_class','source_type','quality_group',
                'payment_type','extraction_type_class','extraction_type', 'waterpoint_type_group'],inplace=True )

df_test_set['decade'] = df_test_set['construction_year'] 

df_test_set['construction_year'].replace(to_replace = 0, value = 1986, inplace=True)

df_test_set= decade_encode(df_test_set)

df_test_set.drop(columns=['recorded_by'],inplace=True )

df_train_set['installer'].fillna(value='Unknown',inplace=True)
df_test_set['installer'].replace(to_replace = '0', value ='Unknown' , inplace=True)
df_test_set = installer_encode(df_test_set)
df_test_set['installer_cat'] = df_test_set['installer'] 
df_test_set.loc[~df_test_set["installer_cat"].isin(c_ins), "installer_cat"] = "Others"

df_test_set['funder'].fillna(value='Unknown',inplace=True)
df_test_set['funder'].replace(to_replace = '0', value ='Unknown' , inplace=True)
df_test_set['funder_cat'] = df_test_set['funder'] 
df_test_set.loc[~df_test_set["funder_cat"].isin(c_fund), "funder_cat"] = "Others"

df_test_set['longitude'].replace(to_replace = 0 , value =35.15, inplace=True)

df_test_set.drop(columns=['wpt_name','scheme_name','region_code','amount_tsh','num_private','subvillage','id'],inplace=True )

df_test_set['population'].replace(to_replace = 0 , value =180, inplace=True)

df_test_set.date_recorded = pd.datetime(2013, 12, 3) - pd.to_datetime(df_test_set.date_recorded)
df_test_set.columns = ['days_since_recorded' if x=='date_recorded' else x for x in df_test_set.columns]
df_test_set.days_since_recorded = df_test_set.days_since_recorded.astype('timedelta64[D]').astype(int)

df_test_set['permit'].fillna(value=True, inplace=True)

df_test_set['public_meeting'].fillna(value=True,inplace=True)

df_test_set.drop(columns=['funder','installer','construction_year','lga','ward'],inplace=True )

df1_test = df_test_set.copy()

df1_test['permit'] = df1_test['permit'].astype(bool).astype(int)

df1_test['public_meeting'] = df1_test['public_meeting'].astype(bool).astype(int)

used_cols = [c for c in df1.columns.tolist() if c not in [target]]
X=df1[used_cols]
Y=df1_test
y=df1[target]

columns=['basin','region','extraction_type_group','management','payment','water_quality','quantity',
               'source','waterpoint_type','decade','installer_cat','funder_cat']
train, test = one_hot_encode(X, Y, columns)

train.head()

test.head()

model = RandomForestClassifier()
model.fit(train, y)
predictions = model.predict(test)
print(predictions)

column_names = ["id", "status_group"]
submission_dataset = pd.DataFrame(columns = column_names)
submission_dataset["id"] = df_test_set_value["id"]
submission_dataset.head()
np.testing.assert_array_equal(test.index.values, submission_dataset.index.values)
submission_dataset["status_group"] = predictions
print(submission_dataset.head())
submission_dataset.to_csv('pump_it_up_submission9.csv', index=False)
print(submission_dataset.columns)

from google.colab import files
files.download('pump_it_up_submission9.csv')