diff --git a/__pycache__/cross_validation.cpython-36.pyc b/__pycache__/cross_validation.cpython-36.pyc index 833f2b4..10f5fc4 100644 Binary files a/__pycache__/cross_validation.cpython-36.pyc and b/__pycache__/cross_validation.cpython-36.pyc differ diff --git a/__pycache__/function.cpython-36.pyc b/__pycache__/function.cpython-36.pyc index eb6a27e..e4eabd3 100644 Binary files a/__pycache__/function.cpython-36.pyc and b/__pycache__/function.cpython-36.pyc differ diff --git a/__pycache__/init_node.cpython-36.pyc b/__pycache__/init_node.cpython-36.pyc index 2318c30..dcab6eb 100644 Binary files a/__pycache__/init_node.cpython-36.pyc and b/__pycache__/init_node.cpython-36.pyc differ diff --git a/cross_validation.py b/cross_validation.py index 185fc1e..ada7789 100644 --- a/cross_validation.py +++ b/cross_validation.py @@ -7,6 +7,7 @@ import init_node as init import copy + def readFile(file): data = pandas.read_csv(file) dataframe = pandas.DataFrame(data) @@ -24,26 +25,18 @@ def featureScaling(input_data, output_data): merged_data = [] merged_data.extend(input_data) merged_data.extend(output_data) - # print("merged_data" + str(merged_data)) min_value = min(merged_data) max_value = max(merged_data) - # print("MIN value : " + str(min_value)) - # print("MAX value : " + str(max_value)) for element in input_data: - # data[index] = (data[index] - mean_value)/(max_value - min_value) result = (element - min_value)/(max_value - min_value) result = round(result,5) normalized_input_data.append(result) - # print("testtttttttttt") # print("normalized_input_data : " + str(normalized_input_data)) normalized_output_data = [] for element in output_data: - # data[index] = (data[index] - mean_value)/(max_value - min_value) result = (element - min_value)/(max_value - min_value) result = round(result,5) normalized_output_data.append(result) - # print("testtttttttttt") - # print("normalized_output_data : " + str(normalized_output_data)) return normalized_input_data, normalized_output_data @@ -51,7 +44,6 @@ def convertBack(value, input_data, output_data): merged_data = [] merged_data.extend(input_data) merged_data.extend(output_data) - # print("merged_data" + str(merged_data)) min_value = min(merged_data) max_value = max(merged_data) @@ -62,7 +54,6 @@ def normalizeError(value, input_data, output_data): merged_data = [] merged_data.extend(input_data) merged_data.extend(output_data) - # print("merged_data" + str(merged_data)) min_value = min(merged_data) max_value = max(merged_data) @@ -90,14 +81,10 @@ def calculateError(actual_output, desired_output): sse = 0 for index in range(0, len(actual_output)): error_value = (desired_output[index] - actual_output[index]) - # error_percentage = ((error_value/actual_output[index]) * 100) arr_error.append(error_value) - # print("absolute error of output #" + str(index) + " = " + str(error_value) + "(" + str(error_percentage) + " %)") - # print("arr _error : " + str(arr_error)) # calculate Sum Square Error (SSE) for element in arr_error: sse += (1/2)*(element * element) - # print("sse : " + str(sse)) return sse, arr_error def calcualteMSE(arr_error, size): @@ -109,37 +96,30 @@ def calcualteMSE(arr_error, size): def forward (dataframe_input, dataframe_output, data_all, line, arr_input_nodes, arr_output_nodes, arr_Y, arr_hidden_layers,\ arr_weight_bias, arr_bias, arr_weight_bias_output, arr_bias_output, function_number, beta, number_of_classes): - # calculate min, max, mean to be used in feature scaling - # min_value = data_all.min() - # max_value = data_all.max() - # mean_value = data_all.mean() - # mean_value = round(mean_value, 5) - # print("MIN from ALL : " + str(min_value[1])) - # print("MAX from ALL : " + str(max_value[1])) - # print("MEAN from ALL : " + str(mean_value[1])) # change number of line in to dataframe line = line - 2 - # print("line : " + str(line + 2)) - # print("line:" + str(line)) - # print("dataframe_input : " + str(dataframe_input)) data_input = dataframe_input.iloc[line] - # print("dataframe_input.iloc[line]" + str(dataframe_input.iloc[line].to_string(header=None,index=False))) data_input_template = copy.deepcopy(data_input) - # print(data_input) - # data_input = featureScaling(data_input) - # print(data_input) data_output = dataframe_output.iloc[line] data_output_template = copy.deepcopy(data_output) - # print(data_output) - # data_output = featureScaling(data_output) - data_input, data_output = featureScaling(data_input, data_output) - # print(len(data_input)) + + check_input = True + check_output = True + for element in data_input: + if((element < -1) or (element > 1)): + check_input = False + break + for element in data_output: + if((element < -1) or (element > 1)): + check_output = False + break + + if((check_input == False) and (check_output == False)): + data_input, data_output = featureScaling(data_input, data_output) # check if input nodes are enough input_check = False if (len(data_input) == len(arr_input_nodes)): - # print("Input later : OK") - # print() input_check = True else: print("invalid input nodes") @@ -150,34 +130,17 @@ def forward (dataframe_input, dataframe_output, data_all, line, arr_input_nodes, for data_element in data_input: arr_input_nodes[count] = data_element count += 1 - # print("input : " + str(arr_input_nodes)) - # print() # check if output nodes are enough output_check = False if (len(data_output) == len(arr_output_nodes)): - # print("Output layer : OK") - # print() output_check = True else: print("invalid output nodes") print() - # count = 0 - # if (check == True): - # for data_element in data_output: - # arr_output_nodes[count] = data_element - # count += 1 - # print("output : " + str(arr_output_nodes)) - # print() - # print("arr_output_nodes : " + str(arr_output_nodes)) - # print() - # CALCULATE Y of each node only when INPUT and OUTPUT are VALID + # CALCULATE Y of each node only when INPUT and OUTPUT are VALID if ((input_check == True) and (output_check == True)): - # print("BEFORE...") - # print("arr_output_nodes : " + str(arr_output_nodes)) - # print("arr_Y : " + str(arr_Y)) - print(arr_output_nodes) for layer_index in range(0, len(arr_Y) + 1): # calculate output if(layer_index == (len(arr_Y))): @@ -188,47 +151,30 @@ def forward (dataframe_input, dataframe_output, data_all, line, arr_input_nodes, result += (arr_hidden_layers[2][weight_node_index] * arr_Y[len(arr_Y) - 1][weight_node_index]) result += (arr_weight_bias_output[output_index] * arr_bias_output[output_index]) arr_output_nodes[output_index] = result - # print("BEFORE -> arr_output_nodes[" + str(output_index) + "] = " + str(arr_output_nodes[output_index])) arr_output_nodes[output_index] = useFunction(arr_output_nodes[output_index], function_number, beta) - # print("AFTER -> arr_output_nodes[" + str(output_index) + "] = " + str(arr_output_nodes[output_index])) else: for output_index in range(0, len(arr_output_nodes)): for weight_node_index in range(0, len(arr_hidden_layers[2])): result = 0 for weight_to_node_index in range(0, len(arr_hidden_layers[2][weight_node_index])): - # print("arr_hidden_layers[2][" + str(weight_node_index) + "][" + str(weight_to_node_index) + "] = " + str((arr_hidden_layers[2][weight_node_index][weight_to_node_index]))) - # print("arr_Y[" + str(len(arr_Y) - 1) + "][" + str(weight_node_index) + "] = " + str(arr_hidden_layers[2][weight_node_index][weight_to_node_index] * arr_Y[len(arr_Y) - 1][weight_node_index])) - # print("arr_weight_bias_output[0][" + str(output_index) +"] = " + str(arr_weight_bias_output[0][output_index])) - # print("arr_weight_bias_output = " + str(arr_weight_bias_output)) result += (arr_hidden_layers[2][weight_node_index][weight_to_node_index] * arr_Y[len(arr_Y) - 1][weight_node_index]) result += (arr_weight_bias_output[0][output_index]* arr_bias_output[0][output_index]) - # print(result) arr_output_nodes[output_index] = result - # print("arr_output_nodes[" + str(output_index) + "] = " + str(arr_output_nodes[output_index])) arr_output_nodes[output_index] = useFunction(arr_output_nodes[output_index], function_number, beta) - # print("AFTER -> arr_output_nodes[" + str(output_index) + "] = " + str(arr_output_nodes[output_index])) # y at the first hidden layer elif(layer_index == 0): - # for arr_Y_node_index in range(0, len(arr_Y[0])): for weight_node_index in range(0, len(arr_hidden_layers[0])): result = 0 - # print(len(arr_hidden_layers[0])) - # print("arr_hidden_layers[0] = " + str(arr_hidden_layers[0])) - # print("arr_input_nodes = " + str(arr_input_nodes)) if(number_of_classes == "1"): for weight_to_node_index in range(0, len(arr_hidden_layers[0][weight_node_index])): - # print("arr_input_nodes[weight_node_index] = " + str(arr_input_nodes[weight_node_index])) result += (arr_input_nodes[weight_node_index] * arr_hidden_layers[0][weight_node_index][weight_to_node_index]) else: for arr_input_index in range(0, len(arr_input_nodes)): for weight_to_node_index in range(0, len(arr_hidden_layers[0][weight_node_index])): - # print("arr_input_nodes[arr_input_index] = " + str(arr_input_nodes[arr_input_index])) result += (arr_input_nodes[arr_input_index] * arr_hidden_layers[0][weight_node_index][weight_to_node_index]) result += (arr_bias[0][weight_node_index] * arr_weight_bias[0][weight_node_index]) arr_Y[0][weight_node_index] = result - # print("BEFORE -> arr_Y[0][" + str(weight_node_index) + "] = " + str(arr_Y[0][weight_node_index])) arr_Y[0][weight_node_index] = useFunction(arr_Y[0][weight_node_index], function_number, beta) - # print("AFTER -> arr_Y[0][" + str(weight_node_index) + "] = " + str(arr_Y[0][weight_node_index])) # y at all hidden layers except the first layer else: for arr_Y_layer_index in range(1, len(arr_Y)): @@ -244,36 +190,29 @@ def forward (dataframe_input, dataframe_output, data_all, line, arr_input_nodes, arr_Y[arr_Y_layer_index - 1][weight_to_node_index]) result += (arr_bias[weight_layer_index][arr_Y_node_index] * arr_weight_bias[weight_layer_index][arr_Y_node_index]) arr_Y[arr_Y_layer_index][arr_Y_node_index] = result - # print("BEFORE -> arr_Y{" + str(arr_Y_layer_index) + "][" + str(arr_Y_node_index) + "] = " + str(arr_Y[arr_Y_layer_index][arr_Y_node_index])) arr_Y[arr_Y_layer_index][arr_Y_node_index] = useFunction(arr_Y[arr_Y_layer_index][arr_Y_node_index], function_number, beta) - # print("AFTER -> arr_Y{" + str(arr_Y_layer_index) + "][" + str(arr_Y_node_index) + "] = " + str(arr_Y[arr_Y_layer_index][arr_Y_node_index])) - # print() - # print("AFTER...") - # print("arr_Y" + str(arr_Y)) - # print("arr_output_nodes(actual output) : " + str(arr_output_nodes)) - # print("data output(desired output) : " + str(data_output)) - converted_arr_output_node = [] - for element_index in range(0, len(arr_output_nodes)): - converted_value = convertBack(arr_output_nodes[element_index], data_input_template, data_output_template) - converted_arr_output_node.append(converted_value) - # print("actual output : " + str(converted_arr_output_node)) - # print("desired output : " + str(data_output_template) ) - - sse, arr_error = calculateError(converted_arr_output_node, data_output_template) - # print("SSE = " + str(sse)) - # print("sse = " + str(sse)) - predicted_output = copy.deepcopy(converted_arr_output_node) - converted_arr_output_node.clear() - - #normalize error - normalized_arr_error = [] - for element_index in range(0, len(arr_error)): - error = normalizeError(arr_error[element_index], data_input_template, data_output_template) - normalized_arr_error.append(error) - - # sse, arr_error = calculateError(arr_output_nodes, data_output) - # return arr_input_nodes, sse, arr_error - return arr_input_nodes, sse, normalized_arr_error, predicted_output, data_output_template + + if((check_input == False) and (check_output == False)): + converted_arr_output_node = [] + for element_index in range(0, len(arr_output_nodes)): + converted_value = convertBack(arr_output_nodes[element_index], data_input_template, data_output_template) + converted_arr_output_node.append(converted_value) + + sse, arr_error = calculateError(converted_arr_output_node, data_output_template) + predicted_output = copy.deepcopy(converted_arr_output_node) + converted_arr_output_node.clear() + + #normalize error + normalized_arr_error = [] + for element_index in range(0, len(arr_error)): + error = normalizeError(arr_error[element_index], data_input_template, data_output_template) + normalized_arr_error.append(error) + + return arr_input_nodes, sse, normalized_arr_error, predicted_output, data_output_template + else: + sse, arr_error = calculateError(arr_output_nodes, data_output_template) + predicted_output = copy.deepcopy(arr_output_nodes) + return arr_input_nodes, sse, arr_error, predicted_output, data_output_template else: print("cannot do FORWARDING!") print() @@ -284,16 +223,9 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne arr_output_merged.append(arr_Y) arr_output_merged.append(arr_output_nodes) arr_grad = [] - # print("arr_grad_output" + str(arr_grad_output)) arr_grad.append(arr_grad_hidden) arr_grad.append(arr_grad_output) - # print("arr_grad = " +str(arr_grad)) - # print("INPUT : " + str(arr_input_nodes_with_value)) - # print("BEFORE.......") - # print("arr_Y : " + str(arr_Y)) - # print("arr_output_merged" + str(arr_output_merged)) - # print("arr_grad_hidden, arr_grad_output" + str(arr_grad)) - # print("arr_error : " + str(arr_error)) + # calculate local gradient # iterate loop in common way but call element in reversed position for list_index in range(0, len(arr_grad)): @@ -317,7 +249,6 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne else: arr_grad[len(arr_grad) - list_index - 1][output_index] = arr_error[output_index] * ( 2 * arr_output_nodes[output_index] * \ (1 - arr_output_nodes[output_index])) - # print("arr_grad after output calculation: " + str(arr_grad)) #in case of hidden layers else: reversed_layer_index = len(arr_grad) - list_index - 1 @@ -331,12 +262,10 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne (arr_Y[reversed_grad_layer_index][grad_node_index] * (1 - arr_Y[reversed_grad_layer_index][grad_node_index])) sum = 0 next_reversed_layer_index = reversed_layer_index + 1 - # for grad_output_node in range(0, len(arr_grad[next_reversed_layer_index])): for weight in arr_hidden_layers[len(arr_hidden_layers) - 1]: if(number_of_classes == "1"): sum += weight * arr_grad[next_reversed_layer_index] else: - # print("arr_grad = " + str(arr_grad)) for weight_node_index in range(0, len(arr_hidden_layers[len(arr_hidden_layers) - 1])): for weight_to_node_index in range(0, len(arr_hidden_layers[len(arr_hidden_layers) - 1][weight_node_index])): sum += (arr_hidden_layers[len(arr_hidden_layers) - 1][weight_node_index][weight_to_node_index] * arr_grad[next_reversed_layer_index][grad_node_index]) @@ -347,26 +276,14 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne (2 * arr_Y[reversed_grad_layer_index][grad_node_index] * (1 - arr_Y[reversed_grad_layer_index][grad_node_index])) sum = 0 next_reversed_layer_index = reversed_layer_index + 1 - # print("next_reversed_layer_index = " + str(next_reversed_layer_index)) - # for grad_output_node in range(0, len(arr_grad[next_reversed_layer_index])): if(number_of_classes == "1"): for weight in arr_hidden_layers[len(arr_hidden_layers) - 1]: - # if(number_of_classes == "1"): sum += weight * arr_grad[next_reversed_layer_index] else: for grad_output_index in range(0, len(arr_grad[next_reversed_layer_index])): for weight_node_index in range(0, len(arr_hidden_layers[len(arr_hidden_layers) - 1])): for weight_to_node_index in range(0, len(arr_hidden_layers[len(arr_hidden_layers) - 1][weight_node_index])): - - # print(arr_grad) - # print("weight = " +str(arr_hidden_layers[len(arr_hidden_layers) - 1][weight_node_index][weight_to_node_index])) - # print("arr_grad[" + str(next_reversed_layer_index) + "] = " +str(arr_grad[next_reversed_layer_index])) - # print("arr_grad[" + str(next_reversed_layer_index) +"][" + str(grad_node_index) +"] = " +str(arr_grad[next_reversed_layer_index][grad_node_index])) - # if(weight_node_index == grad_node_index): - # print("arr_grad = " + str(arr_grad)) sum += (arr_hidden_layers[len(arr_hidden_layers) - 1][weight_node_index][weight_to_node_index] * arr_grad[next_reversed_layer_index][grad_output_index]) - # sum += (arr_hidden_layers[len(arr_hidden_layers) - 1][weight_node_index][weight_to_node_index] * arr_grad[next_reversed_layer_index][grad_node_index]) - # print("sum = " + str(sum)) arr_grad[reversed_layer_index][reversed_grad_layer_index][grad_node_index] += sum # Input layer -> First Hidden layer else: @@ -381,8 +298,6 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne for weight_to_node_index in range(0, len(arr_hidden_layers[1][weight_layer_index][weight_node_index])): sum += (arr_hidden_layers[1][weight_layer_index][weight_node_index][weight_to_node_index] * \ arr_grad[reversed_layer_index][reversed_grad_layer_index + 1][grad_node_index]) - # print("arr_hidden_layers[1][" + str(weight_layer_index) + "][" + str(weight_node_index) + "][" + str(weight_to_node_index) + "]") - # print("arr_grad[" + str(reversed_layer_index) + "][" + str(reversed_grad_layer_index + 1) + "][" + str(grad_node_index) + "]") arr_grad[reversed_layer_index][reversed_grad_layer_index][grad_node_index] += sum elif(function_number == "2"): for grad_node_index in range(0, len(arr_grad[reversed_layer_index][reversed_grad_layer_index])): @@ -395,8 +310,6 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne for weight_to_node_index in range(0, len(arr_hidden_layers[1][weight_layer_index][weight_node_index])): sum += (arr_hidden_layers[1][weight_layer_index][weight_node_index][weight_to_node_index] * \ arr_grad[reversed_layer_index][reversed_grad_layer_index + 1][grad_node_index]) - # print("arr_hidden_layers[1][" + str(weight_layer_index) + "][" + str(weight_node_index) + "][" + str(weight_to_node_index) + "]") - # print("arr_grad[" + str(reversed_layer_index) + "][" + str(reversed_grad_layer_index + 1) + "][" + str(grad_node_index) + "]") arr_grad[reversed_layer_index][reversed_grad_layer_index][grad_node_index] += sum # calculate update weight for list_index in range(0, len(arr_hidden_layers)): @@ -407,8 +320,6 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne for weight_node_index in range(0, len(arr_hidden_layers[reversed_list_index][weight_layer_index])): result = 0 # for weight_to_node_index in range(0, len(arr_hidden_layers[reversed_list_index][weight_layer_index][weight_node_index])): - # print("BEFORE UPDATE -> arr_hidden_layers_new[2]["+str(weight_layer_index) + "][" + str(weight_node_index) + "]" \ - # + " = " + str(arr_hidden_layers_new[2][weight_layer_index][weight_node_index]) ) result += arr_hidden_layers[2][weight_layer_index][weight_node_index] result += (float(momentum) * (arr_hidden_layers_new[2][weight_layer_index][weight_node_index] - arr_hidden_layers[2][weight_layer_index][weight_node_index])) if(number_of_classes == "1"): @@ -417,56 +328,37 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne else: for grad_node_index in range(0, len(arr_grad[1])): if(weight_node_index == grad_node_index): - print("arr_grad[1][grad_node_index] = " +str(arr_grad[1][grad_node_index]) ) result += (float(learning_rate) * arr_grad[1][grad_node_index] * arr_Y[len(arr_Y) - 1][grad_node_index]) result = round(result,8) # #update weight arr_hidden_layers_new[2][weight_layer_index][weight_node_index] = result - # print("AFTER UPDATE -> arr_hidden_layers_new[2]["+str(weight_layer_index) + "][" + str(weight_node_index) + "]" \ - # + " = " + str(arr_hidden_layers_new[2][weight_layer_index][weight_node_index])) # update weight for bias if(number_of_classes == "1"): for bias_node_index in range(0, len(arr_weight_bias_output)): result = 0 result += (arr_weight_bias_output[bias_node_index] ) result += (float(momentum) * (arr_weight_bias_output_new[bias_node_index] - arr_weight_bias_output[bias_node_index] )) - # if(number_of_classes == "1"): result += (float(learning_rate) * arr_grad[1] * arr_Y[len(arr_Y) - 1][weight_node_index]) - # result = round(result,8) arr_weight_bias_output_new = result - # else: - # result += (float(learning_rate) * arr_grad[1][weight_node_index] * arr_Y[len(arr_Y) - 1][weight_node_index]) else: for bias_node_index in range(0, len(arr_weight_bias_output)): result = 0 result += (arr_weight_bias_output[bias_node_index]) result += (float(momentum) * (arr_weight_bias_output_new[bias_node_index] - arr_weight_bias_output[bias_node_index])) - # if(number_of_classes == "1"): - # result += (float(learning_rate) * arr_grad[1] * arr_Y[len(arr_Y) - 1][weight_node_index]) - # result = round(result,8) - # else: result += (float(learning_rate) * arr_grad[1][weight_node_index] * arr_Y[len(arr_Y) - 1][weight_node_index]) - # result = round(result,8) arr_weight_bias_output_new[bias_node_index] = result # weight at an input layer -> the first hidden layer elif(list_index == len(arr_hidden_layers) - 1): reversed_list_index = len(arr_hidden_layers) - list_index - 1 - # print("arr_hidden_layers[reversed_list_index] = " + str(arr_hidden_layers[reversed_list_index])) for weight_node_index in range(0, len(arr_hidden_layers[reversed_list_index])): for weight_to_node_index in range(0, len(arr_hidden_layers[reversed_list_index][weight_node_index])): result = 0 - # for weight_to_node_index in range(0, len(arr_hidden_layers[reversed_layer_index][weight_layer_index][weight_node_index])): - # print("BEFORE UPDATE -> arr_hidden_layers_new[0]["+str(weight_node_index) + "][" + str(weight_to_node_index) + \ - # "] = " + str(arr_hidden_layers_new[0][weight_node_index][weight_to_node_index]) ) result += arr_hidden_layers[0][weight_node_index][weight_to_node_index] result += (float(momentum) * (arr_hidden_layers_new[0][weight_node_index][weight_to_node_index] - \ arr_hidden_layers[0][weight_node_index][weight_to_node_index])) - # print("arr_input_nodes_with_value[weight_node_index] : " +str(arr_input_nodes_with_value[weight_to_node_index])) result += (float(learning_rate) * arr_grad[0][0][weight_node_index] * arr_input_nodes_with_value[weight_to_node_index]) arr_hidden_layers_new[0][weight_node_index][weight_to_node_index] = result - # print("AFTER UPDATE -> arr_hidden_layers_new[0]["+str(weight_node_index) + "][" + str(weight_to_node_index) + \ - # "] = " + str(arr_hidden_layers_new[0][weight_node_index][weight_to_node_index])) # update weight bias for bias_node_index in range(0, len(arr_weight_bias[0])): result = 0 @@ -484,15 +376,11 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne for weight_node_index in range(0, len(arr_hidden_layers[reversed_list_index][weight_layer_index])): for weight_to_node_index in range(0, len(arr_hidden_layers[reversed_list_index][weight_layer_index][weight_node_index])): result = 0 - # print("BEFORE UPDATE -> arr_hidden_layers_new[1]["+str(weight_layer_index) + "][" + str(weight_node_index) \ - # + "][" + str(weight_to_node_index) +"] = " + str(arr_hidden_layers_new[1][weight_layer_index][weight_node_index][weight_to_node_index]) ) result += arr_hidden_layers[reversed_list_index][weight_layer_index][weight_node_index][weight_to_node_index] result += (float(momentum) * (arr_hidden_layers_new[reversed_list_index][weight_layer_index][weight_node_index][weight_to_node_index] - \ arr_hidden_layers[reversed_list_index][weight_layer_index][weight_node_index][weight_to_node_index])) result += (float(learning_rate) * arr_grad[0][weight_layer_index - 1][weight_node_index]) arr_hidden_layers_new[reversed_list_index][weight_layer_index][weight_node_index][weight_to_node_index] = result - # print("AFTER UPDATE -> arr_hidden_layers_new[1]["+str(weight_layer_index) + "][" + str(weight_node_index) \ - # + "][" + str(weight_to_node_index) +"] = " + str(arr_hidden_layers_new[1][weight_layer_index][weight_node_index][weight_to_node_index]) ) #update weight bias for bias_layer_index in range(1, len(arr_weight_bias)): for bias_node_index in range(0, len(arr_weight_bias[bias_layer_index])): @@ -502,12 +390,6 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne result += (float(learning_rate) * arr_grad[0][bias_layer_index - 1][bias_node_index]) arr_weight_bias[bias_layer_index][bias_node_index] = result - # print("AFTER.......") - # print("arr_Y : " + str(arr_Y)) - # print("arr_output_merged" + str(arr_output_merged)) - # print("arr_grad_hidden, arr_grad_output" + str(arr_grad)) - # print("arr_error : " + str(arr_error)) - #reset arr_grad for list_index in range(0, len(arr_grad)): if(list_index == 0): @@ -515,7 +397,6 @@ def backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_ne for node_index in range(0, len(arr_grad[list_index][layer_index])): arr_grad[list_index][layer_index][node_index] = 0 else: - # for output_grad_index in range(0, len(arr_grad[list_index])): arr_grad[list_index]= 0 def crossValidation(input_file, output_file, full_data_file, number_of_fold, arr_input_nodes, arr_hidden_layers, arr_hidden_layers_new, arr_hidden_layers_template, \ @@ -525,20 +406,23 @@ def crossValidation(input_file, output_file, full_data_file, number_of_fold, arr data_input, dataframe_input, number_of_data_input, arr_row_input = readFile(input_file) data_output, dataframe_output, number_of_data_output, arr_row_output = readFile(output_file) data_all, dataframe_all, number_of_data_all, arr_row_all = readFile(full_data_file) - # print(dataframe_output) - # number_of_fold = 5 # JUST FOR TEST!!! size = math.ceil(number_of_data_input/int(number_of_fold)) - # print(size) + # split data into k parts data_chunk_input = list(chunks(arr_row_input, size)) print("\nData chunks ...") print(data_chunk_input) - # print (len(data_chunk)) + # test and train count = 0 - for test_element in data_chunk_input: - all_mse = [] + all_mse = [] + all_accuracy = [] + for test_element in data_chunk_input: # all_sse = [] + count_AC = 0 + count_BC = 0 + count_AD = 0 + count_BD = 0 count += 1 print("------------------------------" + str(count) + " fold ------------------------------") test_part = test_element @@ -553,74 +437,85 @@ def crossValidation(input_file, output_file, full_data_file, number_of_fold, arr for element_index in range(0, len(data_chunk_input[train_element_index])): # print("testtttt") # all_sse = [] + count_AC = 0 + count_BC = 0 + count_AD = 0 + count_BD = 0 for epoch_count in range(0, int(epoch)): - # print("*****************************************************************************************************") - # print(" FORWARD ") - # print("*****************************************************************************************************") + + # Forwarding arr_input_nodes_with_value, sse, arr_error, predicted_output, data_output_template = forward(dataframe_input, dataframe_output, data_all, data_chunk_input[train_element_index][element_index], arr_input_nodes, arr_output_nodes, arr_Y, \ arr_hidden_layers, arr_weight_bias, arr_bias, arr_weight_bias_output, arr_bias_output, function_number, beta, number_of_classes) - # all_sse.append(sse) - - # print("*****************************************************************************************************") - # print(" BACKWARD ") - # print("*****************************************************************************************************") + + # Backwarding arr_hidden_layers_template = copy.deepcopy(arr_hidden_layers_new) arr_weight_bias_output_template = copy.deepcopy(arr_weight_bias_output_new) arr_weight_bias_template = copy.deepcopy(arr_weight_bias_new) - # print("arr_hidden_layers_template = ") - # print("arr_hidden_layers = ") - # print(str(arr_hidden_layers)) - # print(str(arr_hidden_layers_template)) + backward(arr_input_nodes_with_value, arr_hidden_layers, arr_hidden_layers_new, arr_grad_hidden, arr_grad_output, arr_Y, arr_output_nodes, arr_error, function_number, \ momentum, learning_rate, number_of_classes, arr_weight_bias, arr_weight_bias_output, arr_weight_bias_new, arr_weight_bias_output_new) arr_hidden_layers = copy.deepcopy(arr_hidden_layers_template) arr_weight_bias_output = copy.deepcopy(arr_weight_bias_output_template) arr_weight_bias = copy.deepcopy(arr_weight_bias_template) - # print("arr_hidden_layers = ") - # print(str(arr_hidden_layers)) #reset arr_Y for layer_index in range(0, len(arr_Y)): for node_index in range(0,len(arr_Y[layer_index])): arr_Y[layer_index][node_index] = 0 - # print("arr_Y after reset: " + str(arr_Y)) #reset arr_output_nodes for node_index in range(0, len(arr_output_nodes)): arr_output_nodes[node_index] = 0 - - # #testing - # for test_element_index in range(0, len(test_part)): - # if(element_index == test_element_index): - # print("*****************************************************************************************************") - # print(" TESTING ") - # print("*****************************************************************************************************") - # print("arr_hidden_layers : " + str(arr_hidden_layers)) - # print("arr_hidden_layers_new : " + str(arr_hidden_layers_new)) - # print("arr_output_nodes : " + str(arr_output_nodes)) - # print("arr_Y : " + str(arr_Y)) - # all_sse = [] + + # Testing all_sse = [] for test_element_index in range(0, len(test_part)): - # if (element_index < len(test_part)): - print("test_part[" + str(element_index) + "] = " +str(test_part[test_element_index])) + desired_output = [] + print("test_part[" + str(test_element_index) + "] = " +str(test_part[test_element_index])) arr_input_nodes_with_value, sse, arr_error, predicted_output, data_output_template = forward(dataframe_input, dataframe_output, data_all, test_part[test_element_index], arr_input_nodes, arr_output_nodes, arr_Y, \ arr_hidden_layers_new, arr_weight_bias, arr_bias, arr_weight_bias_output, arr_bias_output, function_number, beta, number_of_classes) all_sse.append(sse) print("Predicted : " + str(predicted_output)) - # print("Desired Output : " + str(data_output_template.iloc[0:int(number_of_classes)].to_string(header=None,index=False))) if(number_of_classes == "1"): print("Desired Output:" + str(data_output_template[0])) elif(number_of_classes == "2"): - print("Desired Output:" + str(data_output_template[0]) + " " + str(data_output_template[1])) - # print("all_sse : " + str(all_sse)) - # print("number of all sse : " + str(len(all_sse))) - # print("sum sse : " + str(sum(all_sse))) + desired_output.append(data_output_template[0]) + desired_output.append(data_output_template[1]) + print("Desired Output:" + str(desired_output)) + + if(input_file == "cross-pat-input.csv"): + # format output + if(predicted_output[0] > predicted_output[1]): + output = [1,0] + elif(predicted_output[0] < predicted_output[1]): + output = [0,1] + # check condition + if(output == desired_output): + if(desired_output == [0,1]): + count_AC += 1 + elif(desired_output == [1,0]): + count_BD += 1 + else: + if(desired_output == [0,1]): + count_BC += 1 + elif(desired_output == [1,0]): + count_AD += 1 + mse = calcualteMSE(all_sse, len(test_part)) all_sse.clear() all_mse.append(mse) print("MSE : " + str(mse)) - # print("MSE (" + str(element_index) + ") : " + str(mse)) + print() + if(input_file == "cross-pat-input.csv"): + print("-------------------------------------------- CONFUSION MATRIX -----------------------------------------") + print("| Desire Output | -------------------------- Predicted Output -----------------------------------------") + print("| | (0,1) (1,0) ") + print("| (0,1) | " + str(count_AC) + " " + str(count_BC) + " ") + print("| (1,0) | " + str(count_AD) + " " + str(count_BD) + " ") + print("--------------------------------------------------------------------------------------------------------") + accuracy = ((count_AC + count_BD)/(count_AC + count_AD + count_BC + count_BD)) * 100 + print(" ACCURACY = " + str(accuracy) + " % ") + all_accuracy.append(accuracy) # #reset weight arr_hidden_layers = init.createHiddenLayers(number_of_features, number_of_layers, number_of_nodes, number_of_classes) @@ -633,38 +528,15 @@ def crossValidation(input_file, output_file, full_data_file, number_of_fold, arr for layer_index in range(0, len(arr_Y)): for node_index in range(0,len(arr_Y[layer_index])): arr_Y[layer_index][node_index] = 0 - # print("arr_Y after reset: " + str(arr_Y)) #reset arr_output_nodes for node_index in range(0, len(arr_output_nodes)): arr_output_nodes[node_index] = 0 - # print("arr_output_nodes after reset : " + str(arr_output_nodes)) print("------------------------------------------------------------------------------------------------------") - # print("Number of test data : " + str(len(test_part))) - # print("all_sse : " + str(len(all_sse))) - # mse = calcualteMSE(all_sse, len(test_part)) - # print("MSE = " + str(mse)) - print("Minimum MSE : " + str(min(all_mse))) - print() - # mse = calcualteMSE(all_sse, number_of_data_all) - # print("MSE : " + str(mse)) - # print("arr_hidden_layers : ") - # print(arr_hidden_layers) - # print("arr_hidden_layers_new : ") - # print(arr_hidden_layers_new) - # print("arr_hidden_layers_template : ") - # print(arr_hidden_layers_template) - # print() - # for element in test_part: - # print("*****************************************************************************************************") - # print(" TESTING ") - # print("*****************************************************************************************************") - # all_sse = [] - # arr_input_nodes_with_value, sse, arr_error = forward(dataframe_input, dataframe_output, data_all, element, arr_input_nodes, arr_output_nodes, arr_Y, \ - # arr_hidden_layers, arr_weight_bias, arr_bias, arr_weight_bias_output, arr_bias_output, function_number, beta) - # all_sse.append(sse) - - # print("TEST------") - # print(test_part) - # print() \ No newline at end of file + desired_output.clear() + + print("Minimum MSE : " + str(min(all_mse))) + print("Average MSE : " + str(sum(all_mse)/len(all_mse))) + if(input_file == "cross-pat-input.csv"): + print("Average accuracy = " + str((sum(all_accuracy)/len(all_accuracy)))) diff --git a/function.py b/function.py index e632df3..59f8aaa 100644 --- a/function.py +++ b/function.py @@ -4,22 +4,3 @@ def sigmoid(x): return 1 / (1 + math.exp(-x)) def hyperbolicTangent(x): return math.tanh(x/2) - -def unitStep(x,beta): - if (x >= beta): - y = 1 - return y - elif (x < beta): - y = -1 - return y - -def ramp(x,beta): - if(x >= beta): - y = 1 - return y - elif(-beta < x < beta): - y = x - return y - elif(x <= -beta): - y = -1 - return y \ No newline at end of file diff --git a/init_node.py b/init_node.py index c322d00..9f7f4f1 100644 --- a/init_node.py +++ b/init_node.py @@ -35,7 +35,6 @@ def createHiddenLayers(number_of_features,number_of_layers,number_of_nodes,numbe node.clear() final.append(layer) - # final.append(layer) #The last hidden layer connected to an output layer count = 0 while count < int(number_of_nodes): @@ -57,16 +56,11 @@ def createBias(number_of_nodes,number_of_layers): for layer_count in range(0,int(number_of_layers)): arr = np.random.uniform(low=-1.0,high=1.0,size=int(number_of_nodes)) weight_bias.append(arr) - # weight_bias.append(layer1) #initial bias as 1 for layer_count in range(0,int(number_of_layers)): arr = np.ones(int(number_of_nodes)) bias.append(arr) - # bias.append(layer2) - # print(weight_bias) - # print() - # print(bias) return weight_bias, bias @@ -77,7 +71,6 @@ def createY(number_of_nodes, number_of_layers): for layer_count in range(0, int(number_of_layers)): arr = np.zeros(int(number_of_nodes)) layer.append(arr) - # final.append(layer) return layer def createLocalGradOutput(number_of_classes): @@ -87,9 +80,7 @@ def createLocalGradOutput(number_of_classes): def createLocalGradHidden(number_of_nodes, number_of_layers): node = [] arr_grad_hidden = [] - # final = [] for layer_count in range(0, int(number_of_layers)): arr = np.zeros(int(number_of_nodes)) arr_grad_hidden.append(arr) - # final.append(layer) return arr_grad_hidden diff --git a/mlp.py b/mlp.py index cb005e2..ce65556 100644 --- a/mlp.py +++ b/mlp.py @@ -59,8 +59,6 @@ def main(): arr_hidden_layers = init.createHiddenLayers(number_of_features,number_of_layers,number_of_nodes,number_of_classes) arr_hidden_layers_new = init.createHiddenLayers(number_of_features,number_of_layers,number_of_nodes,number_of_classes) arr_hidden_layers_template = init.createHiddenLayers(number_of_features,number_of_layers,number_of_nodes,number_of_classes) - # arr_hidden_layers_new = copy.deepcopy(arr_hidden_layers) - # arr_hidden_layers_template = copy.deepcopy(arr_hidden_layers) arr_Y = init.createY(number_of_nodes, number_of_layers) arr_weight_bias, arr_bias = init.createBias(number_of_nodes, number_of_layers) arr_weight_bias_new, arr_bias_output_new = init.createBias(number_of_nodes, number_of_layers) @@ -71,37 +69,20 @@ def main(): arr_weight_bias_output_template, arr_bias_output_template =init.createBias(number_of_classes, 1) arr_grad_output = init.createLocalGradOutput(number_of_classes) arr_grad_hidden = init.createLocalGradHidden(number_of_nodes, number_of_layers) - cv.crossValidation("flood-input.csv", "flood-output.csv", "flood-data.csv", fold, arr_input_nodes, arr_hidden_layers, arr_hidden_layers_new, arr_hidden_layers_template, \ + + input_file = "cross-pat-input.csv" + output_file = "cross-pat-output.csv" + data_file = "cross-pat.csv" + cv.crossValidation(input_file, output_file, data_file, fold, arr_input_nodes, arr_hidden_layers, arr_hidden_layers_new, arr_hidden_layers_template, \ arr_Y, arr_output_nodes, arr_weight_bias, arr_bias, arr_weight_bias_output, arr_bias_output, function, momentum, learning_rate, beta, arr_grad_hidden, arr_grad_output,\ number_of_features, number_of_layers, number_of_nodes, number_of_classes, epoch, arr_weight_bias_template, arr_weight_bias_output_template, arr_weight_bias_new, \ arr_weight_bias_output_new) - # print("arr_hidden_layers : ") - # print(arr_hidden_layers) - # print("arr_hidden_layers_new : ") - # print(arr_hidden_layers_new) - # print("arr_hidden_layers_template : ") - # print(arr_hidden_layers_template) - # print() + print("size of list containing hidden layer : " + str(len(arr_hidden_layers))) print(str(len(arr_hidden_layers[1])) + " layer(s) of weigh connected to hidden node") print("1 layer of weight connected to INPUT layer") print("1 layer connected to OUTPUT layer") print("total layer of weight : " + str(1 + len(arr_hidden_layers))) - #FOR DEBUGGING!!! - # print("all layer : " + str(len(arr_hidden_layers))) - # print("hidden : " + str(len(arr_hidden_layers[1]))) - # print("member in hidden : " + str(len(arr_hidden_layers[1][0]))) - # print(arr_Y) - print("arr_weight_bias : " + str(arr_weight_bias)) - print("arr_bias : " + str(arr_bias)) - print("arr_weight_bias_output : " + str(arr_weight_bias_output)) - print("arr_bias_output : " + str(arr_bias_output)) - # print("arr_weight_bias : " + str(arr_weight_bias)) - # print("arr_bias : " + str(arr_bias)) - # print("arr_grad_output : " + str(arr_grad_output)) - # print("arr_grad_hidden : " + str(arr_grad_hidden)) - # print() - if __name__ == '__main__': main() \ No newline at end of file