diff --git a/Examples/train_network.py b/Examples/train_network.py index eb5cb972..cfcf8c87 100644 --- a/Examples/train_network.py +++ b/Examples/train_network.py @@ -1,4 +1,5 @@ import multiprocessing as mp +from GravNN.Networks.Callbacks import LossComponentsCallback from GravNN.Networks.script_utils import save_training from GravNN.Networks.utils import configure_run_args from GravNN.Networks.Configs import * @@ -24,6 +25,7 @@ def main(): "epochs" : [5000], "batch_size" : [4096], "PINN_constraint_fcn": ["pinn_alc"], + "dtype" : ['float64'] }) diff --git a/GravNN/Networks/Callbacks.py b/GravNN/Networks/Callbacks.py index 946f0828..28123fbf 100644 --- a/GravNN/Networks/Callbacks.py +++ b/GravNN/Networks/Callbacks.py @@ -91,6 +91,79 @@ def on_train_end(self, logs=None): self.time_delta = np.round(self.end_time - self.train_start, 2) +class RMSComponentsCallback(tf.keras.callbacks.Callback): + """Loss Components Callback that prints out RMS component metrics every 100 epochs""" + def __init__(self, batch_size, print_interval=100): + super().__init__() + self.batch_size = batch_size + self.print_interval = print_interval + + self.N_train_samples = 0 + self.loss_components = [0, 0, 0, 0, 0, 0, 0] + + self.N_val_samples = 0 + self.val_loss_components = [0, 0, 0, 0, 0, 0, 0] + + def incremental_average_loss(self, avg, val, N, M): + old_avg = [N * i for i in avg] + new_avg = [sum(col)/len(col) for col in zip(*val)] + full_new_avg = [M * i for i in new_avg] + + final_avg = [] + for i in range(len(full_new_avg)): + final_avg.append(full_new_avg[i] + old_avg[i]) + final_final_avg = [(i / (N+M)) for i in final_avg] + return final_final_avg + + def incremental_average(self, avg, val, N, M): + np.mean(val) + return ((N*avg + M*val)/(M+N)) + + def on_train_batch_end(self, batch, logs=None): + self.loss_components = self.incremental_average_loss( + self.loss_components, + logs['loss_components'], + self.N_train_samples, + self.batch_size + ) + self.N_train_samples += self.batch_size + + def on_test_batch_end(self, batch, logs=None): + self.val_loss_components = self.incremental_average_loss( + self.val_loss_components, + logs['loss_components'], + self.N_val_samples, + self.batch_size + ) + self.N_val_samples += self.batch_size + + def on_epoch_begin(self,epoch,logs=None): + self.N_train_samples = 0 + self.loss_components = [0, 0, 0, 0, 0, 0, 0] + + self.N_val_samples = 0 + self.val_loss_components = [0, 0, 0, 0, 0, 0, 0] + + def on_epoch_end(self, epoch, logs=None): + if epoch % self.print_interval == 0: + print("Loss Components: ") + print(self.loss_components) + print("Validation Loss Components: ") + print(self.val_loss_components) + + # Overwrite batch logs for epoch logs (to be saved in history obj) + logs['loss_components'] = self.loss_components + logs['val_loss_components'] = self.val_loss_components + + def on_train_begin(self, logs=None): + self.train_start = time.time() + self.start_time = time.time() + + def on_train_end(self, logs=None): + self.end_time = time.time() + self.time_delta = np.round(self.end_time - self.train_start, 2) + + class GradientCallback(tf.keras.callbacks.Callback): """Callback that plots out the gradients for each hidden layer after every 1000 epochs""" diff --git a/GravNN/Networks/Configs/Eros_Configs.py b/GravNN/Networks/Configs/Eros_Configs.py index bec5b9f1..bbd7701c 100644 --- a/GravNN/Networks/Configs/Eros_Configs.py +++ b/GravNN/Networks/Configs/Eros_Configs.py @@ -20,8 +20,12 @@ def get_default_eros_config(): "analytic_truth": ["poly_stats_"], "gravity_data_fcn" : [get_poly_data], "remove_point_mass": [False], # remove point mass from polyhedral model + "x_transformer": [UniformScaler(feature_range=(-1, 1))], + "u_transformer": [UniformScaler(feature_range=(-1, 1))], + "a_transformer": [UniformScaler(feature_range=(-1, 1))], + "dummy_transformer": [DummyScaler()], "override" : [False], - "ref_radius" : [Eros().radius], + 'ref_radius' : [Eros().radius], "ref_radius_min" : [Eros().radius_min], } diff --git a/GravNN/Networks/Configs/Network_Configs.py b/GravNN/Networks/Configs/Network_Configs.py index bb000831..36e2b6b1 100644 --- a/GravNN/Networks/Configs/Network_Configs.py +++ b/GravNN/Networks/Configs/Network_Configs.py @@ -65,5 +65,4 @@ def PINN_III(): "loss_fcn" : ['avg_percent_summed_rms'], } config.update(network_config) - return config - + return config \ No newline at end of file diff --git a/GravNN/Networks/Constraints.py b/GravNN/Networks/Constraints.py index 8271f939..1f1e752b 100644 --- a/GravNN/Networks/Constraints.py +++ b/GravNN/Networks/Constraints.py @@ -115,7 +115,6 @@ def pinn_ALC(f, x, training): u_xx = g1.batch_jacobian(u_x, x, experimental_use_pfor=True) accel = tf.multiply(u_x, -1.0) # u_x must be first s.t. -1 dtype is inferred - laplace = laplacian(u_xx) curl_x = tf.math.subtract(u_xx[:, 2, 1], u_xx[:, 1, 2]) diff --git a/GravNN/Networks/Model.py b/GravNN/Networks/Model.py index 2dea755c..46dd0a42 100644 --- a/GravNN/Networks/Model.py +++ b/GravNN/Networks/Model.py @@ -13,7 +13,7 @@ from GravNN.Networks.Losses import compute_rms_components, compute_percent_error from GravNN.Networks.Schedules import get_schedule from GravNN.Networks.utils import populate_config_objects, configure_optimizer -from GravNN.Networks.Callbacks import SimpleCallback +from GravNN.Networks.Callbacks import SimpleCallback, RMSComponentsCallback from GravNN.Networks.HIG import HIG_pinv import GravNN @@ -133,11 +133,12 @@ def train_step_fcn(self, data): self.optimizer.apply_gradients([ (grad, var) for (grad, var) in zip(gradients, self.network.trainable_variables) if grad is not None ]) - return { "loss": loss, "percent_mean": tf.reduce_mean(percent_components), "percent_max": tf.reduce_max(percent_components), + "loss_components" : rms_components, + "percent_components" : percent_components # "weighted_percent": w_percent_error, # "adaptive_constant": adaptive_constant, } # 'grads' : grad_comp_list} @@ -155,6 +156,8 @@ def test_step_fcn(self, data): return {"loss": loss, "percent_mean": tf.reduce_mean(percent_components), "percent_max": tf.reduce_max(percent_components), + "loss_components" : rms_components, + "percent_components" : percent_components # "weighted_percent": w_percent_error, } @@ -168,6 +171,7 @@ def train(self, data, initialize_optimizer=True): # Train network callback = SimpleCallback(self.config['batch_size'][0], print_interval=10) + rmscallback = RMSComponentsCallback(self.config['batch_size'][0]) schedule = get_schedule(self.config) history = self.fit( @@ -175,7 +179,7 @@ def train(self, data, initialize_optimizer=True): epochs=self.config["epochs"][0], verbose=0, validation_data=data.valid_data, - callbacks=[callback, schedule], + callbacks=[callback, rmscallback, schedule], use_multiprocessing=True ) history.history["time_delta"] = callback.time_delta diff --git a/Scripts/Networks/visualize_loss.py b/Scripts/Networks/visualize_loss.py new file mode 100644 index 00000000..a4cb3f78 --- /dev/null +++ b/Scripts/Networks/visualize_loss.py @@ -0,0 +1,56 @@ +''' Visualize the magnitudes of each loss component against one another''' +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np +import ast + +df = pd.read_csv('history.csv', sep=",") +loss_components = [] +for i in range(len(df['loss_components'])): + to_ints = ast.literal_eval(df['loss_components'][i]) + loss_components.append(to_ints) + +percent_components = [] +for i in range(len(df['loss_components'])): + to_ints = ast.literal_eval(df['loss_components'][i]) + loss_components.append(to_ints) + +general_x = [j[0] for j in loss_components] +general_y = [j[1] for j in loss_components] +general_z = [j[2] for j in loss_components] +laplace = [j[3] for j in loss_components] +curl_x = [j[4] for j in loss_components] +curl_y = [j[5] for j in loss_components] +curl_z = [j[6] for j in loss_components ] + +general_x_percent = [j[0] for j in percent_components] +general_y_percent = [j[1] for j in percent_components] +general_z_percent = [j[2] for j in percent_components] +laplace_percent = [j[3] for j in percent_components] +curl_x_percent = [j[4] for j in percent_components] +curl_y_percent = [j[5] for j in percent_components] +curl_z_percent = [j[6] for j in percent_components] + +epochs = range(1,100) + +plt.plot(epochs, general_x, 'b', label='General Loss X') +plt.plot(epochs, general_y, 'g', label='General Loss Y') +plt.plot(epochs, general_z, 'r', label='General Loss Z') +plt.plot(epochs, laplace, 'c', label='Laplace Loss') +plt.plot(epochs, curl_x, 'm', label='Curl Loss X') +plt.plot(epochs, curl_y, 'y', label='Curl Loss Y') +plt.plot(epochs, curl_z, 'k', label='Curl Loss Z') + +plt.plot(epochs, general_x_percent, 'b', label='General Loss X Percent') +plt.plot(epochs, general_y_percent, 'g', label='General Loss Y Percent') +plt.plot(epochs, general_z_percent, 'r', label='General Loss Z Percent') +plt.plot(epochs, laplace_percent, 'c', label='Laplace Loss Percent') +plt.plot(epochs, curl_x_percent, 'm', label='Curl Loss X Percent') +plt.plot(epochs, curl_y_percent, 'y', label='Curl Loss Y Percent') +plt.plot(epochs, curl_z_percent, 'k', label='Curl Loss Z Percent') + +plt.title('Training and Validation Loss') +plt.xlabel('Epochs') +plt.ylabel('Loss') +plt.legend() +plt.show() \ No newline at end of file diff --git a/Scripts/Networks/visualize_networks_comparison.py b/Scripts/Networks/visualize_networks_comparison.py new file mode 100644 index 00000000..cf337ca3 --- /dev/null +++ b/Scripts/Networks/visualize_networks_comparison.py @@ -0,0 +1,59 @@ +''' +Plot visualization of different model hyperparameters against their corresponding errors +''' + +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np + +# Read to dataframe +column_names = ["Model", "Error", "Error_STD", "Time", "Layers", "Batch_Size", "Learning_Rate", "Activation"] +df = pd.read_csv('test_hyperparameters.txt', sep=",", names=column_names) + +# Check for nulls and drop +df = df.dropna() + +# Make graphs +print("Smallest Error: \n") +best_models = df.nsmallest(20, 'Error') +best_batch_size = best_models["Batch_Size"].mode() +best_learning_rate = best_models["Learning_Rate"].mode() +best_activation = best_models["Activation"].mode() +best_layers = best_models["Layers"].mode() +print(best_models["Layers"]) +print(best_models["Layers"].nsmallest()) + +print("Best Models: \n") +print(best_models) +print("Batch Size: " + str(best_batch_size)) +print("Learning Rate: " + str(best_learning_rate)) +print("Activation: " + str(best_activation)) +print("Layers: " + str(best_layers)) + +plt.figure(1) +plt.plot(df["Learning_Rate"].tolist(), df["Error"].tolist(), 'bo') +plt.xlabel("Learning Rate") +plt.ylabel("Error") +plt.title("Learning Rate vs. Error") +plt.show() + +plt.figure(2) +plt.plot(df["Batch_Size"].tolist(), df["Error"].tolist(), 'bo') +plt.xlabel("Batch Size") +plt.ylabel("Error") +plt.title("Batch Size vs. Error") +plt.show() + +plt.figure(3) +plt.plot(df["Layers"].tolist(), df["Error"].tolist(), 'bo') +plt.xlabel("Layers") +plt.ylabel("Error") +plt.title("Layers vs. Error") +plt.show() + +plt.figure(4) +plt.plot(df["Activation"].tolist(), df["Error"].tolist(), 'bo') +plt.xlabel("Activation") +plt.ylabel("Error") +plt.title("Activation vs. Error") +plt.show() \ No newline at end of file