First Pull Request

Examples/train_network.py

@@ -1,4 +1,5 @@
 import multiprocessing as mp
+from GravNN.Networks.Callbacks import LossComponentsCallback
 from GravNN.Networks.script_utils import save_training
 from GravNN.Networks.utils import configure_run_args
 from GravNN.Networks.Configs import *
@@ -24,6 +25,7 @@ def main():
         "epochs" : [5000],
         "batch_size" : [4096],
         "PINN_constraint_fcn": ["pinn_alc"],
+        "dtype" : ['float64']
     })
 
 

GravNN/Networks/Callbacks.py

@@ -91,6 +91,79 @@ def on_train_end(self, logs=None):
         self.time_delta = np.round(self.end_time - self.train_start, 2)
 
 
+class RMSComponentsCallback(tf.keras.callbacks.Callback):
+    """Loss Components Callback that prints out RMS component metrics every 100 epochs"""
+    def __init__(self, batch_size, print_interval=100):
+        super().__init__()
+        self.batch_size = batch_size
+        self.print_interval = print_interval
+
+        self.N_train_samples = 0
+        self.loss_components = [0, 0, 0, 0, 0, 0, 0]
+
+        self.N_val_samples = 0
+        self.val_loss_components = [0, 0, 0, 0, 0, 0, 0]
+
+    def incremental_average_loss(self, avg, val, N, M):
+        old_avg = [N * i for i in avg]
+        new_avg = [sum(col)/len(col) for col in zip(*val)]
+        full_new_avg = [M * i for i in new_avg]
+
+        final_avg = []
+        for i in range(len(full_new_avg)):
+            final_avg.append(full_new_avg[i] + old_avg[i])
+        final_final_avg = [(i / (N+M)) for i in final_avg]
+        return final_final_avg
+
+    def incremental_average(self, avg, val, N, M):
+        np.mean(val)
+        return ((N*avg + M*val)/(M+N))
+
+    def on_train_batch_end(self, batch, logs=None):
+        self.loss_components = self.incremental_average_loss(
+            self.loss_components,
+            logs['loss_components'],
+            self.N_train_samples,
+            self.batch_size
+        )
+        self.N_train_samples += self.batch_size
+
+    def on_test_batch_end(self, batch, logs=None):
+        self.val_loss_components = self.incremental_average_loss(
+            self.val_loss_components,
+            logs['loss_components'],
+            self.N_val_samples,
+            self.batch_size
+        )
+        self.N_val_samples += self.batch_size
+
+    def on_epoch_begin(self,epoch,logs=None):
+        self.N_train_samples = 0
+        self.loss_components = [0, 0, 0, 0, 0, 0, 0]
+
+        self.N_val_samples = 0
+        self.val_loss_components = [0, 0, 0, 0, 0, 0, 0]
+
+    def on_epoch_end(self, epoch, logs=None):
+        if epoch % self.print_interval == 0:
+            print("Loss Components: ")
+            print(self.loss_components)
+            print("Validation Loss Components: ")
+            print(self.val_loss_components)        
+
+        # Overwrite batch logs for epoch logs (to be saved in history obj)
+        logs['loss_components'] = self.loss_components
+        logs['val_loss_components'] = self.val_loss_components
+
+    def on_train_begin(self, logs=None):
+        self.train_start = time.time()
+        self.start_time = time.time()
+
+    def on_train_end(self, logs=None):
+        self.end_time = time.time()
+        self.time_delta = np.round(self.end_time - self.train_start, 2)
+
+
 class GradientCallback(tf.keras.callbacks.Callback):
     """Callback that plots out the gradients for each hidden layer after every 1000 epochs"""
 

GravNN/Networks/Configs/Eros_Configs.py

@@ -20,8 +20,12 @@ def get_default_eros_config():
         "analytic_truth": ["poly_stats_"],
         "gravity_data_fcn" : [get_poly_data],
         "remove_point_mass": [False],  # remove point mass from polyhedral model
+        "x_transformer": [UniformScaler(feature_range=(-1, 1))],
+        "u_transformer": [UniformScaler(feature_range=(-1, 1))],
+        "a_transformer": [UniformScaler(feature_range=(-1, 1))],
+        "dummy_transformer": [DummyScaler()],
         "override" : [False],
-        "ref_radius" : [Eros().radius],
+        'ref_radius' : [Eros().radius],
         "ref_radius_min" : [Eros().radius_min],
     }
 

GravNN/Networks/Configs/Network_Configs.py

@@ -65,5 +65,4 @@ def PINN_III():
         "loss_fcn" : ['avg_percent_summed_rms'],
     }
     config.update(network_config)
-    return config
-
+    return config

GravNN/Networks/Constraints.py

@@ -115,7 +115,6 @@ def pinn_ALC(f, x, training):
     u_xx = g1.batch_jacobian(u_x, x, experimental_use_pfor=True)
 
     accel = tf.multiply(u_x, -1.0) # u_x must be first s.t. -1 dtype is inferred
-
     laplace = laplacian(u_xx)
 
     curl_x = tf.math.subtract(u_xx[:, 2, 1], u_xx[:, 1, 2])

GravNN/Networks/Model.py

@@ -13,7 +13,7 @@
 from GravNN.Networks.Losses import compute_rms_components, compute_percent_error
 from GravNN.Networks.Schedules import get_schedule
 from GravNN.Networks.utils import populate_config_objects, configure_optimizer
-from GravNN.Networks.Callbacks import SimpleCallback
+from GravNN.Networks.Callbacks import SimpleCallback, RMSComponentsCallback
 from GravNN.Networks.HIG import HIG_pinv
 import GravNN
 
@@ -133,11 +133,12 @@ def train_step_fcn(self, data):
         self.optimizer.apply_gradients([
             (grad, var) for (grad, var) in zip(gradients, self.network.trainable_variables) if grad is not None
             ])
-
         return {
             "loss": loss,
             "percent_mean": tf.reduce_mean(percent_components),
             "percent_max": tf.reduce_max(percent_components),
+            "loss_components" : rms_components,
+            "percent_components" : percent_components
             # "weighted_percent": w_percent_error,
            # "adaptive_constant": adaptive_constant,
         }  # 'grads' : grad_comp_list}
@@ -155,6 +156,8 @@ def test_step_fcn(self, data):
         return {"loss": loss, 
                 "percent_mean": tf.reduce_mean(percent_components),
                 "percent_max": tf.reduce_max(percent_components),
+                "loss_components" : rms_components,
+                "percent_components" : percent_components
                 # "weighted_percent": w_percent_error,
                 }
 
@@ -168,14 +171,15 @@ def train(self, data, initialize_optimizer=True):
 
         # Train network
         callback = SimpleCallback(self.config['batch_size'][0], print_interval=10)
+        rmscallback = RMSComponentsCallback(self.config['batch_size'][0])
         schedule = get_schedule(self.config)
 
         history = self.fit(
             data.train_data,
             epochs=self.config["epochs"][0],
             verbose=0,
             validation_data=data.valid_data,
-            callbacks=[callback, schedule],
+            callbacks=[callback, rmscallback, schedule],
             use_multiprocessing=True
         )
         history.history["time_delta"] = callback.time_delta

Scripts/Networks/visualize_loss.py

@@ -0,0 +1,56 @@
+''' Visualize the magnitudes of each loss component against one another'''
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+import ast
+
+df = pd.read_csv('history.csv', sep=",")
+loss_components = []
+for i in range(len(df['loss_components'])):
+    to_ints = ast.literal_eval(df['loss_components'][i])
+    loss_components.append(to_ints)
+
+percent_components = []
+for i in range(len(df['loss_components'])):
+    to_ints = ast.literal_eval(df['loss_components'][i])
+    loss_components.append(to_ints)
+
+general_x = [j[0] for j in loss_components]
+general_y = [j[1] for j in loss_components]
+general_z = [j[2] for j in loss_components]
+laplace = [j[3] for j in loss_components]
+curl_x = [j[4] for j in loss_components]
+curl_y = [j[5] for j in loss_components]
+curl_z = [j[6] for j in loss_components ]
+
+general_x_percent = [j[0] for j in percent_components]
+general_y_percent = [j[1] for j in percent_components]
+general_z_percent = [j[2] for j in percent_components]
+laplace_percent = [j[3] for j in percent_components]
+curl_x_percent = [j[4] for j in percent_components]
+curl_y_percent = [j[5] for j in percent_components]
+curl_z_percent = [j[6] for j in percent_components]
+
+epochs = range(1,100)
+
+plt.plot(epochs, general_x, 'b', label='General Loss X')
+plt.plot(epochs, general_y, 'g', label='General Loss Y')
+plt.plot(epochs, general_z, 'r', label='General Loss Z')
+plt.plot(epochs, laplace, 'c', label='Laplace Loss')
+plt.plot(epochs, curl_x, 'm', label='Curl Loss X')
+plt.plot(epochs, curl_y, 'y', label='Curl Loss Y')
+plt.plot(epochs, curl_z, 'k', label='Curl Loss Z')
+
+plt.plot(epochs, general_x_percent, 'b', label='General Loss X Percent')
+plt.plot(epochs, general_y_percent, 'g', label='General Loss Y Percent')
+plt.plot(epochs, general_z_percent, 'r', label='General Loss Z Percent')
+plt.plot(epochs, laplace_percent, 'c', label='Laplace Loss Percent')
+plt.plot(epochs, curl_x_percent, 'm', label='Curl Loss X Percent')
+plt.plot(epochs, curl_y_percent, 'y', label='Curl Loss Y Percent')
+plt.plot(epochs, curl_z_percent, 'k', label='Curl Loss Z Percent')
+
+plt.title('Training and Validation Loss')
+plt.xlabel('Epochs')
+plt.ylabel('Loss')
+plt.legend()
+plt.show()

Scripts/Networks/visualize_networks_comparison.py

@@ -0,0 +1,59 @@
+'''
+Plot visualization of different model hyperparameters against their corresponding errors
+'''
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Read to dataframe
+column_names = ["Model", "Error", "Error_STD", "Time", "Layers", "Batch_Size", "Learning_Rate", "Activation"]
+df = pd.read_csv('test_hyperparameters.txt', sep=",", names=column_names)
+
+# Check for nulls and drop
+df = df.dropna()
+
+# Make graphs
+print("Smallest Error: \n")
+best_models = df.nsmallest(20, 'Error')
+best_batch_size = best_models["Batch_Size"].mode()
+best_learning_rate = best_models["Learning_Rate"].mode()
+best_activation = best_models["Activation"].mode()
+best_layers = best_models["Layers"].mode()
+print(best_models["Layers"])
+print(best_models["Layers"].nsmallest())
+
+print("Best Models: \n")
+print(best_models)
+print("Batch Size: " + str(best_batch_size))
+print("Learning Rate: " + str(best_learning_rate))
+print("Activation: " + str(best_activation))
+print("Layers: " + str(best_layers))
+
+plt.figure(1)
+plt.plot(df["Learning_Rate"].tolist(), df["Error"].tolist(), 'bo')
+plt.xlabel("Learning Rate")
+plt.ylabel("Error")
+plt.title("Learning Rate vs. Error")
+plt.show()
+
+plt.figure(2)
+plt.plot(df["Batch_Size"].tolist(), df["Error"].tolist(), 'bo')
+plt.xlabel("Batch Size")
+plt.ylabel("Error")
+plt.title("Batch Size vs. Error")
+plt.show()
+
+plt.figure(3)
+plt.plot(df["Layers"].tolist(), df["Error"].tolist(), 'bo')
+plt.xlabel("Layers")
+plt.ylabel("Error")
+plt.title("Layers vs. Error")
+plt.show()
+
+plt.figure(4)
+plt.plot(df["Activation"].tolist(), df["Error"].tolist(), 'bo')
+plt.xlabel("Activation")
+plt.ylabel("Error")
+plt.title("Activation vs. Error")
+plt.show()