fix NC scalability

Author: yh-yao

.gitignore

@@ -59,7 +59,7 @@ cover/
 *.pot
 
 # Django stuff:
-*.log
+#*.log
 *.csv
 local_settings.py
 db.sqlite3

benchmark/benchmark_NC_Distributed-PyG.py

@@ -10,10 +10,8 @@
 
 # Distributed PyG imports
 from torch_geometric.loader import NeighborLoader
-from torch_geometric.nn.models import GCN as PyGGCN
 from torch.distributed import init_process_group, destroy_process_group
 from torch.nn.parallel import DistributedDataParallel as DDP
-import torch.multiprocessing as mp
 import os
 
 DATASETS = ['cora', 'citeseer', 'pubmed']

benchmark/figure/NC_comm_costs/NC10.log

@@ -13,29 +13,25 @@
 sns.set_context("talk")
 
 
-def extract_nc_scalability_data(logfile):
-    """Extract training and communication time data from NC log files"""
+def extract_nc_scalability_data(logfile, expected_trainers=None):
     with open(logfile, "r", encoding="utf-8", errors="replace") as f:
         log_content = f.read()
 
     results = []
 
-    # Find CSV FORMAT RESULT sections
     csv_sections = re.findall(
-        r"CSV FORMAT RESULT:.*?DS,IID,BS,Time\[s\],FinalAcc\[%\],CompTime\[s\],CommCost\[MB\],PeakMem\[MB\],AvgRoundTime\[s\],ModelSize\[MB\],TotalParams\n(.*?)\n",
+        r"CSV FORMAT RESULT:.*?DS,IID,BS,TotalTime\[s\],PureTrainingTime\[s\],CommTime\[s\],FinalAcc\[%\],CommCost\[MB\],PeakMem\[MB\],AvgRoundTime\[s\],ModelSize\[MB\],TotalParams\n(.*?)\n",
         log_content,
         re.DOTALL,
     )
 
-    # Extract number of trainers from experiment configuration
     trainer_matches = re.findall(r"Trainers: (\d+)", log_content)
 
     for csv_idx, csv_line in enumerate(csv_sections):
         parts = csv_line.strip().split(",")
-        if len(parts) >= 11:
+        if len(parts) >= 12:
             try:
-                # Get number of trainers for this experiment
-                num_trainers = (
+                num_trainers = expected_trainers if expected_trainers else (
                     int(trainer_matches[csv_idx])
                     if csv_idx < len(trainer_matches)
                     else 10
@@ -44,17 +40,16 @@ def extract_nc_scalability_data(logfile):
                 result = {
                     "Dataset": parts[0],
                     "IID_Beta": float(parts[1]),
-                    "Batch_Size": int(parts[2]),
+                    "Batch_Size": int(parts[2]) if parts[2] != '-1' else -1,
                     "Total_Time": float(parts[3]),
-                    "Final_Accuracy": float(parts[4]),
-                    "Training_Time": float(parts[5]),  # CompTime[s]
-                    "Communication_Cost": float(
-                        parts[6]
-                    ),  # CommCost[MB] - will convert to time
-                    "Peak_Memory": float(parts[7]),
-                    "Avg_Round_Time": float(parts[8]),
-                    "Model_Size": float(parts[9]),
-                    "Total_Params": int(float(parts[10])),
+                    "Training_Time": float(parts[4]),
+                    "Communication_Time": float(parts[5]),
+                    "Final_Accuracy": float(parts[6]),
+                    "Communication_Cost": float(parts[7]),
+                    "Peak_Memory": float(parts[8]),
+                    "Avg_Round_Time": float(parts[9]),
+                    "Model_Size": float(parts[10]),
+                    "Total_Params": int(float(parts[11])),
                     "Num_Trainers": num_trainers,
                 }
                 results.append(result)
@@ -64,90 +59,54 @@ def extract_nc_scalability_data(logfile):
     return pd.DataFrame(results)
 
 
-def estimate_communication_time(comm_cost_mb, num_trainers):
-    """Estimate communication time based on communication cost and network assumptions"""
-    # Assume network bandwidth: 100 Mbps = 12.5 MB/s
-    # This is a reasonable assumption for federated learning scenarios
-    network_bandwidth_mbps = 100 / 8  # Convert to MB/s
-
-    # Communication time = Total communication cost / bandwidth
-    comm_time = comm_cost_mb / network_bandwidth_mbps
-
-    return comm_time
-
-
 def load_all_nc_logs():
-    """Load data from all NC log files"""
-    log_files = ["NC.log", "NC5.log", "NC10.log", "NC20.log", "NC40.log"]
-    trainer_counts = [10, 5, 10, 20, 40]  # Default mapping
+    log_files = ["NC5.log", "NC10.log", "NC15.log", "NC20.log"]
+    trainer_counts = [5, 10, 15, 20]
 
     all_data = []
 
-    for log_file, default_trainers in zip(log_files, trainer_counts):
+    for log_file, expected_trainers in zip(log_files, trainer_counts):
         if os.path.exists(log_file):
-            df = extract_nc_scalability_data(log_file)
+            df = extract_nc_scalability_data(log_file, expected_trainers)
             if not df.empty:
-                # If trainer count not detected, use default
-                if "Num_Trainers" not in df.columns or df["Num_Trainers"].isna().all():
-                    df["Num_Trainers"] = default_trainers
+                df["Num_Trainers"] = expected_trainers
                 all_data.append(df)
-                print(
-                    f"Loaded {len(df)} records from {log_file} (Trainers: {default_trainers})"
-                )
 
     if all_data:
         combined_df = pd.concat(all_data, ignore_index=True)
         return combined_df
     else:
-        print("No NC log files found")
         return pd.DataFrame()
 
 
 def create_scalability_plot(df):
-    """Create scalability plot showing training time and communication time vs number of clients"""
-
     if df.empty:
-        print("No data available for plotting")
         return
 
-    # Filter for IID_Beta = 10.0 (as specified in your benchmark)
     df_filtered = df[df["IID_Beta"] == 10.0].copy()
 
     if df_filtered.empty:
-        print("No data found for IID_Beta = 10.0")
         return
 
-    # Add estimated communication time
-    df_filtered["Communication_Time"] = df_filtered.apply(
-        lambda row: estimate_communication_time(
-            row["Communication_Cost"], row["Num_Trainers"]
-        ),
-        axis=1,
-    )
-
-    # Group by number of trainers and calculate average times
     scalability_data = (
         df_filtered.groupby("Num_Trainers")
         .agg(
             {
                 "Training_Time": "mean",
                 "Communication_Time": "mean",
                 "Total_Time": "mean",
+                "Final_Accuracy": "mean",
+                "Communication_Cost": "mean",
+                "Peak_Memory": "mean",
             }
         )
         .reset_index()
     )
 
-    # Sort by number of trainers
     scalability_data = scalability_data.sort_values("Num_Trainers")
 
-    print("Scalability Data Summary:")
-    print(scalability_data)
-
-    # Create the plot
     plt.figure(figsize=(12, 8))
 
-    # Plot training time
     plt.plot(
         scalability_data["Num_Trainers"],
         scalability_data["Training_Time"],
@@ -158,7 +117,6 @@ def create_scalability_plot(df):
         label="Training Time",
     )
 
-    # Plot communication time
     plt.plot(
         scalability_data["Num_Trainers"],
         scalability_data["Communication_Time"],
@@ -169,7 +127,6 @@ def create_scalability_plot(df):
         label="Communication Time",
     )
 
-    # Add value labels on points
     for _, row in scalability_data.iterrows():
         plt.annotate(
             f'{row["Training_Time"]:.1f}s',
@@ -191,19 +148,16 @@ def create_scalability_plot(df):
             color="#ff7f0e",
         )
 
-    # Customize plot
     plt.xlabel("Number of Clients", fontsize=16)
     plt.ylabel("Time (seconds)", fontsize=16)
     plt.title("Federated Learning Scalability Analysis", fontsize=18, fontweight="bold")
     plt.legend(fontsize=14, loc="upper left")
     plt.grid(True, alpha=0.3)
 
-    # Set x-axis to show all client numbers
     client_numbers = sorted(scalability_data["Num_Trainers"].unique())
     plt.xticks(client_numbers, fontsize=14)
     plt.yticks(fontsize=14)
 
-    # Add some padding to y-axis
     y_max = max(
         scalability_data["Training_Time"].max(),
         scalability_data["Communication_Time"].max(),
@@ -214,66 +168,15 @@ def create_scalability_plot(df):
     plt.savefig("federated_learning_scalability.pdf", dpi=300, bbox_inches="tight")
     plt.close()
 
-    print("Generated: federated_learning_scalability.pdf")
-
-    # Create additional analysis table
-    scalability_data["Training_Growth"] = (
-        scalability_data["Training_Time"] / scalability_data["Training_Time"].iloc[0]
-    )
-    scalability_data["Communication_Growth"] = (
-        scalability_data["Communication_Time"]
-        / scalability_data["Communication_Time"].iloc[0]
-    )
-
-    print(f"\n{'='*60}")
-    print("SCALABILITY ANALYSIS SUMMARY")
-    print("=" * 60)
-    print(
-        f"{'Clients':<8} {'Train Time':<12} {'Comm Time':<12} {'Train Growth':<13} {'Comm Growth':<12}"
-    )
-    print("-" * 60)
-
-    for _, row in scalability_data.iterrows():
-        print(
-            f"{row['Num_Trainers']:<8.0f} "
-            f"{row['Training_Time']:<12.1f} "
-            f"{row['Communication_Time']:<12.1f} "
-            f"{row['Training_Growth']:<13.2f}x "
-            f"{row['Communication_Growth']:<12.2f}x"
-        )
-
-    # Save detailed results
     scalability_data.to_csv("scalability_analysis.csv", index=False)
-    print(f"\nDetailed results saved to: scalability_analysis.csv")
 
 
 def main():
-    """Main function to analyze federated learning scalability"""
-    print("Loading federated learning scalability data...")
-
-    # Load all NC log data
     df = load_all_nc_logs()
-
-    if df.empty:
-        print("No data found. Please check if NC log files exist:")
-        print("- NC.log, NC5.log, NC10.log, NC20.log, NC40.log")
-        return
-
-    print(f"\nLoaded data summary:")
-    print(f"Total records: {len(df)}")
-    print(f"Client counts: {sorted(df['Num_Trainers'].unique())}")
-    print(f"Datasets: {list(df['Dataset'].unique())}")
-    print(f"IID Betas: {sorted(df['IID_Beta'].unique())}")
-
-    # Create scalability analysis
-    print("\nGenerating scalability analysis...")
-    create_scalability_plot(df)
-
-    print(f"\nScalability analysis completed!")
-    print("Generated files:")
-    print("- federated_learning_scalability.pdf")
-    print("- scalability_analysis.csv")
+    
+    if not df.empty:
+        create_scalability_plot(df)
 
 
 if __name__ == "__main__":
-    main()
+    main()

benchmark/figure/NC_comm_costs/NC15.log

@@ -1,5 +1,5 @@
-Num_Trainers,Training_Time,Communication_Time,Total_Time,Training_Growth,Communication_Growth
-5,71.67500000000001,20.494,106.875,1.0,1.0
-10,49.2,40.989999999999995,84.725,0.6864318102546215,2.0000975895384014
-20,48.9,81.64,87.9,0.6822462504359957,3.983604957548551
-40,77.30000000000001,251.90800000000002,113.5,1.0784792465992326,12.291792719820435
+Num_Trainers,Training_Time,Communication_Time,Total_Time,Final_Accuracy,Communication_Cost,Peak_Memory
+5,33.2,2.625,112.4,0.5725,256.175,750.325
+10,12.7,4.475,87.175,0.47250000000000003,512.375,752.075
+15,6.575,16.150000000000002,93.45,0.5425,768.55,735.95
+20,5.824999999999999,8.075,82.175,0.5425,973.4749999999999,713.675