xpcr/run_log_processing.py at main · raphischer/xpcr · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
import argparse
import os

import pandas as pd
import numpy as np

from strep.load_experiment_logs import assemble_database
from strep.util import format_software, format_hardware
from data_loader import subsampled_to_orig

PROPERTIES = {
    'meta': {
        'task': lambda log: log['directory_name'].split('_')[0],
        'dataset': lambda log: log['config']['dataset'] + '_' + str(log['config']['ds_seed']) if log['config']['ds_seed'] != -1 else log['config']['dataset'],
        'model': lambda log: log['config']['model'],
        'architecture': lambda log: format_hardware(log['execution_platform']['Processor']),
        'software': lambda log: format_software('GluonTS', log['requirements']),
    },

    'train': {
        'train_running_time': lambda log: log['emissions']['duration']['0'],
        'train_power_draw': lambda log: log['emissions']['energy_consumed']['0'] * 3.6e6
    },

    'infer': {
        'running_time': lambda log: log['emissions']['duration']['0'] / log['validation_results']['num_samples'],
        'power_draw': lambda log: log['emissions']['energy_consumed']['0'] * 3.6e6 / log['validation_results']['num_samples'],
        'RMSE': lambda log: log['validation_results']['metrics']['aggregated']['RMSE'],
        'MAPE': lambda log: log['validation_results']['metrics']['aggregated']['MAPE'],
        'MASE': lambda log: log['validation_results']['metrics']['aggregated']['MASE'],
        'parameters': lambda log: log['validation_results']['model']['params'],
        'fsize': lambda log: log['validation_results']['model']['fsize'],
    }
}

RES_DIR = 'results'
DB_COMPLETE = os.path.join(RES_DIR, 'logs.pkl') # later logs.pkl
DB_BL = os.path.join(RES_DIR, 'baselines.pkl')
DB_META = DB_COMPLETE.replace('.pkl', '_meta.pkl')

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    # data and model input
    parser.add_argument("--output-dir", default='/data/d1/xpcr/logs', type=str, help="path to saved outputs")
    parser.add_argument("--merged-dir", default='results/merged')

    args = parser.parse_args()

    basename = os.path.basename(args.output_dir)
    mergedir = os.path.join(args.merged_dir, basename)
    db_file = os.path.join(RES_DIR, f'{basename}.pkl')
    if not os.path.isdir(RES_DIR):
        os.makedirs(RES_DIR)
    if not os.path.isdir(mergedir):
        os.makedirs(mergedir)

    database = assemble_database(args.output_dir, mergedir, None, PROPERTIES)
    # database = pd.concat([pd.read_pickle(os.path.join(RES_DIR, f'{fname}.pkl')) for fname in ['autokeras', 'autosklearn', 'autogluon']])
    # database = pd.read_pickle(os.path.join(RES_DIR, 'logs.pkl'))
    database = database[~database['dataset'].str.contains('austra')] # ditch australian_electricity_demand_dataset
    # merge for having a single task
    merged_database = []
    for group_field_vals, data in database.groupby(['dataset', 'environment', 'model']):
        assert data.shape[0] < 3, "found too many values"
        data = data.sort_values('task')
        merged = data.bfill().head(1)
        merged.loc[:,'task'] = 'Train and Test'
        merged_database.append(merged)
    database = pd.concat(merged_database).reset_index(drop=True)
    # discard inf MASE values
    database.loc[np.isinf(database['MASE']),'MASE'] = database.loc[~np.isinf(database['MASE']),'MASE'].max()
    numeric = database.select_dtypes('number')
    if np.any(np.isinf(numeric)):
        print('WARNING - found inf values in DB!')
    if np.any(np.isnan(numeric)):
        print('WARNING - found nan values in DB!')
    database = database.dropna().reset_index(drop=True)
    database['environment'] = database['environment'].map(lambda env: env.split(' - ')[0])
    database['dataset_orig'] = database['dataset'].map(subsampled_to_orig) # retrieve original dataset from all subsampled versions
    database['configuration'] = database.aggregate(lambda row: ' - '.join([row['task'], row['dataset'], row['model']]), axis=1)
    database.to_pickle(db_file)

    # # load all dbs
    # dbs = []
    # for fname in sorted(os.listdir(DB_DIR)):
    #     if '.pkl' in fname and fname not in [os.path.basename(fname) for fname in [DB_COMPLETE, DB_BL, DB_SUB]]:
    #         dbs.append( pd.read_pickle(os.path.join(DB_DIR, fname)) )
    #         # print(f'{fname:<20} no nan {str(dbs[-1].dropna().shape):<12} original {str(dbs[-1].shape):<12}')

    # # merge all dbs and do some small fixes
    # complete = pd.concat(dbs)
    # complete['parameters'] = complete['parameters'].fillna(0) # nan parameters are okay (occurs for PFN)
    # complete = complete.dropna().reset_index(drop=True) # get rid of failed PFN evals
    # # for some weird AGL experiments, codecarbon logged extreeemely high and unreasonable consumed energy (in the thousands and even millions of Watt)
    # # we discard these outliers (assuming a max draw of 400 Watt) and do a simple kNN gap filling
    # complete.loc[complete['train_power_draw'] / complete['train_running_time'] > 400,'train_power_draw'] = np.nan
    # complete.loc[complete['power_draw'] / complete['running_time'] > 400,'power_draw'] = np.nan
    # imputer = KNNImputer(n_neighbors=10, weights="uniform")
    # numeric = complete.select_dtypes('number').columns
    # complete.loc[:,numeric] = imputer.fit_transform(complete[numeric])

    # # split into metaqure, baselines and subset
    # baselines = complete[complete['model'].isin(['PFN', 'AGL', 'NAM', 'PFN4', 'PFN16', 'PFN64', 'PFN32'])]
    # complete = complete.drop(baselines.index, axis=0)
    # baselines.reset_index(drop=True).to_pickle(DB_BL)
    # complete.reset_index(drop=True).to_pickle(DB_COMPLETE)
    # subset = complete[complete['dataset'].isin(pd.unique(complete['dataset'])[5:15].tolist() + ['credit-g'])]
    # subset.reset_index(drop=True).to_pickle(DB_SUB)


    # # print some statistics
    # for env, data in complete.groupby('environment'):
    #     print(env)
    #     print('  complete:', data.dropna().shape)
    #     print('  baselines:', baselines[baselines['environment'] == env].dropna().shape)
    #     print(f'  time per baseline: {sum(budgets[env.split(" - ")[0]].values())}')