diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 00000000..ff7d55bc --- /dev/null +++ b/.style.yapf @@ -0,0 +1,413 @@ +[style] +# Align closing bracket with visual indentation. +align_closing_bracket_with_visual_indent=True + +# Allow dictionary keys to exist on multiple lines. For example: +# +# x = { +# ('this is the first element of a tuple', +# 'this is the second element of a tuple'): +# value, +# } +allow_multiline_dictionary_keys=False + +# Allow lambdas to be formatted on more than one line. +allow_multiline_lambdas=False + +# Allow splitting before a default / named assignment in an argument list. +allow_split_before_default_or_named_assigns=True + +# Allow splits before the dictionary value. +allow_split_before_dict_value=True + +# Let spacing indicate operator precedence. For example: +# +# a = 1 * 2 + 3 / 4 +# b = 1 / 2 - 3 * 4 +# c = (1 + 2) * (3 - 4) +# d = (1 - 2) / (3 + 4) +# e = 1 * 2 - 3 +# f = 1 + 2 + 3 + 4 +# +# will be formatted as follows to indicate precedence: +# +# a = 1*2 + 3/4 +# b = 1/2 - 3*4 +# c = (1+2) * (3-4) +# d = (1-2) / (3+4) +# e = 1*2 - 3 +# f = 1 + 2 + 3 + 4 +# +arithmetic_precedence_indication=False + +# Number of blank lines surrounding top-level function and class +# definitions. +blank_lines_around_top_level_definition=2 + +# Number of blank lines between top-level imports and variable +# definitions. +blank_lines_between_top_level_imports_and_variables=1 + +# Insert a blank line before a class-level docstring. +blank_line_before_class_docstring=False + +# Insert a blank line before a module docstring. +blank_line_before_module_docstring=False + +# Insert a blank line before a 'def' or 'class' immediately nested +# within another 'def' or 'class'. For example: +# +# class Foo: +# # <------ this blank line +# def method(): +# pass +blank_line_before_nested_class_or_def=False + +# Do not split consecutive brackets. Only relevant when +# dedent_closing_brackets is set. For example: +# +# call_func_that_takes_a_dict( +# { +# 'key1': 'value1', +# 'key2': 'value2', +# } +# ) +# +# would reformat to: +# +# call_func_that_takes_a_dict({ +# 'key1': 'value1', +# 'key2': 'value2', +# }) +coalesce_brackets=False + +# The column limit. +column_limit=80 + +# The style for continuation alignment. Possible values are: +# +# - SPACE: Use spaces for continuation alignment. This is default behavior. +# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns +# (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs or +# CONTINUATION_INDENT_WIDTH spaces) for continuation alignment. +# - VALIGN-RIGHT: Vertically align continuation lines to multiple of +# INDENT_WIDTH columns. Slightly right (one tab or a few spaces) if +# cannot vertically align continuation lines with indent characters. +continuation_align_style=SPACE + +# Indent width used for line continuations. +continuation_indent_width=4 + +# Put closing brackets on a separate line, dedented, if the bracketed +# expression can't fit in a single line. Applies to all kinds of brackets, +# including function definitions and calls. For example: +# +# config = { +# 'key1': 'value1', +# 'key2': 'value2', +# } # <--- this bracket is dedented and on a separate line +# +# time_series = self.remote_client.query_entity_counters( +# entity='dev3246.region1', +# key='dns.query_latency_tcp', +# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), +# start_ts=now()-timedelta(days=3), +# end_ts=now(), +# ) # <--- this bracket is dedented and on a separate line +dedent_closing_brackets=True + +# Disable the heuristic which places each list element on a separate line +# if the list is comma-terminated. +# +# Note: The behavior of this flag changed in v0.40.3. Before, if this flag +# was true, we would split lists that contained a trailing comma or a +# comment. Now, we have a separate flag, `DISABLE_SPLIT_LIT_WITH_COMMENT`, +# that controls splitting when a list contains a comment. To get the old +# behavior, set both flags to true. More information in CHANGELOG.md. +disable_ending_comma_heuristic=False + +# +# Don't put every element on a new line within a list that contains +# interstitial comments. +disable_split_list_with_comment=False + +# Place each dictionary entry onto its own line. +each_dict_entry_on_separate_line=True + +# Require multiline dictionary even if it would normally fit on one line. +# For example: +# +# config = { +# 'key1': 'value1' +# } +force_multiline_dict=False + +# The regex for an i18n comment. The presence of this comment stops +# reformatting of that line, because the comments are required to be +# next to the string they translate. +i18n_comment= + +# The i18n function call names. The presence of this function stops +# reformattting on that line, because the string it has cannot be moved +# away from the i18n comment. +i18n_function_call= + +# Indent blank lines. +indent_blank_lines=False + +# Put closing brackets on a separate line, indented, if the bracketed +# expression can't fit in a single line. Applies to all kinds of brackets, +# including function definitions and calls. For example: +# +# config = { +# 'key1': 'value1', +# 'key2': 'value2', +# } # <--- this bracket is indented and on a separate line +# +# time_series = self.remote_client.query_entity_counters( +# entity='dev3246.region1', +# key='dns.query_latency_tcp', +# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), +# start_ts=now()-timedelta(days=3), +# end_ts=now(), +# ) # <--- this bracket is indented and on a separate line +indent_closing_brackets=False + +# Indent the dictionary value if it cannot fit on the same line as the +# dictionary key. For example: +# +# config = { +# 'key1': +# 'value1', +# 'key2': value1 + +# value2, +# } +indent_dictionary_value=False + +# The number of columns to use for indentation. +indent_width=4 + +# Join short lines into one line. E.g., single line 'if' statements. +join_multiple_lines=True + +# Do not include spaces around selected binary operators. For example: +# +# 1 + 2 * 3 - 4 / 5 +# +# will be formatted as follows when configured with "*,/": +# +# 1 + 2*3 - 4/5 +no_spaces_around_selected_binary_operators=False + +# Use spaces around default or named assigns. +spaces_around_default_or_named_assign=False + +# Adds a space after the opening '{' and before the ending '}' dict +# delimiters. +# +# {1: 2} +# +# will be formatted as: +# +# { 1: 2 } +spaces_around_dict_delimiters=False + +# Adds a space after the opening '[' and before the ending ']' list +# delimiters. +# +# [1, 2] +# +# will be formatted as: +# +# [ 1, 2 ] +spaces_around_list_delimiters=False + +# Use spaces around the power operator. +spaces_around_power_operator=False + +# Use spaces around the subscript / slice operator. For example: +# +# my_list[1 : 10 : 2] +spaces_around_subscript_colon=False + +# Adds a space after the opening '(' and before the ending ')' tuple +# delimiters. +# +# (1, 2, 3) +# +# will be formatted as: +# +# ( 1, 2, 3 ) +spaces_around_tuple_delimiters=False + +# The number of spaces required before a trailing comment. +# This can be a single value (representing the number of spaces +# before each trailing comment) or list of values (representing +# alignment column values; trailing comments within a block will +# be aligned to the first column value that is greater than the maximum +# line length within the block). For example: +# +# With spaces_before_comment=5: +# +# 1 + 1 # Adding values +# +# will be formatted as: +# +# 1 + 1 # Adding values <-- 5 spaces between the end of the +# # statement and comment +# +# With spaces_before_comment=15, 20: +# +# 1 + 1 # Adding values +# two + two # More adding +# +# longer_statement # This is a longer statement +# short # This is a shorter statement +# +# a_very_long_statement_that_extends_beyond_the_final_column # Comment +# short # This is a shorter statement +# +# will be formatted as: +# +# 1 + 1 # Adding values <-- end of line comments in block +# # aligned to col 15 +# two + two # More adding +# +# longer_statement # This is a longer statement <-- end of line +# # comments in block aligned to col 20 +# short # This is a shorter statement +# +# a_very_long_statement_that_extends_beyond_the_final_column # Comment <-- the end of line comments are aligned based on the line length +# short # This is a shorter statement +# +spaces_before_comment=2 + +# Insert a space between the ending comma and closing bracket of a list, +# etc. +space_between_ending_comma_and_closing_bracket=False + +# Use spaces inside brackets, braces, and parentheses. For example: +# +# method_call( 1 ) +# my_dict[ 3 ][ 1 ][ get_index( *args, **kwargs ) ] +# my_set = { 1, 2, 3 } +space_inside_brackets=False + +# Split before arguments. +split_all_comma_separated_values=False + +# Split before arguments, but do not split all subexpressions recursively +# (unless needed). +split_all_top_level_comma_separated_values=False + +# Split before arguments if the argument list is terminated by a +# comma. +split_arguments_when_comma_terminated=False + +# Set to True to prefer splitting before '+', '-', '*', '/', '//', or '@' +# rather than after. +split_before_arithmetic_operator=False + +# Set to True to prefer splitting before '&', '|' or '^' rather than +# after. +split_before_bitwise_operator=False + +# Split before the closing bracket if a list or dict literal doesn't fit on +# a single line. +split_before_closing_bracket=True + +# Split before a dictionary or set generator (comp_for). For example, note +# the split before the 'for': +# +# foo = { +# variable: 'Hello world, have a nice day!' +# for variable in bar if variable != 42 +# } +split_before_dict_set_generator=True + +# Split before the '.' if we need to split a longer expression: +# +# foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) +# +# would reformat to something like: +# +# foo = ('This is a really long string: {}, {}, {}, {}' +# .format(a, b, c, d)) +split_before_dot=False + +# Split after the opening paren which surrounds an expression if it doesn't +# fit on a single line. +split_before_expression_after_opening_paren=False + +# If an argument / parameter list is going to be split, then split before +# the first argument. +split_before_first_argument=False + +# Set to True to prefer splitting before 'and' or 'or' rather than +# after. +split_before_logical_operator=False + +# Split named assignments onto individual lines. +split_before_named_assigns=True + +# Set to True to split list comprehensions and generators that have +# non-trivial expressions and multiple clauses before each of these +# clauses. For example: +# +# result = [ +# a_long_var + 100 for a_long_var in xrange(1000) +# if a_long_var % 10] +# +# would reformat to something like: +# +# result = [ +# a_long_var + 100 +# for a_long_var in xrange(1000) +# if a_long_var % 10] +split_complex_comprehension=False + +# The penalty for splitting right after the opening bracket. +split_penalty_after_opening_bracket=300 + +# The penalty for splitting the line after a unary operator. +split_penalty_after_unary_operator=10000 + +# The penalty of splitting the line around the '+', '-', '*', '/', '//', +# `%`, and '@' operators. +split_penalty_arithmetic_operator=300 + +# The penalty for splitting right before an if expression. +split_penalty_before_if_expr=0 + +# The penalty of splitting the line around the '&', '|', and '^' operators. +split_penalty_bitwise_operator=300 + +# The penalty for splitting a list comprehension or generator +# expression. +split_penalty_comprehension=80 + +# The penalty for characters over the column limit. +split_penalty_excess_character=7000 + +# The penalty incurred by adding a line split to the logical line. The +# more line splits added the higher the penalty. +split_penalty_for_added_line_split=30 + +# The penalty of splitting a list of "import as" names. For example: +# +# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, +# long_argument_2, +# long_argument_3) +# +# would reformat to something like: +# +# from a_very_long_or_indented_module_name_yada_yad import ( +# long_argument_1, long_argument_2, long_argument_3) +split_penalty_import_names=0 + +# The penalty of splitting the line around the 'and' and 'or' operators. +split_penalty_logical_operator=300 + +# Use the Tab character for indentation. +use_tabs=False + diff --git a/regr_smlp/code/smlp_regr.py b/regr_smlp/code/smlp_regr.py index a78f6982..b58c6a60 100755 --- a/regr_smlp/code/smlp_regr.py +++ b/regr_smlp/code/smlp_regr.py @@ -8,7 +8,7 @@ from multiprocessing import Process, Queue, Lock from subprocess import Popen, check_output, PIPE -from time import time +from time import time import csv_comparator as csv_cmp from threading import Timer @@ -17,20 +17,31 @@ TUI_DIFF = 'diff' GUI_DIFF = 'tkdiff' -TREE_PATH = '../' # Path to regression location (where data, code, specs, master and model directories are located) -SOLVERS_PATH = '../../../external' # Path to external solvers - +# Path to regression location (where data, code, specs, master and model +# directories are located) +TREE_PATH = '../' +SOLVERS_PATH = '../../../external' # Path to external solvers + DEBUG = False # used for excluding from diff reports that involve randomness -files_to_ignore_from_diff = ['Test41_doe_two_levels_doe.csv', 'Test42_doe_two_levels_doe.csv'] +files_to_ignore_from_diff = [ + 'Test41_doe_two_levels_doe.csv', 'Test42_doe_two_levels_doe.csv' +] + +RELEASE = False # to run regression with SMLP from release area + -RELEASE = False # to run regression with SMLP from release area def ignored_files(src, filenames): """ - Copy code and regression scripts required to run regression from a different directory + Copy code and regression scripts required to run regression from a different + directory """ - return [filename for filename in filenames if - not (filename.endswith('.py') or filename == "tests.csv" or filename.endswith('.exe') or filename.endswith('.json'))] + return [ + filename for filename in filenames if not ( + filename.endswith('.py') or filename == "tests.csv" or + filename.endswith('.exe') or filename.endswith('.json') + ) + ] def get_all_files_from_dir(dir_path): @@ -64,6 +75,7 @@ def get_conf_name(switches): def get_conf_path(conf, path1): return path.join(path1, conf) + # This function works both with json and txt config files def get_switches_with_conf(switches, path1): i = switches.find('-config ') @@ -73,7 +85,7 @@ def get_switches_with_conf(switches, path1): conf = sub_switches[:j] else: conf = sub_switches - + name_len = len(conf) conf = path.join(path1, conf) @@ -100,15 +112,19 @@ def mode_identifier(switches): mode = switches.find('--mode ') if mode + 7 > len(switches) - 1: return 'no mode' - mode_prefix3 = switches[mode + 7:mode + 10]; #print('mode_prefix3', mode_prefix3) - mode_prefix4 = switches[mode + 7:mode + 11]; #print('mode_prefix4', mode_prefix4); + mode_prefix3 = switches[mode + 7:mode + 10] + #print('mode_prefix3', mode_prefix3) + mode_prefix4 = switches[mode + 7:mode + 11] + #print('mode_prefix4', mode_prefix4); mode = switches[mode + 7] else: mode = switches.find('-mode ') if mode + 6 > len(switches) - 1: return 'no mode' - mode_prefix3 = switches[mode + 6:mode + 9]; #print('mode_prefix3', #mode_prefix3) - mode_prefix4 = switches[mode + 6:mode + 10]; #print('mode_prefix4', #mode_prefix4) + mode_prefix3 = switches[mode + 6:mode + 9] + #print('mode_prefix3', #mode_prefix3) + mode_prefix4 = switches[mode + 6:mode + 10] + #print('mode_prefix4', #mode_prefix4) mode = switches[mode + 6] #print('mode', mode, 'mode_prefix3', mode_prefix3, 'mode_prefix4', mode_prefix4) if mode == 's': @@ -120,7 +136,7 @@ def mode_identifier(switches): if mode_prefix3.startswith('tra'): return 'train' elif mode_prefix3.startswith('tun'): - print('mode tune was renamed'); + print('mode tune was renamed') assert False return 'tune' else: @@ -142,7 +158,7 @@ def mode_identifier(switches): elif mode_prefix4.startswith('opts'): return 'optsyn' else: - print('unknown mode prefix', mode_prefix4); + print('unknown mode prefix', mode_prefix4) assert False elif mode == 'v': return 'verify' @@ -154,7 +170,7 @@ def mode_identifier(switches): elif mode_prefix4 == "corr": return 'correlate' else: - print('unknown mode prefix', mode_prefix4); + print('unknown mode prefix', mode_prefix4) assert False elif mode == 'l': return 'level' @@ -174,7 +190,8 @@ def mode_identifier(switches): return 'unknown' else: return 'no mode' - + + def spec_identifier(switches): if '-spec' in switches or '--spec' in switches: #print('spec', switches.find('-spec '), switches.find('--spec ')) @@ -184,89 +201,106 @@ def spec_identifier(switches): return 'no spec' spec_id = '--spec ' else: - spec = switches.find('-spec '); + spec = switches.find('-spec ') spec_id = '-spec ' #print('spec', spec, 'spec_id', spec_id) - sub_switches = switches[spec + len(spec_id)-1:].strip(); #print('sub_switches', sub_switches) - i = sub_switches.find(' -'); #print('i', i) + sub_switches = switches[spec + len(spec_id) - 1:].strip() + #print('sub_switches', sub_switches) + i = sub_switches.find(' -') + #print('i', i) if i != -1: return sub_switches[:i] else: return sub_switches + def solver_path_identifier(switches): option_short = '-solver_path ' option_full = '--solver_path ' if option_short in switches or option_full in switches: #print('solver', switches.find(option_short), switches.find(option_full)) if option_full in switches: - solver = switches.find(option_full); #print('solver', solver, len(option_full), len(switches)) + solver = switches.find(option_full) + #print('solver', solver, len(option_full), len(switches)) if solver + len(option_full) > len(switches) - 1: return 'no solver' - #solver_prefix3 = switches[solver + len(option_full):solver + len(option_full)+3]; - solver = switches[solver + len(option_full)]; # - sub_switches = switches[solver + (len((option_short))-1):].strip() + #solver_prefix3 = switches[solver + len(option_full):solver + len(option_full)+3]; + solver = switches[solver + len(option_full)] + # + sub_switches = switches[solver + (len((option_short)) - 1):].strip() else: - solver = switches.find(option_short); - sub_switches = switches[solver + (len((option_short))-1):].strip() + solver = switches.find(option_short) + sub_switches = switches[solver + (len((option_short)) - 1):].strip() #print('solver', solver); print('sub_switches', sub_switches) - i = sub_switches.find(' -'); #print('i', i) + i = sub_switches.find(' -') + #print('i', i) if i != -1: return sub_switches[:i] else: return sub_switches + def model_algo_identifier(switches): # return '-use_model' in switches if not ('-model' in switches or '--model ' in switches): - return None + return None if '--model' in switches: - model_algo_loc = switches.find('--model ') - model_algo_pref8 = switches[model_algo_loc + 8:model_algo_loc + 14] + model_algo_loc = switches.find('--model ') + model_algo_pref8 = switches[model_algo_loc + 8:model_algo_loc + 14] else: - model_algo_loc = switches.find('-model ') - model_algo_pref8 = switches[model_algo_loc + 7:model_algo_loc + 13] - - for algo in ['dt_sklearn', 'rf_sklearn', 'et_sklearn', 'dt_caret', 'rf_caret', 'et_caret', 'nn_keras', 'poly_sklearn', 'system']: + model_algo_loc = switches.find('-model ') + model_algo_pref8 = switches[model_algo_loc + 7:model_algo_loc + 13] + + for algo in [ + 'dt_sklearn', 'rf_sklearn', 'et_sklearn', 'dt_caret', 'rf_caret', + 'et_caret', 'nn_keras', 'poly_sklearn', 'system' + ]: #print('algo', algo, 'pref', model_algo_pref8) if algo.startswith(model_algo_pref8): return algo - raise Exception('Failed to infer model algo from prefix ' + str(model_algo_pref8)) + raise Exception( + 'Failed to infer model algo from prefix ' + str(model_algo_pref8) + ) + def use_model_identifier(switches): # return '-use_model' in switches if not ('-use_model' in switches or '--use_model ' in switches): - return False + return False if '--use_model' in switches: - use_model = switches[switches.find('--use_model ') + len('--use_model ')] + use_model = switches[switches.find('--use_model ') + + len('--use_model ')] else: - use_model = switches[ switches.find('-use_model ') + len('-use_model ')] - + use_model = switches[switches.find('-use_model ') + len('-use_model ')] + if use_model.lower().startswith('t'): - return True + return True elif use_model.lower().startswith('f'): - return False + return False else: - raise Exception('use_model option value cannot be identified') + raise Exception('use_model option value cannot be identified') + - def save_model_identifier(switches): #return '-save_model' in switches if not ('-save_model' in switches or '--save_model ' in switches): - return False + return False if '--save_model ' in switches: - save_model = switches[switches.find('--save_model ') + len('--save_model ')] + save_model = switches[switches.find('--save_model ') + + len('--save_model ')] elif '-save_model ' in switches: - save_model = switches[ switches.find('-save_model ') + len('-save_model ')] + save_model = switches[switches.find('-save_model ') + + len('-save_model ')] else: - save_model = 't' # the default value for svae_model - + save_model = 't' # the default value for svae_model + if save_model.lower().startswith('t'): - return True + return True elif save_model.lower().startswith('f'): - return False + return False else: - raise Exception('save_model option value cannot be identified') + raise Exception('save_model option value cannot be identified') + def get_model_name(switches): i = switches.find('-model_name ') @@ -277,14 +311,20 @@ def get_model_name(switches): else: return sub_switches + # This function was adpated to work with json config files def use_model_in_config(conf): with open(conf, 'r') as c: lines = c.readlines() for line in lines: ln = line.lower() - #if '--use_saved_prediction_model true' in ln or '--use_saved_prediction_model t' in ln or '-use_model true' in ln or '-use_model t' in ln: - # to work with json config file we are are looking to match slightly different patterns + #if ( + # '--use_saved_prediction_model true' in ln or + # '--use_saved_prediction_model t' in ln or + # '-use_model true' in ln or '-use_model t' in ln + #): + # to work with json config file we are are looking to match slightly + # different patterns if '"use_model": "true"' in ln or '"use_model": "true"' in ln: return True return False @@ -296,37 +336,101 @@ def main(): # Regression arguments parser = ArgumentParser(description='SMLP regression') parser.add_argument('-o', '--output', help='Output directory.') - parser.add_argument('-t', '--tests', help='Specify tests to run. It can be a comma-separated list of test numbers\ - like -t 5,8,10; it can be a range of consecutive tests like -t 10:15; one can also run\ - all toy tests, where toy means that the test data name starts with smlp_toy, by specifying\ - -t toy; or run all other tests by specifying -t real; or run all the regression tests by\ - specifying -t all.') - parser.add_argument('-m', '--modes', help='Specify modes (e.g., verify) of tests to run, default is all modes.') - parser.add_argument('-models', '--models', help='Specify models (e.g., dt_sklearn) of tests to run, default is all modes.') - parser.add_argument('-extra', '--extra_options', help='Specify command line options that will be appended to the command line.') + parser.add_argument( + '-t', + '--tests', + help='Specify tests to run. It can be a comma-separated list of test ' + 'numbers like -t 5,8,10; it can be a range of consecutive tests like ' + '-t 10:15; one can also run all toy tests, where toy means that the ' + 'test data name starts with smlp_toy, by specifying -t toy; or run all ' + 'other tests by specifying -t real; or run all the regression tests by ' + 'specifying -t all.' + ) + parser.add_argument( + '-m', + '--modes', + help='Specify modes (e.g., verify) of tests to run, default is all ' + 'modes.' + ) + parser.add_argument( + '-models', + '--models', + help='Specify models (e.g., dt_sklearn) of tests to run, default is ' + 'all modes.' + ) + parser.add_argument( + '-extra', + '--extra_options', + help='Specify command line options that will be appended to the ' + 'command line.' + ) parser.add_argument('-d', '--debug', action='store_true') - parser.add_argument('-p', '--print_command', action='store_true', help='print the command to run manually;\ - the test will not be executed.') + parser.add_argument( + '-p', + '--print_command', + action='store_true', + help='print the command to run manually; the test will not be executed.' + ) parser.add_argument('-diff', '--diff', action='store_true') - #parser.add_argument('-c', '--cross_check', action='store_true', help='Cross check specific csv outputs.') - parser.add_argument('-w', '--workers', help='Number of concurrent tests that will run, default 2.') - #parser.add_argument('-temp', '--tempdir', help='Specify where to copy and run code, default=temp_dir.') - parser.add_argument('-i', '--ignore_tests', help='Ignores test/s that are passed as this argument.') - parser.add_argument('-n', '--no_all', action='store_true', help='Answer no to all file replacements/updates\ - when a mismatch is found between current and master results.') - parser.add_argument('-f', '--fail_txt', action='store_true', help='Don\'t compare all files if .txt main log\ - file comparison fails.') - parser.add_argument('-time', '--timeout', help='Set the timeout for each test to given value, if not provided,\ - no timeout.') - parser.add_argument('-tol', '--tolerance', help='Set the csv comparison tolerance to ignore differences in low\ - decimal bits.') - parser.add_argument('-def', '--default', help='Yes/No/Y/N answer to all master file replacements/updates.') - parser.add_argument('-conf', '--config_default', help='Yes/No/Y/N answer to config file all replacements/updates.') - parser.add_argument('-g', '--no_graphical_compare', action='store_true', help='Answer no on all replacing.') + #parser.add_argument('-c', '--cross_check', action='store_true', + # help='Cross check specific csv outputs.') + parser.add_argument( + '-w', + '--workers', + help='Number of concurrent tests that will run, default 2.' + ) + #parser.add_argument('-temp', '--tempdir', + # help='Specify where to copy and run code, default=temp_dir.') + parser.add_argument( + '-i', + '--ignore_tests', + help='Ignores test/s that are passed as this argument.' + ) + parser.add_argument( + '-n', + '--no_all', + action='store_true', + help='Answer no to all file replacements/updates when a mismatch is ' + 'found between current and master results.' + ) + parser.add_argument( + '-f', + '--fail_txt', + action='store_true', + help="Don't compare all files if .txt main log file comparison fails." + ) + parser.add_argument( + '-time', + '--timeout', + help='Set the timeout for each test to given value, if not provided, ' + 'no timeout.' + ) + parser.add_argument( + '-tol', + '--tolerance', + help='Set the csv comparison tolerance to ignore differences in low ' + 'decimal bits.' + ) + parser.add_argument( + '-def', + '--default', + help='Yes/No/Y/N answer to all master file replacements/updates.' + ) + parser.add_argument( + '-conf', + '--config_default', + help='Yes/No/Y/N answer to config file all replacements/updates.' + ) + parser.add_argument( + '-g', + '--no_graphical_compare', + action='store_true', + help='Answer no on all replacing.' + ) args = parser.parse_args() if not args.output: - output_path = './' #file_path.replace('\\', '/') + output_path = './' #file_path.replace('\\', '/') else: output_path = args.output.replace('\\', '/') if not args.tests: @@ -340,24 +444,33 @@ def main(): ignored_tests = [] if args.ignore_tests: if ',' in args.ignore_tests: - ignored_tests = args.ignore_tests.replace(" ", "").replace("\'", "").split(',') + ignored_tests = args.ignore_tests.replace(" ", + "").replace("\'", "" + ).split(',') else: - ignored_tests.append(args.ignore_tests.replace(" ", "").replace("\'", "")) + ignored_tests.append( + args.ignore_tests.replace(" ", "").replace("\'", "") + ) #print('ignored_tests', ignored_tests); relevant_modes = [] if args.modes: if ',' in args.modes: - relevant_modes = args.modes.replace(" ", "").replace("\'", "").split(',') + relevant_modes = args.modes.replace(" ", "").replace("\'", + "").split(',') else: - relevant_modes.append(args.modes.replace(" ", "").replace("\'", "")) + relevant_modes.append(args.modes.replace(" ", "").replace("\'", "")) #print('relevant_modes',relevant_modes); - + relevant_models = [] if args.models: if ',' in args.models: - relevant_models = args.models.replace(" ", "").replace("\'", "").split(',') + relevant_models = args.models.replace(" ", + "").replace("\'", + "").split(',') else: - relevant_models.append(args.models.replace(" ", "").replace("\'", "")) + relevant_models.append( + args.models.replace(" ", "").replace("\'", "") + ) #print('relevant_models',relevant_models) """def read_txt_file_to_list(file_path): with open(file_path, 'r') as rFile: @@ -369,11 +482,13 @@ def main(): else: DIFF = TUI_DIFF - code_path = file_path # Path to SMLP regression code - also where smlp_tests.csv file and this script are located. + # Path to SMLP regression code - also where smlp_tests.csv file and this + # script are located. + code_path = file_path - # Create and migrate code to temp dir - if False: # currently use development code w/o copying to temp area #not args.print_command: + if False: #not args.print_command: + # currently use development code w/o copying to temp area tempdir = 'temp' if args.tempdir: tempdir += args.tempdir @@ -381,26 +496,36 @@ def main(): tempdir += tests else: tempdir = 'temp_code4' - temp_code_dir = path.join(TREE_PATH, tempdir) # Path of temp copied code dir. + # Path of temp copied code dir. + temp_code_dir = path.join(TREE_PATH, tempdir) if path.exists(temp_code_dir): rmtree(temp_code_dir) - copytree(dst=temp_code_dir, src=code_path, ignore=ignored_files) # Copies code to temp dir. + # Copies code to temp dir. + copytree(dst=temp_code_dir, src=code_path, ignore=ignored_files) chdir(temp_code_dir) # Changes working dir to temp code dir. else: temp_code_dir = code_path - master_path = path.join(TREE_PATH, 'master') # Path to master results (to compare with) - models_path = path.join(TREE_PATH, 'models') # Path to saved models and everything required to re-run it - data_path = path.join(TREE_PATH, 'data') # Path to the data - doe_path = path.join(TREE_PATH, 'grids') # Path to the doe grids data - specs_path = path.join(TREE_PATH, 'specs') # Path to the domain spec for model exploration - tests_data = path.join(temp_code_dir, 'smlp_regr.csv') # Path of the tests config file - - + + # Path to master results (to compare with) + master_path = path.join(TREE_PATH, 'master') + + # Path to saved models and everything required to re-run it + models_path = path.join(TREE_PATH, 'models') + + data_path = path.join(TREE_PATH, 'data') # Path to the data + doe_path = path.join(TREE_PATH, 'grids') # Path to the doe grids data + + # Path to the domain spec for model exploration + specs_path = path.join(TREE_PATH, 'specs') + + # Path of the tests config file + tests_data = path.join(temp_code_dir, 'smlp_regr.csv') + diff = 'diff' if args.tolerance: csv_cmp.set_threshold(int(args.tolerance)) - + tests_list = [] tests_queue = Queue() print_lock = Lock() @@ -418,11 +543,18 @@ def main(): csvreader = reader(rFile, delimiter=',') next(csvreader, None) for row in csvreader: - if (row[1].startswith('smlp_toy') or row[1].startswith('mlbt_toy') or row[2].startswith('smlp_toy') or row[ - 2].startswith('mlbt_toy') or ( - conf_identifier(row[3]) and get_conf_name(row[3]).startswith('smlp_toy')) or ( - not conf_identifier(row[3]) and row[1] == '' and row[2] == '')) and ( - row[0] not in ignored_tests): + if ( + row[1].startswith('smlp_toy') or + row[1].startswith('mlbt_toy') or + row[2].startswith('smlp_toy') or + row[2].startswith('mlbt_toy') or ( + conf_identifier(row[3]) and + get_conf_name(row[3]).startswith('smlp_toy') + ) or ( + not conf_identifier(row[3]) and row[1] == '' and + row[2] == '' + ) + ) and (row[0] not in ignored_tests): tests_list.append(row[0]) tests_queue.put(row) elif tests == 'real': @@ -430,10 +562,17 @@ def main(): csvreader = reader(rFile, delimiter=',') next(csvreader, None) for row in csvreader: - if (not (row[1].startswith('smlp_toy') or row[1].startswith('mlbt_toy') or row[2].startswith('smlp_toy') or - row[2].startswith('mlbt_toy') or ( - conf_identifier(row[3]) and get_conf_name(row[3]).startswith('smlp_toy')))) and ( - row[0] not in ignored_tests): + if ( + not ( + row[1].startswith('smlp_toy') or + row[1].startswith('mlbt_toy') or + row[2].startswith('smlp_toy') or + row[2].startswith('mlbt_toy') or ( + conf_identifier(row[3]) and + get_conf_name(row[3]).startswith('smlp_toy') + ) + ) + ) and (row[0] not in ignored_tests): tests_list.append(row) tests_queue.put(row) elif tests == 'test': @@ -448,10 +587,14 @@ def main(): elif ',' in tests: t_list = tests.split(',') for e in t_list: - if ':' in e: # this option to support tests range, eg: 5:10 + if ':' in e: # this option to support tests range, eg: 5:10 t_list.remove(e) e_range = e.split(':') - t_list = t_list + [str(e) for e in list(range(int(e_range[0]), int(e_range[1])+1))] + t_list = t_list + [ + str(e) + for e in list(range(int(e_range[0]), + int(e_range[1]) + 1)) + ] #print('t_list', t_list) with open(tests_data, 'r') as rFile: csvreader = reader(rFile, delimiter=',') @@ -460,7 +603,7 @@ def main(): if row[0] in t_list: tests_list.append(row) tests_queue.put(row) - elif ':' in tests: # this option to support tests range, eg: 5:10 + elif ':' in tests: # this option to support tests range, eg: 5:10 t_range = tests.split(':') start = t_range[0] end = t_range[1] @@ -476,12 +619,8 @@ def main(): tests_queue.put(row) else: #print('tests', tests, 'tests_data', tests_data) - tests_list.append(fetch_test(tests,tests_data)) - tests_queue.put(fetch_test(tests,tests_data)) - - - - + tests_list.append(fetch_test(tests, tests_data)) + tests_queue.put(fetch_test(tests, tests_data)) """def fetch_test_outputs(test_id): test = fetch_test(test_id) new_prefix = 'Test' + test_id @@ -490,7 +629,6 @@ def main(): test_switches = test[3] test_type = mode_identifier(test_switches) return test_outputs(test_id, new_prefix, test_data, test_new_data, test_type, test_switches)""" - ''' Indexes for test list: 0 - id @@ -507,7 +645,14 @@ def kill_process(pr): pr.kill() def popen_timeout(command, timeout): - p = Popen(command,shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True) + p = Popen( + command, + shell=True, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True + ) my_timer = Timer(timeout, kill_process, [p]) cm = False try: @@ -523,15 +668,22 @@ def worker(q, id_q, print_l): return True test = q.get() test_id = test[0] - test_data = test[1]; #print('test_data', test_data) - test_new_data = test[2]; #print('test_new_data', test_new_data) - test_switches = test[3]; #print('test_switvhed', test_switches) + test_data = test[1] + #print('test_data', test_data) + test_new_data = test[2] + #print('test_new_data', test_new_data) + test_switches = test[3] + #print('test_switvhed', test_switches) test_description = test[4] - use_model = use_model_identifier(test_switches); #print('use_model', use_model) - save_model = save_model_identifier(test_switches); #print('save_model', save_model) - test_type = mode_identifier(test_switches); #print('test_type', test_type) - model_algo = model_algo_identifier(test_switches); #print('model_algo', model_algo) - + use_model = use_model_identifier(test_switches) + #print('use_model', use_model) + save_model = save_model_identifier(test_switches) + #print('save_model', save_model) + test_type = mode_identifier(test_switches) + #print('test_type', test_type) + model_algo = model_algo_identifier(test_switches) + #print('model_algo', model_algo) + if DEBUG: print('test_data', test_data) print('test_new_data', test_new_data) @@ -541,13 +693,16 @@ def worker(q, id_q, print_l): print('save_model', save_model) print('test_type', test_type) print('model_algo', model_algo) - - use_config_file = conf_identifier(test_switches); #print('use_config_file', use_config_file) + + use_config_file = conf_identifier(test_switches) + #print('use_config_file', use_config_file) #print('config file', get_conf_path(get_conf_name(test_switches), models_path)) if use_config_file: - use_model = use_model_in_config(get_conf_path(get_conf_name(test_switches), models_path)); + use_model = use_model_in_config( + get_conf_path(get_conf_name(test_switches), models_path) + ) #print('use_model updated', use_model) - + test_errors = [] model = False # flag if test uses model status = True # test run status @@ -564,16 +719,23 @@ def worker(q, id_q, print_l): test_type = 'help' if test_type == 'unknown' and not (use_config_file): execute_test = False - test_errors.append(['Build', 'Unknown mode or was not specified']) - + test_errors.append( + ['Build', 'Unknown mode or was not specified'] + ) + if DEBUG: - print("DEBUG 2"); + print("DEBUG 2") print('execute_test', execute_test) - + if execute_test: new_prefix = 'Test' + test_id - #print('test_data', test_data); print('use_model', use_model ); print('use_config_file', use_config_file) - if (test_data == '' and not use_model and not use_config_file and not '-doe_spec' in test_switches): + #print('test_data', test_data) + #print('use_model', use_model) + #print('use_config_file', use_config_file) + if ( + test_data == '' and not use_model and + not use_config_file and not '-doe_spec' in test_switches + ): if test_type != 'help': execute_test = False test_errors.append(['Build', 'No test data specified']) @@ -582,62 +744,98 @@ def worker(q, id_q, print_l): test_data = 'h' else: test_data = 'help' - test_out = path.join(output_path, new_prefix + '_' + test_data + '.txt') + test_out = path.join( + output_path, new_prefix + '_' + test_data + '.txt' + ) test_type = 'help' elif use_model: # model_name = path.join(data_path, test_data).replace('\\', '/') - model_name = path.join(models_path, test_data).replace('\\', '/'); #print('model_name', model_name) - test_data_path = '-model_name \"{0}\"'.format(model_name) # here we use a model instead of data + model_name = path.join(models_path, + test_data).replace('\\', '/') + #print('model_name', model_name) + test_data_path = '-model_name \"{0}\"'.format( + model_name + ) # here we use a model instead of data if DEBUG: print('model_name', model_name) else: if test_data != "": if test_type == 'doe': - test_data_path = path.join(doe_path, test_data).replace('\\', '/') - #print('test_data_path', test_data_path); print('test_data', test_data) + test_data_path = path.join(doe_path, test_data + ).replace('\\', '/') + #print('test_data_path', test_data_path) + #print('test_data', test_data) if path.exists(test_data_path + '.csv'): - test_data_path = '-doe_spec \"{0}.csv\"'.format(test_data_path) + test_data_path = '-doe_spec \"{0}.csv\"'.format( + test_data_path + ) else: execute_test = False - test_errors.append(['Build', 'DOE file does not exist']) + test_errors.append( + ['Build', 'DOE file does not exist'] + ) else: - test_data_path = path.join(data_path, test_data).replace('\\', '/') - #print('test_data_path', test_data_path); print('test_data', test_data) + test_data_path = path.join(data_path, test_data + ).replace('\\', '/') + #print('test_data_path', test_data_path) + #print('test_data', test_data) if path.exists(test_data_path): - test_data_path = '-data \"{0}\"'.format(test_data_path) + test_data_path = '-data \"{0}\"'.format( + test_data_path + ) elif path.exists(test_data_path + '.csv'): - test_data_path = '-data \"{0}.csv\"'.format(test_data_path) + test_data_path = '-data \"{0}.csv\"'.format( + test_data_path + ) else: execute_test = False - test_errors.append(['Build', 'Data file does not exist']) + test_errors.append( + ['Build', 'Data file does not exist'] + ) else: test_data_path = "" if DEBUG: - print('test_data', test_data) - print('test_new_data', test_new_data) - print('test_data_path', test_data_path) - print('use_config_file', use_config_file ) - print(test_new_data != "") - - if test_type == 'prediction' or (test_new_data != ""): #use_config_file and + print('test_data', test_data) + print('test_new_data', test_new_data) + print('test_data_path', test_data_path) + print('use_config_file', use_config_file) + print(test_new_data != "") + + if test_type == 'prediction' or ( + test_new_data != "" + ): #use_config_file and if not test_new_data == '': - test_new_data_path = path.join(data_path, test_new_data).replace('\\', '/') + test_new_data_path = path.join( + data_path, test_new_data + ).replace('\\', '/') if path.exists(test_new_data_path): - test_new_data_path = '-new_dat \"{0}\"'.format(test_new_data_path) + test_new_data_path = '-new_dat \"{0}\"'.format( + test_new_data_path + ) elif path.exists(test_new_data_path + '.csv'): - test_new_data_path = '-new_dat \"{0}.csv\"'.format(test_new_data_path) + test_new_data_path = '-new_dat \"{0}.csv\"'.format( + test_new_data_path + ) else: execute_test = False - test_errors.append(['Build', 'New data file does not exist']) + test_errors.append( + ['Build', 'New data file does not exist'] + ) else: execute_test = False - test_errors.append(['Build', 'No new data file specified']) - if len(relevant_modes) > 0 and (test_type not in relevant_modes+['no mode']): - execute_test = False - #print('(relevant_models)', relevant_models, 'model_algo', model_algo, flush=True); - if len(relevant_models) > 0 and (model_algo not in relevant_models): - execute_test = False - + test_errors.append( + ['Build', 'No new data file specified'] + ) + if len(relevant_modes) > 0 and ( + test_type not in relevant_modes + ['no mode'] + ): + execute_test = False + #print('(relevant_models)', relevant_models, + # 'model_algo', model_algo, flush=True); + if len(relevant_models + ) > 0 and (model_algo not in relevant_models): + execute_test = False + if DEBUG: print("DEBUG 3") print('execute_test', execute_test) @@ -645,70 +843,100 @@ def worker(q, id_q, print_l): if execute_test: if use_config_file: - test_switches = get_switches_with_conf(test_switches, models_path) + test_switches = get_switches_with_conf( + test_switches, models_path + ) if RELEASE: command = "../../src/run_smlp.py" else: command = "../../src/run_smlp.py" - if args.timeout: # timeout -- TODO !!! - command = '/usr/bin/timeout 600 ' + command + if args.timeout: # timeout -- TODO !!! + command = '/usr/bin/timeout 600 ' + command if DEBUG: - print('command (0)', command); print('test_type', test_type) + print('command (0)', command) + print('test_type', test_type) if test_type == 'help': - command += ' {args} > {output}'.format(args=test_switches, output=test_out) + command += ' {args} > {output}'.format( + args=test_switches, output=test_out + ) else: - if test_type in ['optimize', 'verify', 'query', 'optsyn', 'certify', 'synthesize', 'frontier']: + if test_type in [ + 'optimize', 'verify', 'query', 'optsyn', 'certify', + 'synthesize', 'frontier' + ]: # add relative path to spec file name - spec_fn = spec_identifier(test_switches)# + '.spec'; - print('spec_fn', spec_fn); print('specs_path', specs_path) + spec_fn = spec_identifier(test_switches) # + '.spec'; + print('spec_fn', spec_fn) + print('specs_path', specs_path) if spec_fn is not None: spec_file = os.path.join(specs_path, spec_fn) - test_switches = test_switches.replace(spec_fn, spec_file) + test_switches = test_switches.replace( + spec_fn, spec_file + ) else: - raise Exception('spec file must be specified in command line in model exploration modes') + raise Exception( + 'spec file must be specified in command line ' + 'in model exploration modes' + ) # add relative path to external solver name solver_bin = solver_path_identifier(test_switches) if solver_bin is not None: - solver_path_bin = os.path.join(SOLVERS_PATH, solver_bin) - test_switches = test_switches.replace(solver_bin, solver_path_bin) - #test_switches = test_switches.replace("-solver_path ", ' ').replace(solver_path_bin, ' ') + solver_path_bin = os.path.join( + SOLVERS_PATH, solver_bin + ) + test_switches = test_switches.replace( + solver_bin, solver_path_bin + ) + #test_switches = test_switches.replace("-solver_path ", ' ') + #test_switches = test_switches.replace(solver_path_bin, ' ') #print('test_switches', test_switches); print('test_type', test_type) - command += ' {dat} {out_dir} {pref} {args} {debug} '.format(dat=test_data_path, - out_dir='-out_dir {output_path}'.format( - output_path=output_path), - pref='-pref {prefix}'.format( - prefix=new_prefix), - args=test_switches, - debug=debug) + command += ' {dat} {out_dir} {pref} {args} {debug} '.format( + dat=test_data_path, + out_dir='-out_dir {output_path}'.format( + output_path=output_path + ), + pref='-pref {prefix}'.format(prefix=new_prefix), + args=test_switches, + debug=debug + ) if DEBUG: - print('command (1)', command); - #print('test_type', test_type, 'test_new_data',test_new_data) - if test_type == 'prediction' or (test_new_data != ""): #use_config_file and - command += '{new_dat} '.format(new_dat=test_new_data_path) + print('command (1)', command) + #print('test_type', test_type, 'test_new_data',test_new_data) + if test_type == 'prediction' or ( + test_new_data != "" + ): #use_config_file and + command += '{new_dat} '.format( + new_dat=test_new_data_path + ) #print('command (2)', command); - - # append extra arguments + + # append extra arguments if args.extra_options is not None: command = command + ' ' + args.extra_options - + if DEBUG: - print('command (2)', command); - + print('command (2)', command) + with print_l: - print("Running test {0} test type: {1}, description: {2}".format(test_id, test_type, - test_description)) + print( + "Running test {0} test type: {1}, description: {2}". + format(test_id, test_type, test_description) + ) print(command + '\n') if not args.print_command: if save_model: model = True - if False: #args.timeout: + if False: #args.timeout: pr = popen_timeout(command, int(args.timeout)) if pr: outs, errs = pr if args.debug: - print('Output: \n' + outs + '\n' + 'Errors: \n' + errs + '\n') + print( + 'Output: \n' + outs + '\n' + 'Errors: \n' + + errs + '\n' + ) if extract_smlp_error(errs) != 'OK': status = False test_errors.append(['Run', errs]) @@ -717,10 +945,20 @@ def worker(q, id_q, print_l): test_errors.append(['Run', 'Timeout']) execute_test = False else: - pr = Popen(command, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True) + pr = Popen( + command, + shell=True, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True + ) outs, errs = pr.communicate() if args.debug: - print('Output: \n' + outs + '\n' + 'Errors: \n' + errs + '\n') + print( + 'Output: \n' + outs + '\n' + 'Errors: \n' + + errs + '\n' + ) if extract_smlp_error(errs) != 'OK': status = False test_errors.append(['Run', errs]) @@ -730,7 +968,7 @@ def worker(q, id_q, print_l): if DEBUG: print("DEBUG 4") - + process_list = [] expected_outs = tests_queue.qsize() if args.workers: @@ -739,9 +977,15 @@ def worker(q, id_q, print_l): workers = 2 # Number of concurrent processes if tests_queue.qsize() < workers: workers = tests_queue.qsize() - print("Calling {workers} workers for multiprocessing...".format(workers=workers)) + print( + "Calling {workers} workers for multiprocessing...".format( + workers=workers + ) + ) for i in range(0, workers): - t = Process(target=worker, args=(tests_queue, test_out_queue, print_lock)) + t = Process( + target=worker, args=(tests_queue, test_out_queue, print_lock) + ) process_list.append(t) t.start() print("Initiating {i} worker...".format(i=i)) @@ -749,21 +993,27 @@ def worker(q, id_q, print_l): while counter < expected_outs: test_id_list.append(test_out_queue.get()) counter += 1 - + if DEBUG: print("DEBUG 5") - + for process in process_list: process.join() + # fixing output and master path for the system use: - master_path = master_path.replace('/', path.sep).replace('\"', '').replace('\'', '') - output_path = output_path.replace('/', path.sep).replace('\"', '').replace('\'', '') + # XXX fb: What exactly does "fix" mean here? '\"' and '\'' are perfectly + # valid symbols in a path. + def fix_path(p): + return p.replace('/', path.sep).replace('\"', '').replace('\'', '') + + master_path = fix_path(master_path) + output_path = fix_path(output_path) files_in_master = get_all_files_from_dir(master_path) files_in_output = get_all_files_from_dir(output_path) if DEBUG: print("DEBUG 6") - + def get_file_from_list_underscore(prefix_list, list1): outs_list = [] for file1 in list1: @@ -773,13 +1023,24 @@ def get_file_from_list_underscore(prefix_list, list1): return outs_list cross_tests = [] - new_error_ids = [] # test IDs of crashes reporrted in error files -- ones that should not happen / do not occur in masters - new_error_fns = [] # filenames of crashes reporrted in error files -- ones that should not happen / do not occur in masters - #missing_errors = [] # crashes that are expected / are part of masters but do not occur in current run -- not implemented yet + # test IDs of crashes reporrted in error files -- ones that should not + # happen / do not occur in masters + new_error_ids = [] + # filenames of crashes reporrted in error files -- ones that should not + # happen / do not occur in masters + new_error_fns = [] + + # crashes that are expected / are part of masters but do not occur in + # current run -- not implemented yet + #missing_errors = [] + # main log comparing def smlp_txt_file(fname): - if 'config' in fname or 'error' in fname or 'mrmr_features_summary' in fname or '_formula' in fname: + if ( + 'config' in fname or 'error' in fname or + 'mrmr_features_summary' in fname or '_formula' in fname + ): return False elif fname.endswith('.txt'): return True @@ -804,9 +1065,9 @@ def comapre_files(file1, file2): return False return True - log = tests in {'all', 'real', 'toy', 'test'} # to tell if there is a main log compare needed - master_log_file = path.join(master_path, - tests + '_log.txt') + # to tell if there is a main log compare needed + log = tests in {'all', 'real', 'toy', 'test'} + master_log_file = path.join(master_path, tests + '_log.txt') log_file = path.join(output_path, tests + '_log.txt') if DEBUG: @@ -827,10 +1088,10 @@ def get_id(l): if DEBUG: print("DEBUG 8") - print('args.print_command', args.print_command) - print('args.diff', args.diff) + print('args.print_command', args.print_command) + print('args.diff', args.diff) print('args.debug', args.debug) - + # sort test list # new_list = [] # new_dict = dict() @@ -846,8 +1107,12 @@ def get_id(l): output_prefixes = [test_prefix] if test_model: output_prefixes.append(test_model) - new_files = get_file_from_list_underscore(output_prefixes, files_in_output) - master_files = get_file_from_list_underscore(output_prefixes, files_in_master) + new_files = get_file_from_list_underscore( + output_prefixes, files_in_output + ) + master_files = get_file_from_list_underscore( + output_prefixes, files_in_master + ) ''' for d in range(0, len(double_tests) - 1): double = double_tests[d] @@ -881,42 +1146,79 @@ def get_id(l): if new_file.endswith('_plots'): if os.path.exists(master_file): assert master_file.endswith('_plots') - file_to_minitor = 'plotReport.html' + file_to_minitor = 'plotReport.html' new_file = os.path.join(new_file,) - master_file = os.path.join(master_file, file_to_minitor) - #print('dropping from master_files', file) + master_file = os.path.join( + master_file, file_to_minitor + ) + #print('dropping from master_files', file) master_files.remove(file) - file = os.path.join(file, file_to_minitor) + file = os.path.join(file, file_to_minitor) #print('appending to master files', file) master_files.append(file) - #print('update new_file', new_file); print('updated master file', master_file); - + #print('update new_file', new_file) + #print('updated master file', master_file) + file_name = file config_file = 'config' in file_name - # model_file = 'model' in file_name # if its a model file it needs to be replaced in data as well - model_file = file_name.startswith('test' + str(test_id) + '_model') + # if its a model file it needs to be replaced in data as well + # model_file = 'model' in file_name + model_file = file_name.startswith( + 'test' + str(test_id) + '_model' + ) txt_file = False if path.exists(master_file): - if new_file.endswith('.txt') and not config_file : + if new_file.endswith('.txt') and not config_file: txt_file = True - # condition before, dropping from it h5 file checks because getting UnicodeDecodeError error on Sles 15, say on Test 13. - # (new_file.endswith('.csv') or new_file.endswith('.txt') or new_file.endswith('.html') or new_file.endswith('.json') or new_file.endswith('.h5')) and not file_name in files_to_ignore_from_diff: + # condition before, dropping from it h5 file checks + # because getting UnicodeDecodeError error on Sles 15, + # say on Test 13. + # (new_file.endswith('.csv') or + # new_file.endswith('.txt') or + # new_file.endswith('.html') or + # new_file.endswith('.json') or + # new_file.endswith('.h5')) and + # not file_name in files_to_ignore_from_diff: exclude_cond = file_name in files_to_ignore_from_diff - exclude_cond = file_name in files_to_ignore_from_diff or file_name.endswith('_model_term.json') - if (new_file.endswith('.csv') or new_file.endswith('.txt') or new_file.endswith('.html') or new_file.endswith('.json')) and not exclude_cond: - print('comparing {file} to master'.format(file=file_name)) + exclude_cond = ( + file_name in files_to_ignore_from_diff or + file_name.endswith('_model_term.json') + ) + if ( + new_file.endswith('.csv') or + new_file.endswith('.txt') or + new_file.endswith('.html') or + new_file.endswith('.json') + ) and not exclude_cond: + print( + 'comparing {file} to master'.format( + file=file_name + ) + ) p = Popen( - '{diff} -B -I \'Feature selection.*file .*\' -I \'\\[-v-] Input.*\' -I \'usage:.*\' {k} {l}'.format( - diff=diff, - k=new_file, - l=master_file), - shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) + '{diff} -B -I \'Feature selection.*file .*\' ' + '-I \'\\[-v-] Input.*\' -I \'usage:.*\' {k} {l}' + .format(diff=diff, k=new_file, l=master_file), + shell=True, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE + ) output, error = p.communicate() if p.returncode == 1: if not comapre_files(new_file, master_file): if not args.no_graphical_compare and to_show: - Popen('{diff} {l} {k}'.format(diff=DIFF, k=new_file, l=master_file), shell=True).wait() - if args.default or (args.config_default and config_file): + Popen( + '{diff} {l} {k}'.format( + diff=DIFF, + k=new_file, + l=master_file + ), + shell=True + ).wait() + if args.default or ( + args.config_default and config_file + ): if args.config_default and config_file: user_input = args.config_default else: @@ -926,36 +1228,69 @@ def get_id(l): user_input = answer else: user_input = input( - 'Do you wish to switch the new file with the master?\n(yes/no|y/n): ').lower() - while user_input not in {'yes', 'no', 'y', 'n'}: - user_input = input('(yes/no|y/n):').lower() + 'Do you wish to switch the new ' + 'file with the master?\n' + '(yes/no|y/n): ' + ).lower() + while user_input not in { + 'yes', 'no', 'y', 'n' + }: + user_input = input('(yes/no|y/n):' + ).lower() if user_input in {'yes', 'y'}: if model_file or config_file: copyfile(new_file, master_file) - copyfile(new_file, path.join(models_path, file_name)) - print('Replacing Files both in master and data') + copyfile( + new_file, + path.join( + models_path, file_name + ) + ) + print( + 'Replacing Files both in ' + 'master and data' + ) else: copyfile(new_file, master_file) print('Replacing Files...') - if path.exists(path.join(data_path, file_name)): + if path.exists( + path.join(data_path, file_name) + ): if args.default: user_input = args.default else: user_input = input( - 'File exists also in data, switch there as well?\n(yes/no|y/n): ').lower() - while user_input not in {'yes', 'no', 'y', 'n'}: - user_input = input('(yes/no|y/n):').lower() + 'File exists also in ' + 'data, switch there as ' + 'well?\n' + '(yes/no|y/n): ' + ).lower() + while user_input not in { + 'yes', 'no', 'y', 'n' + }: + user_input = input( + '(yes/no|y/n):' + ).lower() if user_input in {'yes', 'y'}: - copyfile(new_file, path.join(data_path, file_name)) + copyfile( + new_file, + path.join( + data_path, file_name + ) + ) test_result = False - test_files_check.append((file_name, 'Failed -> content diff')) + test_files_check.append( + (file_name, 'Failed -> content diff') + ) if txt_file and args.fail_txt: to_show = False answer = user_input else: print("Passed!") - test_files_check.append((file_name, 'Passed')) + test_files_check.append( + (file_name, 'Passed') + ) else: print("Passed!") test_files_check.append((file_name, 'Passed')) @@ -968,11 +1303,18 @@ def get_id(l): if os.path.isfile(file): master_files.remove(file) else: - # not comparing directories; such as the range plots directory in mode subgroups + # not comparing directories; such as the range plots + # directory in mode subgroups if os.path.isdir(new_file): continue - print('File master {file} does not exist'.format(file=file)) - test_files_check.append((file, 'Failed -> master file does not exist')) + print( + 'File master {file} does not exist'.format( + file=file + ) + ) + test_files_check.append( + (file, 'Failed -> master file does not exist') + ) if file.endswith("smlp_error.txt"): to_print = 'Test number ' + test_id + ' Crashed!' print(to_print) @@ -983,25 +1325,40 @@ def get_id(l): else: if not args.default: user_input = input( - 'What to do with the new file?\n1 - Nothing\n2 - Copy to master only\n3 - Copy to master and models\n4 - Remove from master only\n5 - Remove from master and models\nOption number: ') - while user_input not in {'1', '2', '3', '4', '5'}: + 'What to do with the new file?\n' + '1 - Nothing\n' + '2 - Copy to master only\n' + '3 - Copy to master and models\n' + '4 - Remove from master only\n' + '5 - Remove from master and models\n' + 'Option number: ' + ) + while user_input not in { + '1', '2', '3', '4', '5' + }: user_input = input('(1|2|3|4|5):') if user_input == '1': pass elif user_input == '2': if os.path.isdir(new_file): - copytree(new_file, master_file, dirs_exist_ok=True) + copytree( + new_file, + master_file, + dirs_exist_ok=True + ) else: copyfile(new_file, master_file) elif user_input == '3': copyfile(new_file, master_file) - copyfile(new_file, path.join(models_path, file_name)) + copyfile( + new_file, + path.join(models_path, file_name) + ) elif user_input == '4': os.remove(master_file) elif user_input == '5': os.remove(master_file) os.remove(path.join(models_path, file_name)) - """ diff_errors.append('File master {file} does not exist'.format(file=file)) user_input = input('Do you wish to copy the new file to master?\n(yes/no|y/n): ').lower() @@ -1012,25 +1369,39 @@ def get_id(l): print('Copying file...') """ for file in master_files: - new_file = path.join(output_path, file); #print('new_file', new_file) - master_file = path.join(master_path, file); #print(' master_file', master_file) + new_file = path.join(output_path, file) + #print('new_file', new_file) + master_file = path.join(master_path, file) + #print(' master_file', master_file) file_name = file print('File new {file} does not exist'.format(file=file)) - test_files_check.append((file, 'Failed -> new file does not exist')) + test_files_check.append( + (file, 'Failed -> new file does not exist') + ) test_result = False # diff_errors.append('File new {file} does not exist'.format(file=file)) if not args.default: - user_input = input( - 'What to do with the master file?\n1 - Nothing\n2 - Remove from master only\n3 - Remove from master and models\nOption number: ') - while user_input not in {'1', '2', '3',}: - user_input = input('(1|2|3):') - if user_input == '1': + user_input = input( + 'What to do with the master file?\n' + '1 - Nothing\n' + '2 - Remove from master only\n' + '3 - Remove from master and models\nOption number: ' + ) + while user_input not in { + '1', + '2', + '3', + }: + user_input = input('(1|2|3):') + if user_input == '1': pass - elif user_input == '2': + elif user_input == '2': os.remove(master_file) - elif user_input == '3': + elif user_input == '3': os.remove(master_file) - if os.path.exists(path.join(models_path, file_name)): + if os.path.exists( + path.join(models_path, file_name) + ): os.remove(path.join(models_path, file_name)) if log: if test_result: @@ -1048,27 +1419,44 @@ def get_id(l): print('Error in {stage} stage:'.format(stage=test_error[0])) print(test_error[1]) if log: - write_to_log('Error in {stage} stage:'.format(stage=test_error[0])) + write_to_log( + 'Error in {stage} stage:'.format( + stage=test_error[0] + ) + ) write_to_log(test_error[1]) if DEBUG: print('9') print('log and not args.diff', log and not args.diff) - + if log and not args.diff: if path.exists(master_log_file): print('Comparing regression logs:') if not path.exists(master_log_file): copyfile(log_file, master_log_file) - p = Popen('diff {master_log} {new_log}'.format(new_log=log_file, master_log=master_log_file), shell=True, - stdin=PIPE, - stdout=PIPE, stderr=PIPE) + p = Popen( + 'diff {master_log} {new_log}'.format( + new_log=log_file, master_log=master_log_file + ), + shell=True, + stdin=PIPE, + stdout=PIPE, + stderr=PIPE + ) output, error = p.communicate() if p.returncode == 1: - Popen('{diff} {master_log} {new_log}'.format(diff=DIFF, new_log=log_file, master_log=master_log_file), - shell=True).wait() + Popen( + '{diff} {master_log} {new_log}'.format( + diff=DIFF, new_log=log_file, master_log=master_log_file + ), + shell=True + ).wait() user_input = input( - 'Do you wish to switch the new log file with the master log file?\n(yes/no|y/n): ').lower() + 'Do you wish to switch the new log file with the master ' + 'log file?\n' + '(yes/no|y/n): ' + ).lower() while user_input not in {'yes', 'no', 'y', 'n'}: user_input = input('(yes/no|y/n):').lower() if user_input in {'yes', 'y'}: @@ -1079,13 +1467,14 @@ def get_id(l): else: print("master log file does not exist!") user_input = input( - 'Do you wish to copy the new log file to master?\n(yes/no|y/n): ').lower() + 'Do you wish to copy the new log file to master?\n' + '(yes/no|y/n): ' + ).lower() while user_input not in {'yes', 'no', 'y', 'n'}: user_input = input('(yes/no|y/n):').lower() if user_input in {'yes', 'y'}: copyfile(log_file, master_log_file) print('Replacing Files...') - """ for testid, errors in test_errors_dict.items(): if len(errors) >= 1: @@ -1096,22 +1485,23 @@ def get_id(l): for err in diff_errors: print(err + '\n') """ - if False: # not args.print_command: + if False: # not args.print_command: chdir(code_path) try: rmtree(temp_code_dir) except: print("Can't delete " + temp_code_dir + " dir.") - - # report tests that crashed -- based on TestXXX_error.txt files that do not exist in master + + # report tests that crashed -- based on TestXXX_error.txt files that do not + # exist in master if len(new_error_fns) > 0: print('Tests crashed (not in the masters):') - for efn in new_error_fns: + for efn in new_error_fns: print(efn) else: print('No new tests crashed (not in the masters)') - - print("Time: " + str((time() - start_time)/60.0) + " minutes") + + print("Time: " + str((time() - start_time) / 60.0) + " minutes") print('End of regression')