Master -> Develop

scrub/scrubme.py

@@ -229,6 +229,8 @@ def main(conf_file=pathlib.Path('./scrub.cfg').resolve(), clean=False, console_l
                     scrub_utilities.check_artifact(tool_analysis_dir, True)
 
                     # Parse the results files
+                    logging.info('')
+                    logging.info('  Parsing results...')
                     parser.parse_warnings(tool_analysis_dir, scrub_conf_data)
 
                     # Check the raw results files

scrub/tools/parsers/get_codesonar_warnings.py

@@ -126,33 +126,44 @@ def parse_xml_warnings(input_file, output_file, codesonar_hub, exclude_p10=False
                 warning_count = warning_count + 1
 
 
-def parse_warnings(analysis_dir, tool_config_data):
+def parse_warnings(analysis_dir, tool_config_data, raw_input_file=None, parsed_output_file=None):
     """This function handles parsing of raw CodeSonar data.
 
     Inputs:
-        - analysis_dir: Absolute path to the raw SonarQube output file directory [string]
+        - analysis_dir: Absolute path to the raw CodeSonar output file directory [string]
         - tool_config_data: Dictionary of scrub configuration data [dict]
+        - raw_input_file: Absolute path to the raw input file [string] [optional]
+        - parsed_output_file: Absolute path to the raw output file [string] [optional]
     """
 
     # Initialize variables
     source_dir = tool_config_data.get('source_dir')
     codesonar_hub = tool_config_data.get('codesonar_hub')
     raw_analysis_metrics_file = analysis_dir.joinpath('analysis_metrics.json')
     raw_file_metrics_file = analysis_dir.joinpath('file_metrics.json')
-    parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('codesonar_raw.scrub')
     parsed_metrics_file = tool_config_data.get('scrub_analysis_dir').joinpath('codesonar_metrics.csv')
 
+    # Set the input file
+    if raw_input_file is None:
+        if tool_config_data.get('codesonar_results_template'):
+            raw_input_file = analysis_dir.joinpath('search.xml')
+        else:
+            raw_input_file = analysis_dir.joinpath('warning_detail_search.sarif')
+
+    # Set the output file
+    if parsed_output_file is None:
+        parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('codesonar_raw.scrub')
+
     # Print a status message
     logging.info('\t>> Executing command: get_codesonar_warnings.parse_warnings(%s, %s)', analysis_dir,
                  parsed_output_file)
     logging.info('\t>> From directory: %s', str(pathlib.Path().absolute()))
 
-    # Parse the SARIF results
-    if tool_config_data.get('codesonar_results_template'):
-        raw_input_file = analysis_dir.joinpath('search.xml')[0]
+    # Parse the results
+    if raw_input_file.suffix == '.xml':
         parse_xml_warnings(raw_input_file, parsed_output_file, codesonar_hub)
     else:
-        raw_input_file = analysis_dir.joinpath('warning_detail_search.sarif')
+        # Parse the SARIF file
         raw_warnings = translate_results.parse_sarif(raw_input_file, source_dir)
 
         # Create the SCRUB output file
@@ -161,7 +172,3 @@ def parse_warnings(analysis_dir, tool_config_data):
     # Parse the metrics files
     parse_metrics.parse_codesonar_metrics(raw_analysis_metrics_file, raw_file_metrics_file, parsed_metrics_file,
                                           source_dir)
-    # if raw_metrics_file.exists():
-    #     parse_metrics.parse_codesonar_metrics(raw_metrics_file, parsed_metrics_file, source_dir)
-    # else:
-    #     logging.info('\tWARNING: Metrics file not found. Check log for more details.')

scrub/tools/parsers/get_coverity_warnings.py

@@ -153,8 +153,8 @@ def parse_warnings(analysis_dir, tool_config_data):
         coverity_findings = translate_results.parse_sarif(analysis_dir.joinpath('coverity.sarif'),
                                                           tool_config_data.get('source_dir'))
 
-    # Parse the metrics file
-    if (cc_threshold >= 0) and (coverity_metrics_file.exists()):
+    # Parse the metrics file, if necessary
+    if cc_threshold >= 0:
         coverity_findings = coverity_findings + parse_cc(coverity_metrics_file, cc_threshold)
 
     # Create the output file

scrub/tools/parsers/get_gbuild_warnings.py

@@ -187,17 +187,23 @@ def parse_doublecheck_warnings(raw_input_file, parsed_output_file):
     translate_results.create_scrub_output_file(raw_warnings, parsed_output_file)
 
 
-def parse_warnings(analysis_dir, tool_config_data):
+def parse_warnings(analysis_dir, tool_config_data, raw_input_file=None, parsed_output_file=None):
     """This function parses the raw gbuild compiler warnings into the SCRUB format.
 
     Inputs:
-        - raw_input_file: Absolute path to the raw gbuild compiler log containing warnings [string]
-        - parsed_output_file: Absolute path to the file where the parsed warnings will be stored [string]
+        - analysis_dir: Absolute path to the raw gbuild output file directory [string]
+        - tool_config_data: Dictionary of scrub configuration data [dict]
+        - raw_input_file: Absolute path to the raw input file [string] [optional]
+        - parsed_output_file: Absolute path to the raw output file [string] [optional]
     """
 
-    # Initialize variables
-    raw_input_file = analysis_dir.joinpath('gbuild_build.log')
-    parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('gbuild_compiler_raw.scrub')
+    # Set input file
+    if raw_input_file is None:
+        raw_input_file = analysis_dir.joinpath('gbuild_build.log')
+
+    # Set the output file
+    if parsed_output_file is None:
+        parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('gbuild_compiler_raw.scrub')
 
     # Print a status message
     logging.info('')

scrub/tools/parsers/get_gcc_warnings.py

@@ -6,7 +6,7 @@
 ID_PREFIX = 'gcc'
 
 
-def parse_warnings(analysis_dir, tool_config_data):
+def parse_warnings(analysis_dir, tool_config_data, raw_input_file=None, parsed_output_file=None):
     """This function parses the raw GCC compiler warnings into the SCRUB format.
 
     Inputs:
@@ -24,8 +24,14 @@ def parse_warnings(analysis_dir, tool_config_data):
     warning_message = []
     parsing = False
     description = False
-    raw_input_file = analysis_dir.joinpath('gcc_build.log')
-    parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('gcc_compiler_raw.scrub')
+
+    # Set the input file
+    if raw_input_file is None:
+        raw_input_file = analysis_dir.joinpath('gcc_build.log')
+
+    # Set the output file
+    if parsed_output_file is None:
+        parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('gcc_compiler_raw.scrub')
 
     # Print a status message
     logging.info('')

scrub/tools/parsers/get_javac_warnings.py

@@ -7,18 +7,26 @@
 ID_PREFIX = 'javac'
 
 
-def parse_warnings(analysis_dir, tool_config_data):
+def parse_warnings(analysis_dir, tool_config_data, raw_input_file=None, parsed_output_file=None):
     """This function parses the raw javac compiler warnings into the SCRUB format.
 
     Inputs:
-        - raw_input_file: Absolute path to the raw javac compiler log containing warnings [string]
-        - parsed_output_file: Absolute path to the file where the parsed warnings will be stored [string]
+        - analysis_dir: Absolute path to the raw javac output file directory [string]
+        - tool_config_data: Dictionary of scrub configuration data [dict]
+        - raw_input_file: Absolute path to the raw input file [string] [optional]
+        - parsed_output_file: Absolute path to the raw output file [string] [optional]
     """
 
     # Initialize variables
     warning_count = 1
-    raw_input_file = analysis_dir.joinpath('javac_build.log')
-    parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('javac_compiler_raw.scrub')
+
+    # Set the input file
+    if raw_input_file is None:
+        raw_input_file = analysis_dir.joinpath('javac_build.log')
+
+    # Set the output file
+    if parsed_output_file is None:
+        parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('javac_compiler_raw.scrub')
 
     # Print a status message
     logging.info('')

scrub/tools/parsers/get_pylint_warnings.py

@@ -6,7 +6,7 @@
 ID_PREFIX = 'pylint'
 
 
-def parse_warnings(analysis_dir, tool_config_data):
+def parse_warnings(analysis_dir, tool_config_data, raw_input_file=None, parsed_output_file=None):
     """This function parses the raw PyLint warnings into the SCRUB format.
 
     Inputs:
@@ -16,8 +16,14 @@ def parse_warnings(analysis_dir, tool_config_data):
 
     # Initialize the variables
     warning_count = 1
-    raw_input_file = analysis_dir.joinpath('pylint_output.json')
-    parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('pylint_compiler_raw.scrub')
+
+    # Set the input file
+    if raw_input_file is None:
+        raw_input_file = analysis_dir.joinpath('pylint_output.json')
+
+    # Set the output file
+    if parsed_output_file is None:
+        parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('pylint_compiler_raw.scrub')
 
     # Read in the input data
     with open(raw_input_file, 'r') as input_fh:

scrub/tools/parsers/get_sonarqube_warnings.py

@@ -5,7 +5,7 @@
 ID_PREFIX = 'sonarqube'
 
 
-def parse_warnings(analysis_dir, tool_config_data):
+def parse_warnings(analysis_dir, tool_config_data, parsed_output_file=None):
     """This function parses the raw SonarQube warnings into the SCRUB format.
 
     Inputs:
@@ -18,11 +18,15 @@ def parse_warnings(analysis_dir, tool_config_data):
     raw_warnings = []
     sonarqube_url = tool_config_data.get('sonarqube_server')
     source_root = tool_config_data.get('source_dir')
-    parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('sonarqube_raw.scrub')
     metrics_output_file = tool_config_data.get('scrub_analysis_dir').joinpath('sonarqube_metrics.csv')
 
+    # Set the output file
+    if parsed_output_file is None:
+        parsed_output_file = tool_config_data.get('raw_results_dir').joinpath('sonarqube_raw.scrub')
+
     # Find all the raw findings results files in the directory
-    findings_results_files = analysis_dir.glob('*.json')
+    findings_results_files = (list(analysis_dir.glob('sonarqube_issues*.json')) +
+                              list(analysis_dir.glob('sonarqube_hotspots*.json')))
 
     # Iterate through every issue results file
     for raw_findings_file in findings_results_files:
@@ -107,5 +111,5 @@ def parse_warnings(analysis_dir, tool_config_data):
     # Create the SCRUB output file
     translate_results.create_scrub_output_file(raw_warnings, parsed_output_file)
 
-    # Parse the metrics file, if it exists
+    # Parse the metrics data
     parse_metrics.parse(analysis_dir, metrics_output_file, source_root, 'sonarqube')