feat: add comprehensive structured logging across project

arat-rl.py

@@ -9,12 +9,16 @@
 import base64
 import random
 import string
+import logging
 import prance
 import difflib
 import requests
 import datetime
 import functools
 from collections import defaultdict
+from logger_config import setup_logging
+
+logger = logging.getLogger('arat-rl')
 
 
 
@@ -232,7 +236,7 @@ def send_request(content_type):
                 elif method == 'head':
                     return requests.head(url, headers=headers, params=query_params)
         except requests.exceptions.RequestException as e:
-            print(f"Request error: {e}")
+            logger.warning('Request error: %s', e)
             return None
 
     for param_value_dict in selected_parameters:
@@ -402,7 +406,7 @@ def report_http_500_errors():
     with open(filename, 'w') as file:
         file.write(report_content)
 
-    print(f"Report saved to {filename}")
+    logger.info('Report saved to %s', filename)
 
 
 def update_q_table(q_table, alpha, gamma, selected_operation, selected_parameters, response, path, query_params, body_params):
@@ -423,6 +427,7 @@ def update_q_table(q_table, alpha, gamma, selected_operation, selected_parameter
         reward = -5
 
     if response.status_code == 500:
+        logger.info('HTTP 500 error detected for operation: %s', operation_id)
         if operation_id not in http_500_operations:
             http_500_operations.append(operation_id)
             total_n[0] = total_n[0] + 1
@@ -685,6 +690,10 @@ def main():
     operations, parameters_frequency = analyze_information(openapi_spec)
     alpha, gamma, q_table = initialize_q_learning(operations, parameters_frequency)
 
+    logger.info('Starting ARAT-RL with spec: %s, base URL: %s, time limit: %s',
+                openapi_spec_file, base_url, sys.argv[3])
+    logger.info('Found %d operations', len(operations))
+
     start_time = time.time()
     time_limit = int(sys.argv[3])
     iteration = 0
@@ -694,6 +703,8 @@ def main():
         elapsed_time = time.time() - start_time
         if elapsed_time >= time_limit:
             break
+        if iteration % 100 == 0:
+            logger.info('Iteration %d, elapsed time: %.1fs', iteration, elapsed_time)
         parameter_values = generate_parameter_values(operations)
         selected_operation, selected_parameters = select_operations_and_parameters(operations, parameter_values,
                                                                                    q_table)
@@ -732,6 +743,7 @@ def main():
         iteration += 1
 
 if __name__ == "__main__":
+    setup_logging('arat-rl', log_file='arat-rl.log')
     base_url = sys.argv[2]
     EPSILON = [0.1]
     ss = [None]
@@ -749,4 +761,5 @@ def main():
     q_value = {}
     MUTATION_RATE = 0.1
     main()
+    logger.info('Testing completed, generating report')
     report_http_500_errors()

logger_config.py

@@ -0,0 +1,70 @@
+"""Centralized logging configuration for ARAT-RL.
+
+Usage:
+    from logger_config import setup_logging, get_logger
+
+    # In entry point scripts (if __name__ == "__main__"):
+    setup_logging('arat-rl', log_file='arat-rl.log')
+
+    # Create a module-level logger:
+    logger = get_logger('arat-rl')
+
+    # Use the logger:
+    logger.info('Starting testing with spec: %s', spec_file)
+    logger.warning('Request error: %s', e)
+    logger.debug('Response value: %s = %s', key, value)
+
+The log level can be controlled via the LOG_LEVEL environment variable.
+Supported levels: DEBUG, INFO, WARNING, ERROR. Default: INFO.
+"""
+
+import os
+import logging
+
+
+def setup_logging(name, log_level='INFO', log_file=None):
+    """Configure and return a logger with console and optional file output.
+
+    Args:
+        name: Logger name.
+        log_level: Default log level (overridden by LOG_LEVEL env var).
+        log_file: Optional path to a log file.
+
+    Returns:
+        Configured logging.Logger instance.
+    """
+    level_str = os.environ.get('LOG_LEVEL', log_level).upper()
+    level = getattr(logging, level_str, logging.INFO)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    formatter = logging.Formatter(
+        fmt='%(asctime)s [%(levelname)s] %(name)s: %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(level)
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+
+    if log_file is not None:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(level)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    return logger
+
+
+def get_logger(name):
+    """Convenience function to get a logger by name.
+
+    Args:
+        name: Logger name.
+
+    Returns:
+        logging.Logger instance.
+    """
+    return logging.getLogger(name)

main.py

@@ -9,12 +9,16 @@
 import base64
 import random
 import string
+import logging
 import prance
 import difflib
 import requests
 import datetime
 import functools
 from collections import defaultdict
+from logger_config import setup_logging
+
+logger = logging.getLogger('arat-rl-main')
 
 
 
@@ -232,7 +236,7 @@ def send_request(content_type):
                 elif method == 'head':
                     return requests.head(url, headers=headers, params=query_params)
         except requests.exceptions.RequestException as e:
-            print(f"Request error: {e}")
+            logger.warning('Request error: %s', e)
             return None
 
     for param_value_dict in selected_parameters:
@@ -642,6 +646,9 @@ def main():
     operations, parameters_frequency = analyze_information(openapi_spec)
     alpha, gamma, q_table = initialize_q_learning(operations, parameters_frequency)
 
+    logger.info('Starting arat-rl-main with spec: %s, base URL: %s', openapi_spec_file, base_url)
+    logger.info('Found %d operations', len(operations))
+
     start_time = time.time()
     time_limit = 3600
     iteration = 0
@@ -651,6 +658,8 @@ def main():
         elapsed_time = time.time() - start_time
         if elapsed_time >= time_limit:
             break
+        if iteration % 100 == 0:
+            logger.info('Iteration %d, elapsed time: %.1fs', iteration, elapsed_time)
         parameter_values = generate_parameter_values(operations)
         selected_operation, selected_parameters = select_operations_and_parameters(operations, parameter_values,
                                                                                    q_table)
@@ -689,6 +698,7 @@ def main():
         iteration += 1
 
 if __name__ == "__main__":
+    setup_logging('arat-rl-main', log_file='arat-rl-main.log')
     base_url = sys.argv[2]
     EPSILON = [0.1]
     ss = [None]

morest/experiment_utils/php_coverage_merger.py

@@ -1,8 +1,11 @@
 import argparse
+import logging
 import os
 import json
 import time
 
+logger = logging.getLogger('morest.experiment.coverage')
+
 
 def get_coverage_logs(path):
     files = os.listdir(path)
@@ -20,7 +23,7 @@ def read_data(path=""):
 def run_single(folder, result={}):
     logs = (get_coverage_logs(folder))
     for log in logs:
-        print("read", log)
+        logger.info('Read: %s', log)
         data = read_data(log)
         for file_name in data:
             if not (file_name in result):
@@ -36,16 +39,16 @@ def run_single(folder, result={}):
             continue
         line_count += len(result[file_name])
     result["line_count"] = line_count
-    print("total line count", line_count)
+    logger.info('Total line count: %d', line_count)
 
 
 def main(args):
     coverage_folder = args.c
     output_file = args.o
     result = {}
     while True:
-        print("read from", coverage_folder)
-        print("write to", output_file)
+        logger.info('Read from: %s', coverage_folder)
+        logger.info('Write to: %s', output_file)
         run_single(coverage_folder, result)
         with open(output_file, 'w') as data:
             json.dump(result, data)

morest/fuzzer.py

@@ -1,11 +1,15 @@
 import os
 import sys
+import logging
 
 from prance import ResolvingParser
 
 from build_graph import parse
 from fuzzer.fuzzer import APIFuzzer
 from utils.auth_util import get_token, SUT
+from logger_config import setup_logging
+
+logger = logging.getLogger('morest')
 
 
 def default_reclimit_handler(limit, parsed_url, recursions=()):
@@ -18,6 +22,7 @@ def default_reclimit_handler(limit, parsed_url, recursions=()):
 
 
 def main():
+    setup_logging('morest')
     # testing arguments format: swagger address, server address, system under test's name, *args to obtain token
     test_args = [sys.argv[1], sys.argv[2], SUT.BITBUCKET]
     parser = ResolvingParser(test_args[0],