[uss_qualifier] Improve check-matching (#1380)

Author: BenjaminPelletier

monitoring/monitorlib/inspection.py

@@ -2,6 +2,8 @@
 import inspect
 import pkgutil
 
+_modules_imported = set()
+
 
 def import_submodules(module) -> None:
     """Ensure that all descendant modules of a module are loaded.
@@ -10,10 +12,13 @@ def import_submodules(module) -> None:
 
     :param module: Parent module from which to start explicitly importing modules.
     """
+    if module in _modules_imported:
+        return
     for loader, module_name, is_pkg in pkgutil.walk_packages(
         module.__path__, module.__name__ + "."
     ):
         importlib.import_module(module_name)
+    _modules_imported.add(module)
 
 
 def get_module_object_by_name(parent_module, object_name: str):

monitoring/uss_qualifier/configurations/configuration.py

@@ -168,24 +168,13 @@ class FullyQualifiedCheck(ImplicitDict):
     """Scenario in which the check occurs."""
 
     test_case_name: str
-    """Test case in which the check occurs."""
+    """Test case in which the check occurs, omitting the ' test case' suffix.  Must be an exact match to documentation; sensitive to case and spacing."""
 
     test_step_name: str
-    """Test step in which the check occurs."""
+    """Test step in which the check occurs, omitting the ' test step' suffix.  Must be an exact match to documentation; sensitive to case and spacing."""
 
     check_name: str
-    """Name of the check."""
-
-    def contained_in(self, collection: Iterable[FullyQualifiedCheck]) -> bool:
-        for other in collection:
-            if (
-                self.scenario_type == other.scenario_type
-                and self.test_case_name == other.test_case_name
-                and self.test_step_name == other.test_step_name
-                and self.check_name == other.check_name
-            ):
-                return True
-        return False
+    """Name of the check, omitting the ' check' suffix.  Must be an exact match to documentation; sensitive to case and spacing."""
 
 
 class TestedRequirementsConfiguration(ImplicitDict):

monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml

@@ -417,7 +417,7 @@ v1:
 
         # It is acceptable for the checks listed below to fail (in terms of determining the status of tested requirements in this artifact)
         acceptable_findings:
-          - scenario_type: scenarios.astm.utm.dss.DSSInteroperability
+          - scenario_type: scenarios.astm.utm.dss.dss_interoperability.DSSInteroperability
             test_case_name: "Prerequisites"
             test_step_name: "Test environment requirements"
             check_name: "DSS instance is publicly addressable"

monitoring/uss_qualifier/reports/tested_requirements/breakdown.py

@@ -37,7 +37,11 @@
 from monitoring.uss_qualifier.requirements.definitions import RequirementID
 from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName
 from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation
-from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name
+from monitoring.uss_qualifier.scenarios.scenario import (
+    are_scenario_types_equal,
+    fully_qualified_check_in_collection,
+    get_scenario_type_by_name,
+)
 from monitoring.uss_qualifier.suites.definitions import (
     ActionType,
     TestSuiteActionDeclaration,
@@ -188,7 +192,7 @@ def _populate_breakdown_with_scenario_report(
                 matches = [
                     s
                     for s in tested_requirement.scenarios
-                    if s.type == scenario_type_name
+                    if are_scenario_types_equal(s.type, scenario_type_name)
                 ]
                 if matches:
                     tested_scenario = matches[0]
@@ -237,8 +241,8 @@ def _populate_breakdown_with_scenario_report(
                         name=check.name,
                         url="",
                         has_todo=False,
-                        is_finding_acceptable=current_check.contained_in(
-                            acceptable_findings
+                        is_finding_acceptable=fully_qualified_check_in_collection(
+                            current_check, acceptable_findings
                         ),
                     )  # TODO: Consider populating has_todo with documentation instead
                     if isinstance(check, FailedCheck):
@@ -340,7 +344,7 @@ def _populate_breakdown_with_scenario(
                     matches = [
                         s
                         for s in tested_requirement.scenarios
-                        if s.type == scenario_type_name
+                        if are_scenario_types_equal(s.type, scenario_type_name)
                     ]
                     if matches:
                         tested_scenario = matches[0]
@@ -383,8 +387,8 @@ def _populate_breakdown_with_scenario(
                             name=check.name,
                             url=check.url,
                             has_todo=check.has_todo,
-                            is_finding_acceptable=current_check.contained_in(
-                                acceptable_findings
+                            is_finding_acceptable=fully_qualified_check_in_collection(
+                                current_check, acceptable_findings
                             ),
                         )
                         tested_step.checks.append(tested_check)

monitoring/uss_qualifier/scenarios/definitions.py

@@ -3,7 +3,12 @@
 from monitoring.uss_qualifier.resources.definitions import ResourceID
 
 TestScenarioTypeName = str
-"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the `uss_qualifier` module"""
+"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the
+`uss_qualifier` module.
+
+Note that equality between different TestScenarioTypeNames (whether they refer to the same type of test scenario) should
+be determined via are_scenario_types_equal as multiple TestScenarioTypeNames may resolve to the same test scenario type.
+"""
 
 
 class TestScenarioDeclaration(ImplicitDict):

monitoring/uss_qualifier/scenarios/scenario.py

@@ -2,7 +2,7 @@
 import time as pytime
 import traceback
 from abc import ABC, abstractmethod
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from datetime import UTC, datetime, timedelta
 from enum import Enum
 from typing import TypeVar
@@ -19,6 +19,7 @@
 from monitoring.monitorlib.temporal import TestTimeContext
 from monitoring.uss_qualifier import scenarios as scenarios_module
 from monitoring.uss_qualifier.common_data_definitions import Severity
+from monitoring.uss_qualifier.configurations.configuration import FullyQualifiedCheck
 from monitoring.uss_qualifier.reports.report import (
     ErrorReport,
     FailedCheck,
@@ -222,6 +223,29 @@ def get_scenario_type_by_name(scenario_type_name: TestScenarioTypeName) -> type:
     return scenario_type
 
 
+def are_scenario_types_equal(
+    scenario_type_name_1: TestScenarioTypeName,
+    scenario_type_name_2: TestScenarioTypeName,
+) -> bool:
+    scenario_type_1 = get_scenario_type_by_name(scenario_type_name_1)
+    scenario_type_2 = get_scenario_type_by_name(scenario_type_name_2)
+    return scenario_type_1 == scenario_type_2
+
+
+def fully_qualified_check_in_collection(
+    check: FullyQualifiedCheck, collection: Iterable[FullyQualifiedCheck]
+) -> bool:
+    for other in collection:
+        if (
+            are_scenario_types_equal(check.scenario_type, other.scenario_type)
+            and check.test_case_name == other.test_case_name
+            and check.test_step_name == other.test_step_name
+            and check.check_name == other.check_name
+        ):
+            return True
+    return False
+
+
 class GenericTestScenario(ABC):
     """Generic Test Scenario allowing mutualization of test scenario implementation.
 

monitoring/uss_qualifier/suites/suite.py

@@ -51,6 +51,8 @@
     ScenarioCannotContinueError,
     TestRunCannotContinueError,
     TestScenario,
+    are_scenario_types_equal,
+    fully_qualified_check_in_collection,
     get_scenario_type_by_name,
 )
 from monitoring.uss_qualifier.suites.definitions import (
@@ -482,7 +484,9 @@ def stop_fast(
                         test_step_name=test_step_name,
                         check_name=check_name,
                     )
-                    if current_check.contained_in(self.acceptable_findings):
+                    if fully_qualified_check_in_collection(
+                        current_check, self.acceptable_findings
+                    ):
                         return False
             return True
         return False
@@ -569,10 +573,15 @@ def _is_selected_by(
                     "types" in f.is_test_scenario
                     and f.is_test_scenario.types is not None
                 ):
-                    if (
-                        action.test_scenario.declaration.scenario_type
-                        not in f.is_test_scenario.types
-                    ):
+                    matches_scenario_type = False
+                    for scenario_type in f.is_test_scenario.types:
+                        if are_scenario_types_equal(
+                            scenario_type,
+                            action.test_scenario.declaration.scenario_type,
+                        ):
+                            matches_scenario_type = True
+                            break
+                    if not matches_scenario_type:
                         return False
                 result = True
             else:

schemas/monitoring/uss_qualifier/configurations/configuration/FullyQualifiedCheck.json

@@ -8,19 +8,19 @@
       "type": "string"
     },
     "check_name": {
-      "description": "Name of the check.",
+      "description": "Name of the check, omitting the ' check' suffix.  Must be an exact match to documentation; sensitive to case and spacing.",
       "type": "string"
     },
     "scenario_type": {
       "description": "Scenario in which the check occurs.",
       "type": "string"
     },
     "test_case_name": {
-      "description": "Test case in which the check occurs.",
+      "description": "Test case in which the check occurs, omitting the ' test case' suffix.  Must be an exact match to documentation; sensitive to case and spacing.",
       "type": "string"
     },
     "test_step_name": {
-      "description": "Test step in which the check occurs.",
+      "description": "Test step in which the check occurs, omitting the ' test step' suffix.  Must be an exact match to documentation; sensitive to case and spacing.",
       "type": "string"
     }
   },