diff --git a/monitoring/monitorlib/inspection.py b/monitoring/monitorlib/inspection.py index 0350cdd196..8eb8c4314c 100644 --- a/monitoring/monitorlib/inspection.py +++ b/monitoring/monitorlib/inspection.py @@ -2,6 +2,8 @@ import inspect import pkgutil +_modules_imported = set() + def import_submodules(module) -> None: """Ensure that all descendant modules of a module are loaded. @@ -10,10 +12,13 @@ def import_submodules(module) -> None: :param module: Parent module from which to start explicitly importing modules. """ + if module in _modules_imported: + return for loader, module_name, is_pkg in pkgutil.walk_packages( module.__path__, module.__name__ + "." ): importlib.import_module(module_name) + _modules_imported.add(module) def get_module_object_by_name(parent_module, object_name: str): diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index 066a14acc6..7ad43a8f19 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -168,24 +168,13 @@ class FullyQualifiedCheck(ImplicitDict): """Scenario in which the check occurs.""" test_case_name: str - """Test case in which the check occurs.""" + """Test case in which the check occurs, omitting the ' test case' suffix. Must be an exact match to documentation; sensitive to case and spacing.""" test_step_name: str - """Test step in which the check occurs.""" + """Test step in which the check occurs, omitting the ' test step' suffix. Must be an exact match to documentation; sensitive to case and spacing.""" check_name: str - """Name of the check.""" - - def contained_in(self, collection: Iterable[FullyQualifiedCheck]) -> bool: - for other in collection: - if ( - self.scenario_type == other.scenario_type - and self.test_case_name == other.test_case_name - and self.test_step_name == other.test_step_name - and self.check_name == other.check_name - ): - return True - return False + """Name of the check, omitting the ' check' suffix. Must be an exact match to documentation; sensitive to case and spacing.""" class TestedRequirementsConfiguration(ImplicitDict): diff --git a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml index c7ee19db26..ee8b860f34 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml @@ -417,7 +417,7 @@ v1: # It is acceptable for the checks listed below to fail (in terms of determining the status of tested requirements in this artifact) acceptable_findings: - - scenario_type: scenarios.astm.utm.dss.DSSInteroperability + - scenario_type: scenarios.astm.utm.dss.dss_interoperability.DSSInteroperability test_case_name: "Prerequisites" test_step_name: "Test environment requirements" check_name: "DSS instance is publicly addressable" diff --git a/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py b/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py index 3ee22f9128..e3c3c05573 100644 --- a/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py +++ b/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py @@ -37,7 +37,11 @@ from monitoring.uss_qualifier.requirements.definitions import RequirementID from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation -from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name +from monitoring.uss_qualifier.scenarios.scenario import ( + are_scenario_types_equal, + fully_qualified_check_in_collection, + get_scenario_type_by_name, +) from monitoring.uss_qualifier.suites.definitions import ( ActionType, TestSuiteActionDeclaration, @@ -188,7 +192,7 @@ def _populate_breakdown_with_scenario_report( matches = [ s for s in tested_requirement.scenarios - if s.type == scenario_type_name + if are_scenario_types_equal(s.type, scenario_type_name) ] if matches: tested_scenario = matches[0] @@ -237,8 +241,8 @@ def _populate_breakdown_with_scenario_report( name=check.name, url="", has_todo=False, - is_finding_acceptable=current_check.contained_in( - acceptable_findings + is_finding_acceptable=fully_qualified_check_in_collection( + current_check, acceptable_findings ), ) # TODO: Consider populating has_todo with documentation instead if isinstance(check, FailedCheck): @@ -340,7 +344,7 @@ def _populate_breakdown_with_scenario( matches = [ s for s in tested_requirement.scenarios - if s.type == scenario_type_name + if are_scenario_types_equal(s.type, scenario_type_name) ] if matches: tested_scenario = matches[0] @@ -383,8 +387,8 @@ def _populate_breakdown_with_scenario( name=check.name, url=check.url, has_todo=check.has_todo, - is_finding_acceptable=current_check.contained_in( - acceptable_findings + is_finding_acceptable=fully_qualified_check_in_collection( + current_check, acceptable_findings ), ) tested_step.checks.append(tested_check) diff --git a/monitoring/uss_qualifier/scenarios/definitions.py b/monitoring/uss_qualifier/scenarios/definitions.py index 6dc3473d1b..6860570653 100644 --- a/monitoring/uss_qualifier/scenarios/definitions.py +++ b/monitoring/uss_qualifier/scenarios/definitions.py @@ -3,7 +3,12 @@ from monitoring.uss_qualifier.resources.definitions import ResourceID TestScenarioTypeName = str -"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the `uss_qualifier` module""" +"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the +`uss_qualifier` module. + +Note that equality between different TestScenarioTypeNames (whether they refer to the same type of test scenario) should +be determined via are_scenario_types_equal as multiple TestScenarioTypeNames may resolve to the same test scenario type. +""" class TestScenarioDeclaration(ImplicitDict): diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 8dcee12631..839e49e103 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -2,7 +2,7 @@ import time as pytime import traceback from abc import ABC, abstractmethod -from collections.abc import Callable +from collections.abc import Callable, Iterable from datetime import UTC, datetime, timedelta from enum import Enum from typing import TypeVar @@ -19,6 +19,7 @@ from monitoring.monitorlib.temporal import TestTimeContext from monitoring.uss_qualifier import scenarios as scenarios_module from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.configurations.configuration import FullyQualifiedCheck from monitoring.uss_qualifier.reports.report import ( ErrorReport, FailedCheck, @@ -222,6 +223,29 @@ def get_scenario_type_by_name(scenario_type_name: TestScenarioTypeName) -> type: return scenario_type +def are_scenario_types_equal( + scenario_type_name_1: TestScenarioTypeName, + scenario_type_name_2: TestScenarioTypeName, +) -> bool: + scenario_type_1 = get_scenario_type_by_name(scenario_type_name_1) + scenario_type_2 = get_scenario_type_by_name(scenario_type_name_2) + return scenario_type_1 == scenario_type_2 + + +def fully_qualified_check_in_collection( + check: FullyQualifiedCheck, collection: Iterable[FullyQualifiedCheck] +) -> bool: + for other in collection: + if ( + are_scenario_types_equal(check.scenario_type, other.scenario_type) + and check.test_case_name == other.test_case_name + and check.test_step_name == other.test_step_name + and check.check_name == other.check_name + ): + return True + return False + + class GenericTestScenario(ABC): """Generic Test Scenario allowing mutualization of test scenario implementation. diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index e6c47e0f2f..a374481af5 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -51,6 +51,8 @@ ScenarioCannotContinueError, TestRunCannotContinueError, TestScenario, + are_scenario_types_equal, + fully_qualified_check_in_collection, get_scenario_type_by_name, ) from monitoring.uss_qualifier.suites.definitions import ( @@ -482,7 +484,9 @@ def stop_fast( test_step_name=test_step_name, check_name=check_name, ) - if current_check.contained_in(self.acceptable_findings): + if fully_qualified_check_in_collection( + current_check, self.acceptable_findings + ): return False return True return False @@ -569,10 +573,15 @@ def _is_selected_by( "types" in f.is_test_scenario and f.is_test_scenario.types is not None ): - if ( - action.test_scenario.declaration.scenario_type - not in f.is_test_scenario.types - ): + matches_scenario_type = False + for scenario_type in f.is_test_scenario.types: + if are_scenario_types_equal( + scenario_type, + action.test_scenario.declaration.scenario_type, + ): + matches_scenario_type = True + break + if not matches_scenario_type: return False result = True else: diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/FullyQualifiedCheck.json b/schemas/monitoring/uss_qualifier/configurations/configuration/FullyQualifiedCheck.json index 67650728de..d2edbbbe75 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/FullyQualifiedCheck.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/FullyQualifiedCheck.json @@ -8,7 +8,7 @@ "type": "string" }, "check_name": { - "description": "Name of the check.", + "description": "Name of the check, omitting the ' check' suffix. Must be an exact match to documentation; sensitive to case and spacing.", "type": "string" }, "scenario_type": { @@ -16,11 +16,11 @@ "type": "string" }, "test_case_name": { - "description": "Test case in which the check occurs.", + "description": "Test case in which the check occurs, omitting the ' test case' suffix. Must be an exact match to documentation; sensitive to case and spacing.", "type": "string" }, "test_step_name": { - "description": "Test step in which the check occurs.", + "description": "Test step in which the check occurs, omitting the ' test step' suffix. Must be an exact match to documentation; sensitive to case and spacing.", "type": "string" } },