[uss_qualifier/netrid] Check response validity when calling all_flights (#1388)

Author: the-glu

.basedpyright/baseline.json

@@ -13509,30 +13509,6 @@
                     "lineCount": 1
                 }
             },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 36,
-                    "endColumn": 50,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 50,
-                    "endColumn": 64,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 80,
-                    "endColumn": 88,
-                    "lineCount": 1
-                }
-            },
             {
                 "code": "reportArgumentType",
                 "range": {
@@ -13573,14 +13549,6 @@
                     "lineCount": 1
                 }
             },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 80,
-                    "endColumn": 88,
-                    "lineCount": 1
-                }
-            },
             {
                 "code": "reportCallIssue",
                 "range": {

monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py

@@ -178,10 +178,12 @@ def poll_func(rect: LatLngRect) -> bool:
             poll_func,
         )
 
+    def _is_area_too_large(self, rect: s2sphere.LatLngRect) -> bool:
+        return geo.get_latlngrect_diagonal_km(rect) > self._rid_version.max_diagonal_km
+
     def _fetch_flights_from_dss(self, rect: LatLngRect) -> dict[str, TelemetryMapping]:
         # We grab all flights from the SPs (which we know how to reach by first querying the DSS).
         # This is authenticated and is expected to succeed
-        # TODO: Add the following requests to the documentation. Possibly split it as a test step.
         sp_observation = rid.all_flights(
             rect,
             include_recent_positions=True,
@@ -191,14 +193,23 @@ def _fetch_flights_from_dss(self, rect: LatLngRect) -> dict[str, TelemetryMappin
             dss_participant_id=self._dss.participant_id,
         )
 
+        self.record_queries(sp_observation.queries)
+
         mapping_by_injection_id = (
             display_data_evaluator.map_fetched_to_injected_flights(
                 self._injected_flights,
                 list(sp_observation.uss_flight_queries.values()),
+                list(sp_observation.uss_flight_details_queries.values()),
                 self._query_cache,
             )
         )
-        self.record_queries(sp_observation.queries)
+
+        display_data_evaluator.check_fetched_flights(
+            sp_observation,
+            self,
+            self._dss.participant_id,
+            self._is_area_too_large(rect),
+        )
 
         return mapping_by_injection_id
 

monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py

@@ -16,6 +16,7 @@
 from monitoring.monitorlib.fetch import Query
 from monitoring.monitorlib.fetch.rid import (
     FetchedFlights,
+    FetchedUSSFlightDetails,
     FetchedUSSFlights,
     Position,
     all_flights,
@@ -41,7 +42,10 @@
 from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import (
     VirtualObserver,
 )
-from monitoring.uss_qualifier.scenarios.scenario import TestScenario
+from monitoring.uss_qualifier.scenarios.scenario import (
+    GenericTestScenario,
+    TestScenario,
+)
 
 SPEED_PRECISION = 0.05
 HEIGHT_PRECISION_M = 1
@@ -102,8 +106,10 @@ class FetchedToInjectedCache:
     mappings: dict[str, ParticipantID] = field(default_factory=dict)
     """Mapping is a map of URL -> ParticipantID that we found until now"""
 
-    unmapped: list[FetchedUSSFlights] = field(default_factory=list)
-    """Unmapped is the list of flights we didn't attributed yet"""
+    unmapped: list[FetchedUSSFlights | FetchedUSSFlightDetails] = field(
+        default_factory=list
+    )
+    """Unmapped is the list of flights or flight details we didn't attributed yet"""
 
 
 def map_observations_to_injected_flights(
@@ -173,9 +179,60 @@ def map_observations_to_injected_flights(
     return mapping
 
 
+def check_fetched_flights(
+    fetched_flights: FetchedFlights,
+    scenario: GenericTestScenario,
+    dss_participant_id: str,
+    area_too_large: bool,
+):
+    """Perform checks on a FetchedFlights result from 'rid.all_flights' helper"""
+
+    with scenario.check("Successful ISA query", [dss_participant_id]) as check:
+        if not fetched_flights.dss_isa_query.success:
+            check.record_failed(
+                summary="Could not query ISAs from DSS",
+                details=f"Query to {dss_participant_id}'s DSS at failed: {', '.join(fetched_flights.dss_isa_query.errors)}",
+                query_timestamps=[
+                    fetched_flights.dss_isa_query.request.initiated_at.datetime
+                ],
+            )
+
+    if area_too_large:  # Don't perform flight queries checks as they should fail
+        return
+
+    for flight_query in fetched_flights.uss_flight_queries.values():
+        with scenario.check(
+            "Successful flight query", flight_query.participant_id
+        ) as check:
+            if not flight_query.success:
+                check.record_failed(
+                    summary="Flight query not successful",
+                    details=f"Query to {flight_query.query.request.url} failed: {', '.join(flight_query.errors)}",
+                    query_timestamps=[flight_query.query.request.initiated_at.datetime]
+                    if flight_query.query.request.initiated_at
+                    else [],
+                )
+
+    for flight_details_query in fetched_flights.uss_flight_details_queries.values():
+        with scenario.check(
+            "Successful flight details query", flight_details_query.participant_id
+        ) as check:
+            if not flight_details_query.success:
+                check.record_failed(
+                    summary="Flight details query not successful",
+                    details=f"Query to {flight_details_query.query.request.url} failed: {', '.join(flight_details_query.errors)}",
+                    query_timestamps=[
+                        flight_details_query.query.request.initiated_at.datetime
+                    ]
+                    if flight_details_query.query.request.initiated_at
+                    else [],
+                )
+
+
 def map_fetched_to_injected_flights(
     injected_flights: list[InjectedFlight],
     fetched_flights: list[FetchedUSSFlights],
+    fetched_flights_details: list[FetchedUSSFlightDetails],
     query_cache: FetchedToInjectedCache,
 ) -> dict[str, TelemetryMapping]:
     """Identify which of the fetched flights (if any) matches to each of the injected flights.
@@ -185,6 +242,7 @@ def map_fetched_to_injected_flights(
 
     :param injected_flights: Flights injected into RID Service Providers under test.
     :param fetched_flights: Flight observed from an RID Display Provider under test.
+    :param fetched_flights_details: Flight details observed from an RID Display Provider under test.
     :param query_cache: A FetchedToInjectedCache, used to maintain participant_id in various queries
     :return: Mapping between InjectedFlight and observed Flight, indexed by injection_id.
     """
@@ -205,7 +263,7 @@ def map_fetched_to_injected_flights(
             )
 
     # Add to the todo list queries to map
-    for uss_query in fetched_flights:
+    for uss_query in fetched_flights + fetched_flights_details:
         if not uss_query.has_field_with_value("participant_id"):
             query_cache.unmapped.append(uss_query)
 
@@ -284,14 +342,23 @@ def evaluate_system_instantaneously(
                 dss_participant_id=self._dss.participant_id,
             )
 
+            for q in sp_observation.queries:
+                self._test_scenario.record_query(q)
+
             # map observed flights to injected flight and attribute participant ID
             mapping_by_injection_id = map_fetched_to_injected_flights(
                 self._injected_flights,
                 list(sp_observation.uss_flight_queries.values()),
+                list(sp_observation.uss_flight_details_queries.values()),
                 self._query_cache,
             )
-            for q in sp_observation.queries:
-                self._test_scenario.record_query(q)
+
+            check_fetched_flights(
+                sp_observation,
+                self._test_scenario,
+                self._dss.participant_id,
+                self._is_area_too_large(rect),
+            )
 
             # Evaluate observations
             self._evaluate_sp_observation(rect, sp_observation, mapping_by_injection_id)
@@ -325,6 +392,9 @@ def evaluate_system_instantaneously(
                 # TODO: If bounding rect is smaller than area-too-large threshold, expand slightly above area-too-large threshold and re-observe
             self._test_scenario.end_test_step()
 
+    def _is_area_too_large(self, rect: s2sphere.LatLngRect) -> bool:
+        return geo.get_latlngrect_diagonal_km(rect) > self._rid_version.max_diagonal_km
+
     def _evaluate_observation(
         self,
         observer: RIDSystemObserver,
@@ -333,10 +403,8 @@ def _evaluate_observation(
         query: fetch.Query,
         verified_sps: set[str],
     ) -> None:
-        diagonal_km = (
-            rect.lo().get_distance(rect.hi()).degrees * geo.EARTH_CIRCUMFERENCE_KM / 360
-        )
-        if diagonal_km > self._rid_version.max_diagonal_km:
+        diagonal_km = geo.get_latlngrect_diagonal_km(rect)
+        if self._is_area_too_large(rect):
             self._evaluate_area_too_large_observation(
                 observer, rect, diagonal_km, query
             )
@@ -755,19 +823,6 @@ def _evaluate_sp_observation(
         # endpoints (and therefore cannot provide a callback/base URL), calling the one-time query endpoint
         # is currently much cleaner.  If this test is applied to a DSS that does not implement the one-time
         # ISA query endpoint, this check can be adapted.
-        with self._test_scenario.check(
-            "ISA query", [self._dss.participant_id]
-        ) as check:
-            if not sp_observation.dss_isa_query.success:
-                check.record_failed(
-                    summary="Could not query ISAs from DSS",
-                    details=f"Query to {self._dss.participant_id}'s DSS at {sp_observation.dss_isa_query.query.request.url} failed {sp_observation.dss_isa_query.query.status_code}",
-                    query_timestamps=[
-                        sp_observation.dss_isa_query.query.request.initiated_at.datetime
-                    ],
-                )
-                return
-
         diagonal_km = (
             requested_area.lo().get_distance(requested_area.hi()).degrees
             * geo.EARTH_CIRCUMFERENCE_KM
@@ -1070,14 +1125,23 @@ def evaluate_disconnected_flights(
             dss_participant_id=self._dss.participant_id,
         )
 
+        for q in sp_observation.queries:
+            self._test_scenario.record_query(q)
+
         # map observed flights to injected flight and attribute participant ID
         mapping_by_injection_id = map_fetched_to_injected_flights(
             self._injected_flights,
             list(sp_observation.uss_flight_queries.values()),
+            list(sp_observation.uss_flight_details_queries.values()),
             self._query_cache,
         )
-        for q in sp_observation.queries:
-            self._test_scenario.record_query(q)
+
+        check_fetched_flights(
+            sp_observation,
+            self._test_scenario,
+            self._dss.participant_id,
+            False,
+        )
 
         # Evaluate observations
         self._evaluate_sp_observation(sp_observation, mapping_by_injection_id)
@@ -1097,19 +1161,6 @@ def _evaluate_sp_observation(
         # endpoints (and therefore cannot provide a callback/base URL), calling the one-time query endpoint
         # is currently much cleaner.  If this test is applied to a DSS that does not implement the one-time
         # ISA query endpoint, this check can be adapted.
-        with self._test_scenario.check(
-            "ISA query", [self._dss.participant_id]
-        ) as check:
-            if not sp_observation.dss_isa_query.success:
-                check.record_failed(
-                    summary="Could not query ISAs from DSS",
-                    details=f"Query to {self._dss.participant_id}'s DSS at {sp_observation.dss_isa_query.query.request.url} failed {sp_observation.dss_isa_query.query.status_code}",
-                    query_timestamps=[
-                        sp_observation.dss_isa_query.query.request.initiated_at.datetime
-                    ],
-                )
-                return
-
         self._evaluate_sp_observation_of_disconnected_flights(sp_observation, mappings)
 
     def _evaluate_sp_observation_of_disconnected_flights(

monitoring/uss_qualifier/scenarios/astm/netrid/v19/fragments/sp_polling_queries.md

@@ -2,10 +2,14 @@
 
 uss_qualifier acts as a Display Provider to query Service Providers under test in this step.
 
-## ⚠️ ISA query check
+## ⚠️ Successful ISA query check
 
 **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification.  If uss_qualifier is unable to query the DSS for ISAs, this check will fail.
 
+## ⚠️ Successful flight query check
+
+**[astm.f3411.v19.NET0710,1](../../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../../requirements/astm/f3411/v19.md) require a Service Provider to implement the GET flight endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.
+
 ## ⚠️ Successful flight details query check
 
 **[astm.f3411.v19.NET0710,2](../../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../../requirements/astm/f3411/v19.md) require a Service Provider to implement the GET flight details endpoint.  This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.

monitoring/uss_qualifier/scenarios/astm/netrid/v19/fragments/sp_simple_queries.md

@@ -0,0 +1,11 @@
+# Service provider queries test step fragment
+
+uss_qualifier acts as a Display Provider to query Service Providers under test in this step, without fetching details.
+
+## ⚠️ Successful ISA query check
+
+**[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification.  If uss_qualifier is unable to query the DSS for ISAs, this check will fail.
+
+## ⚠️ Successful flight query check
+
+**[astm.f3411.v19.NET0710,1](../../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../../requirements/astm/f3411/v19.md) require a Service Provider to implement the GET flight endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.

monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md

@@ -24,36 +24,26 @@ A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required f
 
 ## Invalid requests test case
 
-### Injection test step
-
-In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future.  Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps.
-
-#### 🛑 Successful injection check
-
-Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test.
-
-**[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to provide a persistently supported test instance of their implementation.
-This check will fail if the flight was not successfully injected.
-
-#### 🛑 Identifiable flights check
-
-This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights.  This should generally be achieved by injecting appropriate data.
+### [Injection test step](./fragments/flight_injection.md)
 
 ### Invalid search area test step
 
-This step will attempt to search for flights in a rectangular area with a diagonal greater than [NetMaxDisplayAreaDiagonal] km.
+This step will attempt to search for flights in a rectangular area with a diagonal greater than [NetMaxDisplayAreaDiagonal] km. First, the Service Providers with service in the large area will be determined from the DSS and then each Service Provider will be queried for flights (this should succeed). Then each Service Provider will be queried again for flights, this time using an unacceptably-large area (this should fail).
+
+#### [Service provider queries test step](../v19/fragments/sp_simple_queries.md)
 
 #### ⚠️ Area too large check
 
 **[astm.f3411.v19.NET0250](../../../../requirements/astm/f3411/v19.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*.  If such a large view is requested and a 400 or 413 error code is not received or the response contains Remote ID data, then this check will fail.
 
 ### Unauthenticated requests test step
 
-In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials
-to confirm that the requested data is indeed available to any authorized query.
+in order to properly test whether the SP handles authentication correctly, after identifying the SP contact information via its ISA in the DSS, this step will first attempt to do a flights request with the proper credentials to confirm that the requested data is indeed available to any authorized query.
 
 It then repeats the exact same request without credentials, and expects this to fail.
 
+#### [Service provider queries test step](../v19/fragments/sp_simple_queries.md)
+
 #### ⚠️ Missing credentials check
 
 This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**,
@@ -63,6 +53,8 @@ and that requests for existing flights that are executed with missing credential
 
 This step is similar to unauthenticated requests, but uses incorrectly-authenticated requests instead.
 
+#### [Service provider queries test step](../v19/fragments/sp_simple_queries.md)
+
 #### ⚠️ Invalid credentials check
 This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**,
 and that requests for existing flights that are executed with incorrect credentials fail.