Skip to content

Commit 79ed434

Browse files
the-gluthe-glu
committed
[netrid] Check response validity when calling all_flights
1 parent bac22cb commit 79ed434

File tree

9 files changed

+127
-21
lines changed

9 files changed

+127
-21
lines changed

‎monitoring/monitorlib/fetch/rid.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from monitoring.monitorlib.fetch import Query, QueryType
1818
from monitoring.monitorlib.infrastructure import UTMClientSession
1919
from monitoring.monitorlib.rid import RIDVersion
20+
from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
2021

2122

2223
class ISA(ImplicitDict):
@@ -1359,6 +1360,39 @@ def flights(self) -> list[Flight]:
13591360
def success(self) -> bool:
13601361
return not self.errors
13611362

1363+
def perform_checks(self, scenario: GenericTestScenario, dss_participant_id: str):
1364+
with scenario.check("Successful ISA query", [dss_participant_id]) as check:
1365+
if not self.dss_isa_query.success:
1366+
check.record_failed(
1367+
summary="Could not query ISAs from DSS",
1368+
details=f"Query to {dss_participant_id}'s DSS at failed: {', '.join(self.dss_isa_query.errors)}",
1369+
query_timestamps=[self.dss_isa_query.request.initiated_at.datetime],
1370+
)
1371+
1372+
for flight_query in self.uss_flight_queries.values():
1373+
with scenario.check(
1374+
"Successful flight query", flight_query.participant_id
1375+
) as check:
1376+
if not flight_query.success:
1377+
check.record_failed(
1378+
summary="Flight query not successful",
1379+
details=f"Query to {flight_query.query.request.url} failed: {', '.join(flight_query.errors)}",
1380+
query_timestamps=[flight_query.request.initiated_at.datetime],
1381+
)
1382+
1383+
for flight_details_query in self.uss_flight_details_queries.values():
1384+
with scenario.check(
1385+
"Successful flight details query", flight_details_query.participant_id
1386+
) as check:
1387+
if not flight_details_query.success:
1388+
check.record_failed(
1389+
summary="Flight details query not successful",
1390+
details=f"Query to {flight_details_query.query.request.url} failed: {', '.join(flight_details_query.errors)}",
1391+
query_timestamps=[
1392+
flight_details_query.request.initiated_at.datetime
1393+
],
1394+
)
1395+
13621396

13631397
def all_flights(
13641398
area: s2sphere.LatLngRect,

‎monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,12 @@ def poll_func(rect: LatLngRect) -> bool:
178178
poll_func,
179179
)
180180

181+
def _is_area_too_large(self, rect: s2sphere.LatLngRect) -> bool:
182+
return geo.get_latlngrect_diagonal_km(rect) > self._rid_version.max_diagonal_km
183+
181184
def _fetch_flights_from_dss(self, rect: LatLngRect) -> dict[str, TelemetryMapping]:
182185
# We grab all flights from the SPs (which we know how to reach by first querying the DSS).
183186
# This is authenticated and is expected to succeed
184-
# TODO: Add the following requests to the documentation. Possibly split it as a test step.
185187
sp_observation = rid.all_flights(
186188
rect,
187189
include_recent_positions=True,
@@ -191,14 +193,18 @@ def _fetch_flights_from_dss(self, rect: LatLngRect) -> dict[str, TelemetryMappin
191193
dss_participant_id=self._dss.participant_id,
192194
)
193195

196+
self.record_queries(sp_observation.queries)
197+
194198
mapping_by_injection_id = (
195199
display_data_evaluator.map_fetched_to_injected_flights(
196200
self._injected_flights,
197201
list(sp_observation.uss_flight_queries.values()),
198202
self._query_cache,
199203
)
200204
)
201-
self.record_queries(sp_observation.queries)
205+
206+
if not self._is_area_too_large(rect):
207+
sp_observation.perform_checks(self, self._dss.participant_id)
202208

203209
return mapping_by_injection_id
204210

‎monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -284,14 +284,29 @@ def evaluate_system_instantaneously(
284284
dss_participant_id=self._dss.participant_id,
285285
)
286286

287+
for q in sp_observation.queries:
288+
self._test_scenario.record_query(q)
289+
290+
with self._test_scenario.check(
291+
"ISA query", [self._dss.participant_id]
292+
) as check:
293+
if not sp_observation.success and not self._is_area_too_large(rect):
294+
check.record_failed(
295+
summary="Could not query ISAs from DSS",
296+
details=f"Query to {self._dss.participant_id}'s DSS at failed: {', '.join(sp_observation.errors)}",
297+
query_timestamps=[
298+
query.request.initiated_at.datetime
299+
for query in sp_observation.queries
300+
if query.request.initiated_at
301+
],
302+
)
303+
287304
# map observed flights to injected flight and attribute participant ID
288305
mapping_by_injection_id = map_fetched_to_injected_flights(
289306
self._injected_flights,
290307
list(sp_observation.uss_flight_queries.values()),
291308
self._query_cache,
292309
)
293-
for q in sp_observation.queries:
294-
self._test_scenario.record_query(q)
295310

296311
# Evaluate observations
297312
self._evaluate_sp_observation(rect, sp_observation, mapping_by_injection_id)
@@ -325,6 +340,9 @@ def evaluate_system_instantaneously(
325340
# TODO: If bounding rect is smaller than area-too-large threshold, expand slightly above area-too-large threshold and re-observe
326341
self._test_scenario.end_test_step()
327342

343+
def _is_area_too_large(self, rect: s2sphere.LatLngRect) -> bool:
344+
return geo.get_latlngrect_diagonal_km(rect) > self._rid_version.max_diagonal_km
345+
328346
def _evaluate_observation(
329347
self,
330348
observer: RIDSystemObserver,
@@ -333,10 +351,8 @@ def _evaluate_observation(
333351
query: fetch.Query,
334352
verified_sps: set[str],
335353
) -> None:
336-
diagonal_km = (
337-
rect.lo().get_distance(rect.hi()).degrees * geo.EARTH_CIRCUMFERENCE_KM / 360
338-
)
339-
if diagonal_km > self._rid_version.max_diagonal_km:
354+
diagonal_km = geo.get_latlngrect_diagonal_km(rect)
355+
if self._is_area_too_large(rect):
340356
self._evaluate_area_too_large_observation(
341357
observer, rect, diagonal_km, query
342358
)
@@ -1070,14 +1086,29 @@ def evaluate_disconnected_flights(
10701086
dss_participant_id=self._dss.participant_id,
10711087
)
10721088

1089+
for q in sp_observation.queries:
1090+
self._test_scenario.record_query(q)
1091+
1092+
with self._test_scenario.check(
1093+
"ISA query", [self._dss.participant_id]
1094+
) as check:
1095+
if not sp_observation.success:
1096+
check.record_failed(
1097+
summary="Could not query ISAs from DSS",
1098+
details=f"Query to {self._dss.participant_id}'s DSS at failed: {', '.join(sp_observation.errors)}",
1099+
query_timestamps=[
1100+
query.request.initiated_at.datetime
1101+
for query in sp_observation.queries
1102+
if query.request.initiated_at
1103+
],
1104+
)
1105+
10731106
# map observed flights to injected flight and attribute participant ID
10741107
mapping_by_injection_id = map_fetched_to_injected_flights(
10751108
self._injected_flights,
10761109
list(sp_observation.uss_flight_queries.values()),
10771110
self._query_cache,
10781111
)
1079-
for q in sp_observation.queries:
1080-
self._test_scenario.record_query(q)
10811112

10821113
# Evaluate observations
10831114
self._evaluate_sp_observation(sp_observation, mapping_by_injection_id)

‎monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@ This particular test requires each flight to be uniquely identifiable by its 2D
4141

4242
### Invalid search area test step
4343

44-
This step will attempt to search for flights in a rectangular area with a diagonal greater than [NetMaxDisplayAreaDiagonal] km.
44+
This step will attempt to search for flights in a rectangular area with a diagonal greater than [NetMaxDisplayAreaDiagonal] km. First, the Service Providers with service in the large area will be determined from the DSS (this should succeed), and then each Service Provider will be queried for flights using an unacceptably-large area (this should fail).
45+
46+
#### ⚠️ Initial queries check
47+
48+
**[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail.
4549

4650
#### ⚠️ Area too large check
4751

@@ -54,6 +58,10 @@ to confirm that the requested data is indeed available to any authorized query.
5458

5559
It then repeats the exact same request without credentials, and expects this to fail.
5660

61+
#### ⚠️ Initial queries check
62+
63+
**[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. Then, in order to properly test whether the SP handles authentication correctly, after identifying the SP contact information via its ISA in the DSS, this step will first attempt to do a flights request with the proper credentials to confirm that the requested data is indeed available to any authorized query. If uss_qualifier is unable to query the DSS for ISAs or flights, this check will fail.
64+
5765
#### ⚠️ Missing credentials check
5866

5967
This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**,
@@ -63,6 +71,10 @@ and that requests for existing flights that are executed with missing credential
6371

6472
This step is similar to unauthenticated requests, but uses incorrectly-authenticated requests instead.
6573

74+
#### ⚠️ Initial queries check
75+
76+
**[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. Then, in order to properly test whether the SP handles authentication correctly, after identifying the SP contact information via its ISA in the DSS, this step will first attempt to do a flights request with the proper credentials to confirm that the requested data is indeed available to any authorized query. If uss_qualifier is unable to query the DSS for ISAs or flights, this check will fail.
77+
6678
#### ⚠️ Invalid credentials check
6779
This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**,
6880
and that requests for existing flights that are executed with incorrect credentials fail.

‎monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,34 @@ A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required f
2828

2929
### Invalid search area test step
3030

31-
This step will attempt to search for flights in a rectangular area with a diagonal greater than [NetMaxDisplayAreaDiagonal] km.
31+
This step will attempt to search for flights in a rectangular area with a diagonal greater than [NetMaxDisplayAreaDiagonal] km. First, the Service Providers with service in the large area will be determined from the DSS and then each Service Provider will be queried for flights (this should succeed). Then each Service Provider will be queried again for flights, this time using an unacceptably-large area (this should fail).
32+
33+
#### ⚠️ Successful ISA query check
34+
35+
**[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail.
36+
37+
#### ⚠️ Successful flight query check
38+
39+
**[astm.f3411.v22a.NET0710,1](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.
3240

3341
#### ⚠️ Area too large check
3442

3543
**[astm.f3411.v22a.NET0250](../../../../requirements/astm/f3411/v22a.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 400 or 413 error code is not received or the response contains Remote ID data, then this check will fail.
3644

3745
### Unauthenticated requests test step
3846

39-
In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials
40-
to confirm that the requested data is indeed available to any authorized query.
47+
in order to properly test whether the SP handles authentication correctly, after identifying the SP contact information via its ISA in the DSS, this step will first attempt to do a flights request with the proper credentials to confirm that the requested data is indeed available to any authorized query.
4148

4249
It then repeats the exact same request without credentials, and expects this to fail.
4350

51+
#### ⚠️ Successful ISA query check
52+
53+
**[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail.
54+
55+
#### ⚠️ Successful flight query check
56+
57+
**[astm.f3411.v22a.NET0710,1](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.
58+
4459
#### ⚠️ Missing credentials check
4560

4661
This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0210](../../../../requirements/astm/f3411/v22a.md)**,
@@ -50,6 +65,14 @@ and that requests for existing flights that are executed with missing credential
5065

5166
This step is similar to unauthenticated requests, but uses incorrectly-authenticated requests instead.
5267

68+
#### ⚠️ Successful ISA query check
69+
70+
**[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail.
71+
72+
#### ⚠️ Successful flight query check
73+
74+
**[astm.f3411.v22a.NET0710,1](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.
75+
5376
#### ⚠️ Invalid credentials check
5477

5578
This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0210](../../../../requirements/astm/f3411/v22a.md)**,

‎monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -561,7 +561,7 @@
561561
<tr>
562562
<td><a href="../../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
563563
<td>Implemented</td>
564-
<td><a href="../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md">ASTM NetRID DSS: Concurrent Requests</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../../scenarios/astm/netrid/v19/sp_operator_notify_slow_update.md">ASTM NetRID Service Provider operator notification under slow update rate</a><br><a href="../../../scenarios/astm/netrid/v19/networked_uas_disconnect.md">ASTM NetRID networked UAS disconnection</a><br><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
564+
<td><a href="../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md">ASTM NetRID DSS: Concurrent Requests</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../../scenarios/astm/netrid/v19/misbehavior.md">ASTM NetRID SP clients misbehavior handling</a><br><a href="../../../scenarios/astm/netrid/v19/sp_operator_notify_slow_update.md">ASTM NetRID Service Provider operator notification under slow update rate</a><br><a href="../../../scenarios/astm/netrid/v19/networked_uas_disconnect.md">ASTM NetRID networked UAS disconnection</a><br><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
565565
</tr>
566566
<tr>
567567
<td rowspan="1" style="vertical-align:top;"><a href="../../../requirements/interuss/mock_uss/hosted_instance.md">interuss<br>.mock_uss<br>.hosted_instance</a></td>

‎monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@
618618
<tr>
619619
<td><a href="../../../requirements/astm/f3411/v22a.md">NET0710,1</a></td>
620620
<td>Implemented</td>
621-
<td><a href="../../../scenarios/astm/netrid/v22a/networked_uas_disconnect.md">ASTM NetRID networked UAS disconnection</a><br><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
621+
<td><a href="../../../scenarios/astm/netrid/v22a/misbehavior.md">ASTM NetRID SP clients misbehavior handling</a><br><a href="../../../scenarios/astm/netrid/v22a/networked_uas_disconnect.md">ASTM NetRID networked UAS disconnection</a><br><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
622622
</tr>
623623
<tr>
624624
<td><a href="../../../requirements/astm/f3411/v22a.md">NET0710,2</a></td>
@@ -681,7 +681,7 @@
681681
<tr>
682682
<td><a href="../../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
683683
<td>Implemented</td>
684-
<td><a href="../../../scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md">ASTM NetRID DSS: Concurrent Requests</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../../scenarios/astm/netrid/v22a/sp_operator_notify_slow_update.md">ASTM NetRID Service Provider operator notification under slow update rate</a><br><a href="../../../scenarios/astm/netrid/v22a/networked_uas_disconnect.md">ASTM NetRID networked UAS disconnection</a><br><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
684+
<td><a href="../../../scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md">ASTM NetRID DSS: Concurrent Requests</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../../scenarios/astm/netrid/v22a/misbehavior.md">ASTM NetRID SP clients misbehavior handling</a><br><a href="../../../scenarios/astm/netrid/v22a/sp_operator_notify_slow_update.md">ASTM NetRID Service Provider operator notification under slow update rate</a><br><a href="../../../scenarios/astm/netrid/v22a/networked_uas_disconnect.md">ASTM NetRID networked UAS disconnection</a><br><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
685685
</tr>
686686
<tr>
687687
<td rowspan="1" style="vertical-align:top;"><a href="../../../requirements/interuss/mock_uss/hosted_instance.md">interuss<br>.mock_uss<br>.hosted_instance</a></td>

0 commit comments

Comments
 (0)