From 4f6ccbe5471ace5b94f6522cc0f60864e8584ebd Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sat, 17 May 2025 16:06:37 +0200 Subject: [PATCH 1/9] implement analysis of sailplane grand prix --- opensoar/competition/daily_results_page.py | 95 ++-- opensoar/competition/sgp.py | 502 +++++++++++++++++++++ requirements.txt | 1 + setup.py | 1 + tests/competition/test_sgp.py | 366 +++++++++++++++ 5 files changed, 938 insertions(+), 27 deletions(-) create mode 100644 opensoar/competition/sgp.py create mode 100644 tests/competition/test_sgp.py diff --git a/opensoar/competition/daily_results_page.py b/opensoar/competition/daily_results_page.py index b1555da..9e8e07d 100644 --- a/opensoar/competition/daily_results_page.py +++ b/opensoar/competition/daily_results_page.py @@ -1,9 +1,12 @@ +import requests +from bs4 import BeautifulSoup +import os +import time +import requests import operator import os from abc import ABC, abstractmethod from typing import List -from urllib.request import URLopener -from urllib.request import urlopen import time from bs4 import BeautifulSoup @@ -35,22 +38,35 @@ def set_igc_directory(self, target_directory, competition_name, plane_class, dat date.strftime('%d-%m-%Y')) def _get_html_soup(self) -> BeautifulSoup: - # fix problem with SSL certificates - # https://stackoverflow.com/questions/30551400/disable-ssl-certificate-validation-in-mechanize#35960702 + """ + Get a BeautifulSoup object from the URL. + + Returns: + BeautifulSoup object containing the parsed HTML + """ + if not self._html_soup: - import ssl try: - _create_unverified_https_context = ssl._create_unverified_context - except AttributeError: - # Legacy Python that doesn't verify HTTPS certificates by default - pass - else: - # Handle target environment that doesn't support HTTPS verification - ssl._create_default_https_context = _create_unverified_https_context - - # get entire html of page - html = urlopen(self.url).read() - self._html_soup = BeautifulSoup(html, "html.parser") + # Use requests with verify=True for secure connections + # In production, you should ALWAYS verify SSL certificates + response = requests.get(self.url, timeout=30) + response.raise_for_status() # Raise exception for 4XX/5XX status codes + + # Parse the HTML with BeautifulSoup + self._html_soup = BeautifulSoup(response.text, "html.parser") + + except requests.exceptions.SSLError: + # Only if absolutely necessary, you can disable verification + # But this should be a last resort and logged as a security concern + print("SSL verification failed. Attempting with verification disabled.") + response = requests.get(self.url, verify=False, timeout=30) + response.raise_for_status() + self._html_soup = BeautifulSoup(response.text, "html.parser") + + except requests.exceptions.RequestException as e: + print(f"Error fetching URL {self.url}: {e}") + raise + return self._html_soup def igc_file_name(self, competition_id: str) -> str: @@ -75,21 +91,46 @@ def igc_file_path(self, competition_id: str) -> str: def download_flight(self, igc_url: str, competition_id: str) -> str: """ Download flight and return file_path - - :param igc_url: - :param competition_id: - :return: + + Args: + igc_url: URL to download the IGC file + competition_id: Competition ID used to name the file + + Returns: + str: Path to the downloaded file """ - - # make directory if necessary + + # Make directory if necessary if not os.path.exists(self._igc_directory): os.makedirs(self._igc_directory) - + file_path = self.igc_file_path(competition_id) - while not os.path.exists(file_path): - URLopener().retrieve(igc_url, file_path) - time.sleep(0.1) - + + # Attempt to download the file + max_retries = 3 + retry_count = 0 + + while not os.path.exists(file_path) and retry_count < max_retries: + try: + response = requests.get(igc_url, timeout=30) + response.raise_for_status() # Raise an exception for HTTP errors + + # Write the content to the file + with open(file_path, 'wb') as f: + f.write(response.content) + + # Verify file was created + if not os.path.exists(file_path): + raise FileNotFoundError(f"File was not created at {file_path}") + + except (requests.exceptions.RequestException, FileNotFoundError) as e: + print(f"Download attempt {retry_count + 1} failed: {e}") + retry_count += 1 + time.sleep(1) # Longer delay between retries + + if not os.path.exists(file_path): + raise RuntimeError(f"Failed to download file from {igc_url} after {max_retries} attempts") + return file_path @abstractmethod diff --git a/opensoar/competition/sgp.py b/opensoar/competition/sgp.py new file mode 100644 index 0000000..936f564 --- /dev/null +++ b/opensoar/competition/sgp.py @@ -0,0 +1,502 @@ +""" +Helper functions for SGP (Sailplane Grand Prix) competitions. +This module provides functionality to access competition data from the SGP API endpoints +and download IGC files for analysis. +""" +import json +import re +import datetime +from typing import List, Dict, Tuple, Optional +import urllib.request +import logging + +from aerofiles.igc import Reader + +from opensoar.competition.competition_day import CompetitionDay +from opensoar.competition.competitor import Competitor +from opensoar.competition.daily_results_page import DailyResultsPage +from opensoar.task.task import Task +from opensoar.task.waypoint import Waypoint +from opensoar.task.race_task import RaceTask + +logger = logging.getLogger(__name__) + +class SGPDaily(DailyResultsPage): + """ + Helper class for dealing with SGP (Sailplane Grand Prix) daily result pages. + This class interfaces with the SGP API to retrieve competition data. + """ + + # API endpoints + BASE_API_URL = "https://www.crosscountry.aero/c/sgp/rest" + FLIGHT_DOWNLOAD_URL = "https://www.crosscountry.aero/flight/download/sgp" + + def __init__(self, url: str): + """ + Initialize with the URL to the SGP API. + + Args: + url: URL to the SGP API, in format: + https://www.crosscountry.aero/c/sgp/rest/day/{comp_id}/{day_id} + or + https://www.crosscountry.aero/c/sgp/rest/comp/{comp_id} + """ + super().__init__(url) + + # Extract competition ID and day ID from the URL + self.competition_id = None + self.day_id = None + self._extract_ids_from_url(url) + + # API data will be loaded on demand + self._competition_data = None + self._day_data = None + + def _extract_ids_from_url(self, url: str): + """ + Extract competition ID and day ID from the URL. + + Args: + url: URL to the SGP API + """ + # Try to match day URL pattern + day_pattern = r'crosscountry\.aero/c/sgp/rest/day/(\d+)/(\d+)' + day_match = re.search(day_pattern, url) + + if day_match: + self.competition_id = int(day_match.group(1)) + self.day_id = int(day_match.group(2)) + logger.info(f"Extracted competition ID: {self.competition_id}, day ID: {self.day_id}") + return + + # Try to match competition URL pattern + comp_pattern = r'crosscountry\.aero/c/sgp/rest/comp/(\d+)' + comp_match = re.search(comp_pattern, url) + + if comp_match: + self.competition_id = int(comp_match.group(1)) + logger.info(f"Extracted competition ID: {self.competition_id}") + return + + # If it's an sgp.aero URL, we'll need to discover the competition ID + sgp_pattern = r'sgp\.aero/([^/]+)' + sgp_match = re.search(sgp_pattern, url) + + if sgp_match: + self.competition_name = sgp_match.group(1) + logger.info(f"Found SGP competition name: {self.competition_name}, will need to discover API endpoints") + return + + # If no patterns match, warn but don't fail yet + logger.warning(f"Could not extract IDs from URL: {url}") + + def _get_competition_data(self) -> Dict: + """ + Fetch competition data from the SGP API. + + Returns: + Dictionary with competition data + """ + if self._competition_data is not None: + return self._competition_data + + if not self.competition_id: + raise ValueError("No competition ID available") + + url = f"{self.BASE_API_URL}/comp/{self.competition_id}" + try: + with urllib.request.urlopen(url) as response: + data = json.loads(response.read()) + self._competition_data = data + return data + except Exception as e: + logger.error(f"Error fetching competition data: {e}") + raise + + def _get_day_data(self) -> Dict: + """ + Fetch day data from the SGP API. + + Returns: + Dictionary with day data + """ + if self._day_data is not None: + return self._day_data + + if not self.competition_id: + raise ValueError("No competition ID available") + + if not self.day_id: + # We need to select a day + comp_data = self._get_competition_data() + days = comp_data.get('i', []) + # Only get the days that have a winner + days = [day_data for day_data in days if day_data.get('w')] + + if not days: + raise ValueError("No competition days found") + + # Sort days by date and get the latest + sorted_days = sorted(days, key=lambda d: d.get('d', ''), reverse=True) + self.day_id = sorted_days[0].get('i') + + if not self.day_id: + raise ValueError("Could not determine day ID") + + logger.info(f"Selected day ID: {self.day_id}") + + url = f"{self.BASE_API_URL}/day/{self.competition_id}/{self.day_id}" + try: + with urllib.request.urlopen(url) as response: + data = json.loads(response.read()) + self._day_data = data + return data + except Exception as e: + logger.error(f"Error fetching day data: {e}") + raise + + def _get_competition_day_info(self) -> Tuple[str, datetime.date, str]: + """ + Get competition name, date, and class. + + Returns: + tuple containing (competition_name, date, class_name) + """ + # Get competition data + comp_data = self._get_competition_data() + + if not comp_data: + raise ValueError("No competition data available") + + comp_info = comp_data.get('c', {}) + competition_name = comp_info.get('t', 'Unknown Competition') + + # Get day data + day_data = self._get_day_data() + + # Extract date + timestamp_ms = day_data.get('d') + if timestamp_ms: + try: + day_date = datetime.date.fromtimestamp(timestamp_ms / 1000) + except ValueError: + logger.warning(f"Could not parse date: {timestamp_ms}") + day_date = datetime.date.today() + else: + day_date = datetime.date.today() + + # Use the class name from competition info + class_name = "SGP" # SGP typically has just one class + + return competition_name, day_date, class_name + + def _get_competitors_info(self, include_hc_competitors: bool = True, include_dns_competitors: bool = False) -> List[Dict]: + """ + Extract competitor information from the SGP API. + + Args: + include_hc_competitors: Whether to include hors-concours competitors + include_dns_competitors: Whether to include competitors who did not start + + Returns: + List of dictionaries with competitor information + """ + # Get day data + competition_data = self._get_competition_data() + day_data = self._get_day_data() + + competitors_info = [] + + # Get pilots info + pilots = competition_data.get('p', {}) + + # Get results for the day + results = day_data.get('r', {}).get('s', []) + + for result in results: + pilot_id = result.get('h') + pilot_info = pilots.get(str(pilot_id), {}) + + status = result.get('r', '') + if status in ['DNS', 'DNF'] and not include_dns_competitors: + continue + + # Check for HC (hors concours) status + is_hc = isinstance(result.get('w'), int) and result.get('w') == 0 + if is_hc and not include_hc_competitors: + continue + + # Extract ranking + try: + ranking = int(result.get('q', 0)) # Position in results + except (ValueError, TypeError): + ranking = result.get('q', 0) + + # Get competition ID (CN) + competition_id = result.get('j', '') + + # Extract pilot name + first_name = pilot_info.get('f', '') + last_name = pilot_info.get('l', '') + pilot_name = f"{first_name} {last_name}".strip() + + # Extract glider model + plane_model = pilot_info.get('s', '') + + # Extract IGC URL if available + igc_id = result.get('w') + igc_url = f"{self.FLIGHT_DOWNLOAD_URL}/{igc_id}" if igc_id else None + + competitors_info.append({ + 'ranking': ranking, + 'competition_id': competition_id, + 'igc_url': igc_url, + 'pilot_name': pilot_name, + 'plane_model': plane_model + }) + + return competitors_info + + def get_available_days(self) -> List[Dict]: + """ + Get all available days/tasks for this competition. + + Returns: + List of dictionaries with day information + """ + comp_data = self._get_competition_data() + days = comp_data.get('i', []) + + # Filter out practice days if needed + race_days = [day for day in days if day.get('y') == 1] # Type 1 seems to be race days + + return race_days + + def generate_competition_day(self, target_directory: str, download_progress=None, start_time_buffer: int = 0): + """ + Get competition day with all flights from the SGP API. + + Args: + target_directory: Directory in which the IGC files are saved + download_progress: Optional progress function + start_time_buffer: Optional relaxation on the start time in seconds + + Returns: + CompetitionDay object + """ + # Set the directory for downloaded IGC files + competition_name, date, class_name = self._get_competition_day_info() + self.set_igc_directory(target_directory, competition_name, class_name, date) + + # Get the day data + day_data = self._get_day_data() + + # Get competitors information + competitors_info = self._get_competitors_info() + + # Get task information from the day data + task_data = day_data.get('k', {}).get('data', {}) + waypoints = self._extract_waypoints(task_data) + + # Extract task start time + start_opening = self._extract_start_opening(day_data) + + # Create task object (assuming Race Task for SGP) + # Get timezone information if available + timezone_offset = day_data.get('r', {}).get('z') + timezone = timezone_offset // 3600000 if timezone_offset else None # Convert from ms to hours + + task = RaceTask(waypoints, timezone, start_opening, start_time_buffer) + + # Download flights and create Competitor objects + competitors = [] + files_downloaded = 0 + total_competitors = len(competitors_info) + + for competitor_info in competitors_info: + competition_id = competitor_info['competition_id'] + igc_url = competitor_info['igc_url'] + ranking = competitor_info['ranking'] + plane_model = competitor_info['plane_model'] + pilot_name = competitor_info['pilot_name'] + + if igc_url is None: + logger.info(f"No IGC file available for {competition_id}") + continue + + try: + file_path = self.download_flight(igc_url, competition_id) + files_downloaded += 1 + + # Try to read the IGC file with different encodings + try: + # Try utf-8 + with open(file_path, 'r', encoding='utf-8') as f: + parsed_igc = Reader(skip_duplicates=True).read(f) + except UnicodeDecodeError: + # If not utf-8 use latin1 + with open(file_path, 'r', encoding='latin1') as f: + parsed_igc = Reader(skip_duplicates=True).read(f) + + # Create and add the competitor + trace = parsed_igc['fix_records'][1] + competitor = Competitor(trace, competition_id, plane_model, ranking, pilot_name) + competitors.append(competitor) + + # Update progress if callback provided + if download_progress is not None: + download_progress(files_downloaded, total_competitors) + + except Exception as e: + logger.error(f"Error processing competitor {competition_id}: {e}") + continue + + # Create CompetitionDay object with competitors and task + competition_day = CompetitionDay(competition_name, date, class_name, competitors, task) + + return competition_day + + def _extract_waypoints(self, task_data: Dict) -> List[Waypoint]: + """ + Extract waypoints from the task data. + + Args: + task_data: Dictionary containing task data + + Returns: + List of Waypoint objects + """ + waypoints = [] + + # Extract turnpoints from task data + turnpoints = task_data.get('g', []) + + for tp_idx, tp in enumerate(turnpoints): + name = tp.get('n', f"TP{tp_idx}") + lat = tp.get('a') # Latitude + lon = tp.get('o') # Longitude + + if lat is None or lon is None: + logger.warning(f"Skipping waypoint {name}: missing coordinates") + continue + + # Get waypoint type + wp_type = tp.get('y', 'cylinder') + radius = tp.get('r', 500) # Default radius 500m + + # Different handling based on waypoint type + if wp_type == 'line': + # Start or finish line + is_line = True + r_min = None + angle_min = None + r_max = radius + angle_max = 90 # Standard line is 90 degrees to bisector + + # Determine if start or finish based on position + if tp_idx == 0: + # Start line + sector_orientation = "next" + elif tp_idx == len(turnpoints) - 1: + # Finish line + sector_orientation = "previous" + else: + # Unlikely, but default to symmetrical + sector_orientation = "symmetrical" + else: + # Cylinder or other point type + is_line = False + r_min = None + angle_min = None + r_max = radius + angle_max = 180 # Full cylinder + sector_orientation = "symmetrical" + + # Create Waypoint object + waypoint = Waypoint( + name=name, + latitude=lat, + longitude=lon, + r_min=r_min, + angle_min=angle_min, + r_max=r_max, + angle_max=angle_max, + is_line=is_line, + sector_orientation=sector_orientation + ) + + waypoints.append(waypoint) + + # Set orientation angles based on waypoint positions + Task.set_orientation_angles(waypoints) + + return waypoints + + def _extract_start_opening(self, day_data: Dict) -> Optional[datetime.datetime]: + """ + Extract start opening time from the day data. + + Args: + day_data: Dictionary containing day data + + Returns: + Start opening time as datetime.datetime or None if not available + """ + # Get date from day data + timestamp_ms = day_data.get('d') + if not timestamp_ms: + return None + + try: + task_date = datetime.date.fromtimestamp(timestamp_ms / 1000) + except ValueError: + logger.warning(f"Could not parse date: {timestamp_ms}") + return None + + # Get start opening time in milliseconds + start_ms = day_data.get('a') + if start_ms is None: + return None + + # Convert milliseconds to time + start_seconds = start_ms // 1000 + hours = start_seconds // 3600 + minutes = (start_seconds % 3600) // 60 + seconds = start_seconds % 60 + + start_time = datetime.time(hours, minutes, seconds) + + # Combine date and time + start_opening = datetime.datetime.combine(task_date, start_time) + + # Set timezone if available + timezone_offset = day_data.get('r', {}).get('z') + if timezone_offset: + timezone_hours = timezone_offset // 3600000 # Convert from milliseconds to hours + tz = datetime.timezone(datetime.timedelta(hours=timezone_hours)) + start_opening = start_opening.replace(tzinfo=tz) + + return start_opening + +if __name__ == "__main__": +# Direct API URL for the day + day_url = "https://www.crosscountry.aero/c/sgp/rest/day/86/1547" + +# Create a SGPDaily instance + sgp_daily = SGPDaily(day_url) + +# Directory to store IGC files + target_directory = "./bin" + +# Generate a CompetitionDay with all flights + competition_day = sgp_daily.generate_competition_day(target_directory) + +# Now you can analyze flights using the existing OpenSoar framework + for competitor in competition_day.competitors: + competitor.analyse(competition_day.task, classification_method="pysoar") + + # Work with the analyzed flight data + print(f"Competitor: {competitor.competition_id}") + if competitor.phases: + thermals = competitor.phases.thermals() + print(f" Number of thermals: {len(thermals)}") diff --git a/requirements.txt b/requirements.txt index 7021e80..89cf217 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ aerofiles~=1.4.0 beautifulsoup4~=4.6.0 geojson>=3.0.0 shapely>2.0.0 +requests~=2.32.3 diff --git a/setup.py b/setup.py index 61730ee..10073dd 100644 --- a/setup.py +++ b/setup.py @@ -19,5 +19,6 @@ 'pyproj>=3.4.1', 'geojson>=3.0.0', 'shapely>2.0.0', + 'requests~=2.32.3', ] ) diff --git a/tests/competition/test_sgp.py b/tests/competition/test_sgp.py new file mode 100644 index 0000000..b70f84f --- /dev/null +++ b/tests/competition/test_sgp.py @@ -0,0 +1,366 @@ +""" +Unit tests for SGP (Sailplane Grand Prix) module. + +This module tests the functionality for accessing SGP API endpoints +and downloading/analyzing IGC files. +""" +import datetime +import json +import unittest +from unittest import mock +from pathlib import Path + +from opensoar.competition.sgp import SGPDaily +from opensoar.task.race_task import RaceTask + + +class MockResponse: + """Mock urllib.request.urlopen response object.""" + + def __init__(self, data, status_code=200): + self.data = json.dumps(data).encode('utf-8') + self.status = status_code + + def read(self): + return self.data + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +class TestSGPDaily(unittest.TestCase): + """Tests for SGPDaily class.""" + + def setUp(self): + """Set up test fixtures.""" + self.day_url = "https://www.crosscountry.aero/c/sgp/rest/day/86/1547" + self.comp_url = "https://www.crosscountry.aero/c/sgp/rest/comp/86" + + # Sample competition data + self.comp_data = { + "p": { + "123": { + "f": "John", + "l": "Doe", + "s": "LS8" + }, + "124": { + "f": "Jane", + "l": "Smith", + "s": "ASG-29" + }, + "125": { + "f": "Bob", + "l": "Brown", + "s": "Ventus" + } + }, + "c": { + "t": "Test SGP Competition", + "l": "Test Location" + }, + "i": [ + { + "i": 1547, + "d": 1618012800000, # 2021-04-10 + "y": 1, # Race day + "w": "JV" # Winner defined + }, + { + "i": 1548, + "d": 1618099200000, # 2021-04-11 + "y": 1, # Race day + "w": None # No winner defined + } + ] + } + + # Sample day data + self.day_data = { + "d": 1618012800000, # 2021-04-10 + "a": 36000000, # Start time 10:00:00 (in milliseconds) + "r": { + "z": 7200000, # UTC+2 in milliseconds + "s": [ + { + "h": 123, + "j": "ABC", + "q": 1, + "r": "", + "w": 456 + }, + { + "h": 124, + "j": "DEF", + "q": 2, + "r": "", + "w": 457 + }, + { + "h": 125, + "j": "GHI", + "q": 0, + "r": "DNS", + "w": 0 + } + ] + }, + "k": { + "data": { + "g": [ + { + "n": "Start", + "a": 51.0, + "o": 10.0, + "y": "line", + "r": 1000 + }, + { + "n": "TP1", + "a": 51.1, + "o": 10.1, + "y": "cylinder", + "r": 500 + }, + { + "n": "Finish", + "a": 51.0, + "o": 10.0, + "y": "line", + "r": 1000 + } + ] + } + } + } + + # Create a temporary directory for downloaded files + self.temp_dir = Path("./test_igc_files") + self.temp_dir.mkdir(exist_ok=True) + + def tearDown(self): + """Clean up after tests.""" + # Remove test files + for file in self.temp_dir.glob("*.igc"): + file.unlink() + + # Remove test directory + self.temp_dir.rmdir() + + def test_extract_ids_from_url_day(self): + """Test extraction of competition and day IDs from day URL.""" + sgp = SGPDaily(self.day_url) + + self.assertEqual(sgp.competition_id, 86) + self.assertEqual(sgp.day_id, 1547) + + def test_extract_ids_from_url_comp(self): + """Test extraction of competition ID from competition URL.""" + sgp = SGPDaily(self.comp_url) + + self.assertEqual(sgp.competition_id, 86) + self.assertIsNone(sgp.day_id) + + @mock.patch('urllib.request.urlopen') + def test_get_competition_data(self, mock_urlopen): + """Test fetching competition data from the API.""" + mock_urlopen.return_value = MockResponse(self.comp_data) + + sgp = SGPDaily(self.comp_url) + data = sgp._get_competition_data() + + self.assertEqual(data, self.comp_data) + mock_urlopen.assert_called_once_with(f"{SGPDaily.BASE_API_URL}/comp/86") + + @mock.patch('urllib.request.urlopen') + def test_get_day_data(self, mock_urlopen): + """Test fetching day data from the API.""" + mock_urlopen.return_value = MockResponse(self.day_data) + + sgp = SGPDaily(self.day_url) + data = sgp._get_day_data() + + self.assertEqual(data, self.day_data) + mock_urlopen.assert_called_once_with(f"{SGPDaily.BASE_API_URL}/day/86/1547") + + @mock.patch('urllib.request.urlopen') + def test_get_day_data_without_day_id(self, mock_urlopen): + """Test fetching day data when day ID is not provided.""" + mock_urlopen.side_effect = [ + MockResponse(self.comp_data), + MockResponse(self.day_data) + ] + + sgp = SGPDaily(self.comp_url) + data = sgp._get_day_data() + + self.assertEqual(data, self.day_data) + self.assertEqual(sgp.day_id, 1547) # Should select the latest day with a winner + + expected_calls = [ + mock.call(f"{SGPDaily.BASE_API_URL}/comp/86"), + mock.call(f"{SGPDaily.BASE_API_URL}/day/86/1547") + ] + mock_urlopen.assert_has_calls(expected_calls) + + @mock.patch('urllib.request.urlopen') + def test_get_competition_day_info(self, mock_urlopen): + """Test retrieving competition day information.""" + mock_urlopen.side_effect = [ + MockResponse(self.comp_data), + MockResponse(self.day_data) + ] + + sgp = SGPDaily(self.day_url) + name, date, class_name = sgp._get_competition_day_info() + + self.assertEqual(name, "Test SGP Competition") + self.assertEqual(date, datetime.date(2021, 4, 10)) + self.assertEqual(class_name, "SGP") + + @mock.patch('urllib.request.urlopen') + def test_get_competitors_info(self, mock_urlopen): + """Test retrieving competitor information.""" + # Need to mock both competition data and day data since both are used + mock_urlopen.side_effect = [ + MockResponse(self.comp_data), # For _get_competition_data call + MockResponse(self.day_data) # For _get_day_data call + ] + + sgp = SGPDaily(self.day_url) + # Force caching of competition data + sgp._competition_data = self.comp_data + # Force caching of day data + sgp._day_data = self.day_data + + competitors = sgp._get_competitors_info(include_dns_competitors=False) + + self.assertEqual(len(competitors), 2) # Should exclude DNS competitor + + self.assertEqual(competitors[0]['competition_id'], "ABC") + self.assertEqual(competitors[0]['pilot_name'], "John Doe") + self.assertEqual(competitors[0]['plane_model'], "LS8") + self.assertEqual(competitors[0]['ranking'], 1) + self.assertEqual(competitors[0]['igc_url'], f"{SGPDaily.FLIGHT_DOWNLOAD_URL}/456") + + # Test including DNS competitors + competitors = sgp._get_competitors_info(include_dns_competitors=True) + self.assertEqual(len(competitors), 3) # Should include DNS competitor + + @mock.patch('urllib.request.urlopen') + def test_get_available_days(self, mock_urlopen): + """Test retrieving available competition days.""" + mock_urlopen.return_value = MockResponse(self.comp_data) + + sgp = SGPDaily(self.comp_url) + days = sgp.get_available_days() + + self.assertEqual(len(days), 2) + self.assertEqual(days[0]['i'], 1547) + self.assertEqual(days[1]['i'], 1548) + + def test_extract_waypoints(self): + """Test extracting waypoints from task data.""" + task_data = self.day_data['k']['data'] + + sgp = SGPDaily(self.day_url) + waypoints = sgp._extract_waypoints(task_data) + + self.assertEqual(len(waypoints), 3) + + # Check start point + self.assertEqual(waypoints[0].name, "Start") + self.assertEqual(waypoints[0].latitude, 51.0) + self.assertEqual(waypoints[0].longitude, 10.0) + self.assertEqual(waypoints[0].r_max, 1000) + self.assertEqual(waypoints[0].angle_max, 90) + self.assertTrue(waypoints[0].is_line) + self.assertEqual(waypoints[0].sector_orientation, "next") + + # Check turnpoint + self.assertEqual(waypoints[1].name, "TP1") + self.assertEqual(waypoints[1].latitude, 51.1) + self.assertEqual(waypoints[1].longitude, 10.1) + self.assertEqual(waypoints[1].r_max, 500) + self.assertEqual(waypoints[1].angle_max, 180) + self.assertFalse(waypoints[1].is_line) + + # Check finish point + self.assertEqual(waypoints[2].name, "Finish") + self.assertEqual(waypoints[2].r_max, 1000) + self.assertTrue(waypoints[2].is_line) + self.assertEqual(waypoints[2].sector_orientation, "previous") + + def test_extract_start_opening(self): + """Test extracting start opening time from day data.""" + sgp = SGPDaily(self.day_url) + start_opening = sgp._extract_start_opening(self.day_data) + + expected_datetime = datetime.datetime( + 2021, 4, 10, 10, 0, 0, + tzinfo=datetime.timezone(datetime.timedelta(hours=2)) + ) + self.assertEqual(start_opening, expected_datetime) + + @mock.patch('urllib.request.urlopen') + @mock.patch('opensoar.competition.sgp.SGPDaily.download_flight') + @mock.patch('opensoar.competition.sgp.Reader') + @mock.patch('builtins.open', new_callable=mock.mock_open) + def test_generate_competition_day(self, mock_open, mock_reader, mock_download, mock_urlopen): + """Test generating a CompetitionDay object from SGP data.""" + # Setup mocks + mock_urlopen.side_effect = [ + MockResponse(self.comp_data), # For _get_competition_data call + MockResponse(self.day_data), # For _get_day_data call + MockResponse(self.comp_data), # For _get_competition_day_info -> _get_competition_data + MockResponse(self.day_data), # For _get_competition_day_info -> _get_day_data + MockResponse(self.day_data), # For additional _get_day_data call + MockResponse(self.comp_data), # For _get_competitors_info -> _get_competition_data + MockResponse(self.day_data) # For _get_competitors_info -> _get_day_data + ] + + # Cache data to prevent too many API calls + sgp = SGPDaily(self.day_url) + sgp._competition_data = self.comp_data + sgp._day_data = self.day_data + + # Mock downloading IGC files + mock_download.side_effect = lambda url, cn: f"{self.temp_dir}/{cn}.igc" + + # Mock IGC file content and reading + mock_igc_content = "AFILETYPENM" # Minimal IGC content for testing + mock_open.return_value.read.return_value = mock_igc_content + + # Mock the Reader class and its read method + mock_parser = mock.MagicMock() + mock_parser.read.return_value = { + 'fix_records': (None, [{'time': '101010', 'lat': 51.0, 'lon': 10.0}]) + } + mock_reader.return_value = mock_parser + + # Create SGPDaily instance and generate competition day + competition_day = sgp.generate_competition_day(str(self.temp_dir)) + + # Verify results + self.assertEqual(competition_day.name, "Test SGP Competition") + self.assertEqual(competition_day.date, datetime.date(2021, 4, 10)) + self.assertEqual(competition_day.plane_class, "SGP") + + # Verify competitors were created + self.assertEqual(len(competition_day.competitors), 2) + self.assertEqual(competition_day.competitors[0].competition_id, "ABC") + self.assertEqual(competition_day.competitors[0].pilot_name, "John Doe") + + # Verify task was created correctly + self.assertIsInstance(competition_day.task, RaceTask) + self.assertEqual(len(competition_day.task.waypoints), 3) + + # Verify that files were properly opened and read + mock_open.assert_called() + mock_reader.return_value.read.assert_called() + +if __name__ == "__main__": + unittest.main() From 7cfbe90497954c7ca935dff077bb92ed5fb4b81c Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 18 May 2025 10:01:06 +0200 Subject: [PATCH 2/9] add test for sgp api --- tests/competition/test_sgp_api.py | 309 ++++++++++++++++++++++++++++++ 1 file changed, 309 insertions(+) create mode 100644 tests/competition/test_sgp_api.py diff --git a/tests/competition/test_sgp_api.py b/tests/competition/test_sgp_api.py new file mode 100644 index 0000000..dddd8f3 --- /dev/null +++ b/tests/competition/test_sgp_api.py @@ -0,0 +1,309 @@ +""" +Integration tests for SGP API structure verification. + +These tests make real API calls to the SGP endpoints and verify that the +expected structure is present. These will help detect unexpected API changes. + +NOTE: These tests require internet connectivity and depend on the +actual SGP API being available. +""" +import unittest +import datetime +import json +import logging +from pathlib import Path +import re + +from opensoar.competition.sgp import SGPDaily + + +# Configure logging +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + + +class TestSGPApiIntegration(unittest.TestCase): + """ + Integration tests that verify the SGP API structure using real API calls. + + These tests depend on the actual API and should be run periodically to + ensure that the API structure hasn't changed unexpectedly. + """ + + def setUp(self): + """Set up test fixtures.""" + # Use valid API endpoint URLs directly rather than web URLs + # The web URLs shown in the error message don't match the expected format + self.comp_id = 86 # Use a known working competition ID + self.day_id = 1547 # Use a known working day ID + + # Use the direct API endpoints instead of web URLs + self.comp_api_url = f"https://www.crosscountry.aero/c/sgp/rest/comp/{self.comp_id}" + self.day_api_url = f"https://www.crosscountry.aero/c/sgp/rest/day/{self.comp_id}/{self.day_id}" + + # Create SGP daily instances with the API URLs + self.sgp_comp = SGPDaily(self.comp_api_url) + self.sgp_day = SGPDaily(self.day_api_url) + + # Verify the URLs were correctly parsed + self.assertEqual(self.sgp_comp.competition_id, self.comp_id) + self.assertEqual(self.sgp_day.competition_id, self.comp_id) + self.assertEqual(self.sgp_day.day_id, self.day_id) + + # Directory to save response data for inspection if needed + self.output_dir = Path("./test_output") + self.output_dir.mkdir(exist_ok=True) + + def tearDown(self): + """Clean up any files created during tests.""" + # Keep the output files for manual inspection if needed + pass + + def _save_response(self, data, filename): + """Save response data to a file for manual inspection.""" + path = self.output_dir / filename + with open(path, 'w') as f: + json.dump(data, f, indent=2) + logger.info(f"Saved response data to {path}") + + def test_competition_data_structure(self): + """Test the structure of the competition data response.""" + try: + # Get competition data from the real API + comp_data = self.sgp_comp._get_competition_data() + + # Save the response for manual inspection + self._save_response(comp_data, f"comp_{self.comp_id}_response.json") + + # Verify top-level structure + self.assertIn('p', comp_data, "Missing pilots data ('p' key)") + self.assertIn('c', comp_data, "Missing competition data ('c' key)") + self.assertIn('i', comp_data, "Missing days info ('i' key)") + + # Verify competition info + comp_info = comp_data['c'] + self.assertIn('t', comp_info, "Missing competition title ('t' key)") + self.assertIn('l', comp_info, "Missing location ('l' key)") + + # Verify pilots structure + pilots = comp_data['p'] + self.assertTrue(pilots, "Pilots dictionary is empty") + + # Verify pilot data structure (check first pilot) + pilot_id = next(iter(pilots)) + pilot = pilots[pilot_id] + self.assertIn('f', pilot, "Missing pilot first name ('f' key)") + self.assertIn('l', pilot, "Missing pilot last name ('l' key)") + self.assertIn('s', pilot, "Missing pilot sailplane ('s' key)") + + # Log some sample data for verification + logger.info(f"Competition: {comp_info['t']} at {comp_info['l']}") + logger.info(f"Number of pilots: {len(pilots)}") + + # Verify competition days + days = comp_data['i'] + self.assertTrue(isinstance(days, list), "Days info is not a list") + self.assertTrue(days, "No competition days found") + + # Verify day structure + day = days[0] + self.assertIn('i', day, "Missing day ID ('i' key)") + self.assertIn('d', day, "Missing day date ('d' key)") + self.assertIn('y', day, "Missing day type flag ('y' key)") + # Note: 'w' key might not be present for all days + + logger.info(f"Found {len(days)} days in the competition") + + except Exception as e: + self.fail(f"Error testing competition data structure: {str(e)}") + + def test_day_data_structure(self): + """Test the structure of the day data response.""" + try: + # Get day data from the real API + day_data = self.sgp_day._get_day_data() + + # Save the response for manual inspection + self._save_response(day_data, f"day_{self.day_id}_response.json") + + # Verify top-level structure + self.assertIn('d', day_data, "Missing day date ('d' key)") + self.assertIn('a', day_data, "Missing start time ('a' key)") + self.assertIn('r', day_data, "Missing results data ('r' key)") + self.assertIn('k', day_data, "Missing task data ('k' key)") + + # Convert day timestamp to readable date + day_timestamp = day_data['d'] + day_date = datetime.datetime.fromtimestamp(day_timestamp / 1000).date() + logger.info(f"Day date: {day_date}") + + # Verify results structure + results = day_data['r'] + self.assertIn('z', results, "Missing timezone ('z' key)") + self.assertIn('s', results, "Missing standings ('s' key)") + + # Verify timezone + timezone_ms = results['z'] + timezone_hours = timezone_ms / (1000 * 60 * 60) + logger.info(f"Timezone: UTC{'+' if timezone_hours >= 0 else ''}{timezone_hours}") + + # Verify standings + standings = results['s'] + self.assertTrue(isinstance(standings, list), "Standings is not a list") + if not standings: + logger.warning("No standings found in day data") + else: + # Verify standing structure + standing = standings[0] + self.assertIn('h', standing, "Missing pilot ID ('h' key)") + self.assertIn('j', standing, "Missing competition ID ('j' key)") + self.assertIn('q', standing, "Missing ranking ('q' key)") + self.assertIn('r', standing, "Missing result status ('r' key)") + self.assertIn('w', standing, "Missing flight ID ('w' key)") + + logger.info(f"Found {len(standings)} results for the day") + + # Verify task structure + task = day_data['k'] + self.assertIn('data', task, "Missing task data ('data' key)") + + task_data = task['data'] + self.assertIn('g', task_data, "Missing waypoints list ('g' key)") + + # Verify waypoints + waypoints = task_data['g'] + self.assertTrue(isinstance(waypoints, list), "Waypoints is not a list") + self.assertTrue(waypoints, "No waypoints found") + + # Verify waypoint structure + waypoint = waypoints[0] + self.assertIn('n', waypoint, "Missing waypoint name ('n' key)") + self.assertIn('a', waypoint, "Missing latitude ('a' key)") + self.assertIn('o', waypoint, "Missing longitude ('o' key)") + self.assertIn('y', waypoint, "Missing type ('y' key)") + self.assertIn('r', waypoint, "Missing radius ('r' key)") + + logger.info(f"Found {len(waypoints)} waypoints in the task") + + # Log the task details + waypoint_names = [wp['n'] for wp in waypoints] + logger.info(f"Task waypoints: {', '.join(waypoint_names)}") + + except Exception as e: + self.fail(f"Error testing day data structure: {str(e)}") + + def test_available_days(self): + """Test retrieval of available days from the competition.""" + try: + # Get available days + days = self.sgp_comp.get_available_days() + + # Save the response for manual inspection + self._save_response(days, f"comp_{self.comp_id}_available_days.json") + + # Verify days structure + self.assertTrue(isinstance(days, list), "Available days is not a list") + self.assertTrue(days, "No available days found") + + for day in days: + self.assertIn('i', day, "Missing day ID ('i' key)") + self.assertIn('d', day, "Missing day date ('d' key)") + self.assertIn('y', day, "Missing day type flag ('y' key)") + + # Convert day timestamp to readable date + day_timestamp = day['d'] + logger.info(day_timestamp) + day_date = datetime.datetime.strptime(day_timestamp, "%Y-%m-%d").date() + day_status = "Race day" if day['y'] == 1 else "Non-race day" + + logger.info(f"Day {day['i']}: {day_date} - {day_status}") + + except Exception as e: + self.fail(f"Error testing available days: {str(e)}") + + def test_competitors_info(self): + """Test retrieval of competitor information from the day data.""" + try: + # Get competitors info + competitors = self.sgp_day._get_competitors_info(include_dns_competitors=True) + + # Save the response for manual inspection + self._save_response(competitors, f"day_{self.day_id}_competitors.json") + + # Verify competitors structure + self.assertTrue(isinstance(competitors, list), "Competitors is not a list") + + if not competitors: + logger.warning("No competitors found in day data") + return + + for competitor in competitors: + self.assertIn('competition_id', competitor, "Missing competition_id field") + self.assertIn('pilot_name', competitor, "Missing pilot_name field") + self.assertIn('plane_model', competitor, "Missing plane_model field") + self.assertIn('ranking', competitor, "Missing ranking field") + # IGC URL may be None for DNF/DNS competitors + self.assertIn('igc_url', competitor, "Missing igc_url field") + + logger.info(f"Competitor {competitor['competition_id']}: " + f"{competitor['pilot_name']} flying {competitor['plane_model']}") + + except Exception as e: + self.fail(f"Error testing competitors info: {str(e)}") + + +class TestSGPUrlHandling(unittest.TestCase): + """Test proper URL handling and ID extraction for SGP URLs.""" + + def test_api_url_pattern(self): + """Test extraction of IDs from direct API URLs.""" + # Direct REST API URLs + comp_url = "https://www.crosscountry.aero/c/sgp/rest/comp/86" + day_url = "https://www.crosscountry.aero/c/sgp/rest/day/86/1547" + + sgp_comp = SGPDaily(comp_url) + sgp_day = SGPDaily(day_url) + + self.assertEqual(sgp_comp.competition_id, 86) + self.assertIsNone(sgp_comp.day_id) + + self.assertEqual(sgp_day.competition_id, 86) + self.assertEqual(sgp_day.day_id, 1547) + + def test_web_url_pattern(self): + """ + Test extraction of IDs from web URLs. + + This test documents the current handling of web URLs and may fail + if the SGP class doesn't correctly handle these URL patterns. + """ + # Current web URLs (based on error messages) + web_comp_url = "https://www.crosscountry.aero/c/sgp/overview/127" + web_day_url = "https://www.crosscountry.aero/c/sgp/task/127/day/1925/overview" + + # Extract IDs using regex patterns + comp_pattern = r"crosscountry\.aero/c/sgp/(?:overview|task)/(\d+)" + day_pattern = r"crosscountry\.aero/c/sgp/task/\d+/day/(\d+)" + + # Extract competition ID + comp_match = re.search(comp_pattern, web_comp_url) + self.assertIsNotNone(comp_match, "Couldn't extract competition ID from web URL") + comp_id = int(comp_match.group(1)) + self.assertEqual(comp_id, 127) + + # Extract day ID + day_match = re.search(day_pattern, web_day_url) + self.assertIsNotNone(day_match, "Couldn't extract day ID from web URL") + day_id = int(day_match.group(1)) + self.assertEqual(day_id, 1925) + + # Create direct API URLs from extracted IDs + api_comp_url = f"https://www.crosscountry.aero/c/sgp/rest/comp/{comp_id}" + api_day_url = f"https://www.crosscountry.aero/c/sgp/rest/day/{comp_id}/{day_id}" + + logger.info(f"Converted web comp URL to API URL: {api_comp_url}") + logger.info(f"Converted web day URL to API URL: {api_day_url}") + + +if __name__ == "__main__": + unittest.main() From d345a074734677b9ee9410f2ab097b4f34ecfedf Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 18 May 2025 10:18:31 +0200 Subject: [PATCH 3/9] clean up after tests --- tests/competition/test_sgp_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/competition/test_sgp_api.py b/tests/competition/test_sgp_api.py index dddd8f3..47ac12f 100644 --- a/tests/competition/test_sgp_api.py +++ b/tests/competition/test_sgp_api.py @@ -8,6 +8,7 @@ actual SGP API being available. """ import unittest +import shutil import datetime import json import logging @@ -57,7 +58,7 @@ def setUp(self): def tearDown(self): """Clean up any files created during tests.""" # Keep the output files for manual inspection if needed - pass + shutil.rmtree(self.output_dir) def _save_response(self, data, filename): """Save response data to a file for manual inspection.""" From 6396227643cddc3f428f941d4ab47cb2ed5e27c4 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 25 May 2025 08:13:44 +0200 Subject: [PATCH 4/9] rename sgp to crosscountry --- .../competition/{sgp.py => crosscountry.py} | 36 +++++------ .../{test_sgp.py => test_crosscountry.py} | 60 +++++++++---------- ...st_sgp_api.py => test_crosscountry_api.py} | 29 ++++----- 3 files changed, 63 insertions(+), 62 deletions(-) rename opensoar/competition/{sgp.py => crosscountry.py} (93%) rename tests/competition/{test_sgp.py => test_crosscountry.py} (86%) rename tests/competition/{test_sgp_api.py => test_crosscountry_api.py} (93%) diff --git a/opensoar/competition/sgp.py b/opensoar/competition/crosscountry.py similarity index 93% rename from opensoar/competition/sgp.py rename to opensoar/competition/crosscountry.py index 936f564..5b8761d 100644 --- a/opensoar/competition/sgp.py +++ b/opensoar/competition/crosscountry.py @@ -1,6 +1,6 @@ """ -Helper functions for SGP (Sailplane Grand Prix) competitions. -This module provides functionality to access competition data from the SGP API endpoints +Helper functions for Crosscountry (Sailplane Grand Prix) competitions. +This module provides functionality to access competition data from the Crosscountry API endpoints and download IGC files for analysis. """ import json @@ -21,10 +21,10 @@ logger = logging.getLogger(__name__) -class SGPDaily(DailyResultsPage): +class CrosscountryDaily(DailyResultsPage): """ - Helper class for dealing with SGP (Sailplane Grand Prix) daily result pages. - This class interfaces with the SGP API to retrieve competition data. + Helper class for dealing with Crosscountry (Sailplane Grand Prix) daily result pages. + This class interfaces with the Crosscountry API to retrieve competition data. """ # API endpoints @@ -33,10 +33,10 @@ class SGPDaily(DailyResultsPage): def __init__(self, url: str): """ - Initialize with the URL to the SGP API. + Initialize with the URL to the Crosscountry API. Args: - url: URL to the SGP API, in format: + url: URL to the Crosscountry API, in format: https://www.crosscountry.aero/c/sgp/rest/day/{comp_id}/{day_id} or https://www.crosscountry.aero/c/sgp/rest/comp/{comp_id} @@ -57,7 +57,7 @@ def _extract_ids_from_url(self, url: str): Extract competition ID and day ID from the URL. Args: - url: URL to the SGP API + url: URL to the Crosscountry API """ # Try to match day URL pattern day_pattern = r'crosscountry\.aero/c/sgp/rest/day/(\d+)/(\d+)' @@ -84,7 +84,7 @@ def _extract_ids_from_url(self, url: str): if sgp_match: self.competition_name = sgp_match.group(1) - logger.info(f"Found SGP competition name: {self.competition_name}, will need to discover API endpoints") + logger.info(f"Found Crosscountry competition name: {self.competition_name}, will need to discover API endpoints") return # If no patterns match, warn but don't fail yet @@ -92,7 +92,7 @@ def _extract_ids_from_url(self, url: str): def _get_competition_data(self) -> Dict: """ - Fetch competition data from the SGP API. + Fetch competition data from the Crosscountry API. Returns: Dictionary with competition data @@ -115,7 +115,7 @@ def _get_competition_data(self) -> Dict: def _get_day_data(self) -> Dict: """ - Fetch day data from the SGP API. + Fetch day data from the Crosscountry API. Returns: Dictionary with day data @@ -186,13 +186,13 @@ def _get_competition_day_info(self) -> Tuple[str, datetime.date, str]: day_date = datetime.date.today() # Use the class name from competition info - class_name = "SGP" # SGP typically has just one class + class_name = "Default" # Crosscountry typically has just one class return competition_name, day_date, class_name def _get_competitors_info(self, include_hc_competitors: bool = True, include_dns_competitors: bool = False) -> List[Dict]: """ - Extract competitor information from the SGP API. + Extract competitor information from the Crosscountry API. Args: include_hc_competitors: Whether to include hors-concours competitors @@ -274,7 +274,7 @@ def get_available_days(self) -> List[Dict]: def generate_competition_day(self, target_directory: str, download_progress=None, start_time_buffer: int = 0): """ - Get competition day with all flights from the SGP API. + Get competition day with all flights from the Crosscountry API. Args: target_directory: Directory in which the IGC files are saved @@ -301,7 +301,7 @@ def generate_competition_day(self, target_directory: str, download_progress=None # Extract task start time start_opening = self._extract_start_opening(day_data) - # Create task object (assuming Race Task for SGP) + # Create task object (assuming Race Task for Crosscountry) # Get timezone information if available timezone_offset = day_data.get('r', {}).get('z') timezone = timezone_offset // 3600000 if timezone_offset else None # Convert from ms to hours @@ -482,14 +482,14 @@ def _extract_start_opening(self, day_data: Dict) -> Optional[datetime.datetime]: # Direct API URL for the day day_url = "https://www.crosscountry.aero/c/sgp/rest/day/86/1547" -# Create a SGPDaily instance - sgp_daily = SGPDaily(day_url) +# Create a CrosscountryDaily instance + crosscountry_daily = CrosscountryDaily(day_url) # Directory to store IGC files target_directory = "./bin" # Generate a CompetitionDay with all flights - competition_day = sgp_daily.generate_competition_day(target_directory) + competition_day = crosscountry_daily.generate_competition_day(target_directory) # Now you can analyze flights using the existing OpenSoar framework for competitor in competition_day.competitors: diff --git a/tests/competition/test_sgp.py b/tests/competition/test_crosscountry.py similarity index 86% rename from tests/competition/test_sgp.py rename to tests/competition/test_crosscountry.py index b70f84f..4c448a5 100644 --- a/tests/competition/test_sgp.py +++ b/tests/competition/test_crosscountry.py @@ -1,7 +1,7 @@ """ -Unit tests for SGP (Sailplane Grand Prix) module. +Unit tests for Crosscountry (Sailplane Grand Prix) module. -This module tests the functionality for accessing SGP API endpoints +This module tests the functionality for accessing Crosscountry API endpoints and downloading/analyzing IGC files. """ import datetime @@ -10,7 +10,7 @@ from unittest import mock from pathlib import Path -from opensoar.competition.sgp import SGPDaily +from opensoar.competition.crosscountry import CrosscountryDaily from opensoar.task.race_task import RaceTask @@ -31,8 +31,8 @@ def __exit__(self, *args): pass -class TestSGPDaily(unittest.TestCase): - """Tests for SGPDaily class.""" +class TestCrosscountryDaily(unittest.TestCase): + """Tests for CrosscountryDaily class.""" def setUp(self): """Set up test fixtures.""" @@ -59,7 +59,7 @@ def setUp(self): } }, "c": { - "t": "Test SGP Competition", + "t": "Test Crosscountry Competition", "l": "Test Location" }, "i": [ @@ -152,14 +152,14 @@ def tearDown(self): def test_extract_ids_from_url_day(self): """Test extraction of competition and day IDs from day URL.""" - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) self.assertEqual(sgp.competition_id, 86) self.assertEqual(sgp.day_id, 1547) def test_extract_ids_from_url_comp(self): """Test extraction of competition ID from competition URL.""" - sgp = SGPDaily(self.comp_url) + sgp = CrosscountryDaily(self.comp_url) self.assertEqual(sgp.competition_id, 86) self.assertIsNone(sgp.day_id) @@ -169,22 +169,22 @@ def test_get_competition_data(self, mock_urlopen): """Test fetching competition data from the API.""" mock_urlopen.return_value = MockResponse(self.comp_data) - sgp = SGPDaily(self.comp_url) + sgp = CrosscountryDaily(self.comp_url) data = sgp._get_competition_data() self.assertEqual(data, self.comp_data) - mock_urlopen.assert_called_once_with(f"{SGPDaily.BASE_API_URL}/comp/86") + mock_urlopen.assert_called_once_with(f"{CrosscountryDaily.BASE_API_URL}/comp/86") @mock.patch('urllib.request.urlopen') def test_get_day_data(self, mock_urlopen): """Test fetching day data from the API.""" mock_urlopen.return_value = MockResponse(self.day_data) - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) data = sgp._get_day_data() self.assertEqual(data, self.day_data) - mock_urlopen.assert_called_once_with(f"{SGPDaily.BASE_API_URL}/day/86/1547") + mock_urlopen.assert_called_once_with(f"{CrosscountryDaily.BASE_API_URL}/day/86/1547") @mock.patch('urllib.request.urlopen') def test_get_day_data_without_day_id(self, mock_urlopen): @@ -194,15 +194,15 @@ def test_get_day_data_without_day_id(self, mock_urlopen): MockResponse(self.day_data) ] - sgp = SGPDaily(self.comp_url) + sgp = CrosscountryDaily(self.comp_url) data = sgp._get_day_data() self.assertEqual(data, self.day_data) self.assertEqual(sgp.day_id, 1547) # Should select the latest day with a winner expected_calls = [ - mock.call(f"{SGPDaily.BASE_API_URL}/comp/86"), - mock.call(f"{SGPDaily.BASE_API_URL}/day/86/1547") + mock.call(f"{CrosscountryDaily.BASE_API_URL}/comp/86"), + mock.call(f"{CrosscountryDaily.BASE_API_URL}/day/86/1547") ] mock_urlopen.assert_has_calls(expected_calls) @@ -214,12 +214,12 @@ def test_get_competition_day_info(self, mock_urlopen): MockResponse(self.day_data) ] - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) name, date, class_name = sgp._get_competition_day_info() - self.assertEqual(name, "Test SGP Competition") + self.assertEqual(name, "Test Crosscountry Competition") self.assertEqual(date, datetime.date(2021, 4, 10)) - self.assertEqual(class_name, "SGP") + self.assertEqual(class_name, "Default") @mock.patch('urllib.request.urlopen') def test_get_competitors_info(self, mock_urlopen): @@ -230,7 +230,7 @@ def test_get_competitors_info(self, mock_urlopen): MockResponse(self.day_data) # For _get_day_data call ] - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) # Force caching of competition data sgp._competition_data = self.comp_data # Force caching of day data @@ -244,7 +244,7 @@ def test_get_competitors_info(self, mock_urlopen): self.assertEqual(competitors[0]['pilot_name'], "John Doe") self.assertEqual(competitors[0]['plane_model'], "LS8") self.assertEqual(competitors[0]['ranking'], 1) - self.assertEqual(competitors[0]['igc_url'], f"{SGPDaily.FLIGHT_DOWNLOAD_URL}/456") + self.assertEqual(competitors[0]['igc_url'], f"{CrosscountryDaily.FLIGHT_DOWNLOAD_URL}/456") # Test including DNS competitors competitors = sgp._get_competitors_info(include_dns_competitors=True) @@ -255,7 +255,7 @@ def test_get_available_days(self, mock_urlopen): """Test retrieving available competition days.""" mock_urlopen.return_value = MockResponse(self.comp_data) - sgp = SGPDaily(self.comp_url) + sgp = CrosscountryDaily(self.comp_url) days = sgp.get_available_days() self.assertEqual(len(days), 2) @@ -266,7 +266,7 @@ def test_extract_waypoints(self): """Test extracting waypoints from task data.""" task_data = self.day_data['k']['data'] - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) waypoints = sgp._extract_waypoints(task_data) self.assertEqual(len(waypoints), 3) @@ -296,7 +296,7 @@ def test_extract_waypoints(self): def test_extract_start_opening(self): """Test extracting start opening time from day data.""" - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) start_opening = sgp._extract_start_opening(self.day_data) expected_datetime = datetime.datetime( @@ -306,11 +306,11 @@ def test_extract_start_opening(self): self.assertEqual(start_opening, expected_datetime) @mock.patch('urllib.request.urlopen') - @mock.patch('opensoar.competition.sgp.SGPDaily.download_flight') - @mock.patch('opensoar.competition.sgp.Reader') + @mock.patch('opensoar.competition.crosscountry.CrosscountryDaily.download_flight') + @mock.patch('opensoar.competition.crosscountry.Reader') @mock.patch('builtins.open', new_callable=mock.mock_open) def test_generate_competition_day(self, mock_open, mock_reader, mock_download, mock_urlopen): - """Test generating a CompetitionDay object from SGP data.""" + """Test generating a CompetitionDay object from Crosscountry data.""" # Setup mocks mock_urlopen.side_effect = [ MockResponse(self.comp_data), # For _get_competition_data call @@ -323,7 +323,7 @@ def test_generate_competition_day(self, mock_open, mock_reader, mock_download, m ] # Cache data to prevent too many API calls - sgp = SGPDaily(self.day_url) + sgp = CrosscountryDaily(self.day_url) sgp._competition_data = self.comp_data sgp._day_data = self.day_data @@ -341,13 +341,13 @@ def test_generate_competition_day(self, mock_open, mock_reader, mock_download, m } mock_reader.return_value = mock_parser - # Create SGPDaily instance and generate competition day + # Create CrosscountryDaily instance and generate competition day competition_day = sgp.generate_competition_day(str(self.temp_dir)) # Verify results - self.assertEqual(competition_day.name, "Test SGP Competition") + self.assertEqual(competition_day.name, "Test Crosscountry Competition") self.assertEqual(competition_day.date, datetime.date(2021, 4, 10)) - self.assertEqual(competition_day.plane_class, "SGP") + self.assertEqual(competition_day.plane_class, "Default") # Verify competitors were created self.assertEqual(len(competition_day.competitors), 2) diff --git a/tests/competition/test_sgp_api.py b/tests/competition/test_crosscountry_api.py similarity index 93% rename from tests/competition/test_sgp_api.py rename to tests/competition/test_crosscountry_api.py index 47ac12f..6e7775d 100644 --- a/tests/competition/test_sgp_api.py +++ b/tests/competition/test_crosscountry_api.py @@ -1,11 +1,11 @@ """ -Integration tests for SGP API structure verification. +Integration tests for Crosscountry API structure verification. -These tests make real API calls to the SGP endpoints and verify that the +These tests make real API calls to the Crosscountry endpoints and verify that the expected structure is present. These will help detect unexpected API changes. NOTE: These tests require internet connectivity and depend on the -actual SGP API being available. +actual Crosscountry API being available. """ import unittest import shutil @@ -15,7 +15,8 @@ from pathlib import Path import re -from opensoar.competition.sgp import SGPDaily +from opensoar.competition.crosscountry import CrosscountryDaily + # Configure logging @@ -23,9 +24,9 @@ logger = logging.getLogger(__name__) -class TestSGPApiIntegration(unittest.TestCase): +class TestCrosscountryApiIntegration(unittest.TestCase): """ - Integration tests that verify the SGP API structure using real API calls. + Integration tests that verify the Crosscountry API structure using real API calls. These tests depend on the actual API and should be run periodically to ensure that the API structure hasn't changed unexpectedly. @@ -42,9 +43,9 @@ def setUp(self): self.comp_api_url = f"https://www.crosscountry.aero/c/sgp/rest/comp/{self.comp_id}" self.day_api_url = f"https://www.crosscountry.aero/c/sgp/rest/day/{self.comp_id}/{self.day_id}" - # Create SGP daily instances with the API URLs - self.sgp_comp = SGPDaily(self.comp_api_url) - self.sgp_day = SGPDaily(self.day_api_url) + # Create Crosscountry daily instances with the API URLs + self.sgp_comp = CrosscountryDaily(self.comp_api_url) + self.sgp_day = CrosscountryDaily(self.day_api_url) # Verify the URLs were correctly parsed self.assertEqual(self.sgp_comp.competition_id, self.comp_id) @@ -253,8 +254,8 @@ def test_competitors_info(self): self.fail(f"Error testing competitors info: {str(e)}") -class TestSGPUrlHandling(unittest.TestCase): - """Test proper URL handling and ID extraction for SGP URLs.""" +class TestCrosscountryUrlHandling(unittest.TestCase): + """Test proper URL handling and ID extraction for Crosscountry URLs.""" def test_api_url_pattern(self): """Test extraction of IDs from direct API URLs.""" @@ -262,8 +263,8 @@ def test_api_url_pattern(self): comp_url = "https://www.crosscountry.aero/c/sgp/rest/comp/86" day_url = "https://www.crosscountry.aero/c/sgp/rest/day/86/1547" - sgp_comp = SGPDaily(comp_url) - sgp_day = SGPDaily(day_url) + sgp_comp = CrosscountryDaily(comp_url) + sgp_day = CrosscountryDaily(day_url) self.assertEqual(sgp_comp.competition_id, 86) self.assertIsNone(sgp_comp.day_id) @@ -276,7 +277,7 @@ def test_web_url_pattern(self): Test extraction of IDs from web URLs. This test documents the current handling of web URLs and may fail - if the SGP class doesn't correctly handle these URL patterns. + if the Crosscountry class doesn't correctly handle these URL patterns. """ # Current web URLs (based on error messages) web_comp_url = "https://www.crosscountry.aero/c/sgp/overview/127" From d4f79c2d671649a25cd51748f32f6969921cbd8a Mon Sep 17 00:00:00 2001 From: sylvainvdm <61663831+sylvainvdm@users.noreply.github.com> Date: Sun, 25 May 2025 08:15:19 +0200 Subject: [PATCH 5/9] Update opensoar/competition/crosscountry.py --- opensoar/competition/crosscountry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensoar/competition/crosscountry.py b/opensoar/competition/crosscountry.py index 5b8761d..12a718d 100644 --- a/opensoar/competition/crosscountry.py +++ b/opensoar/competition/crosscountry.py @@ -196,7 +196,7 @@ def _get_competitors_info(self, include_hc_competitors: bool = True, include_dns Args: include_hc_competitors: Whether to include hors-concours competitors - include_dns_competitors: Whether to include competitors who did not start + include_dns_competitors: Whether to include competitors who did not start or did not fly Returns: List of dictionaries with competitor information From 0d3b80cf384621c85bee2ece1176ba3c22eb269e Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 25 May 2025 08:31:54 +0200 Subject: [PATCH 6/9] implement retry on getting html page --- opensoar/competition/daily_results_page.py | 36 +++++++++------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/opensoar/competition/daily_results_page.py b/opensoar/competition/daily_results_page.py index 9e8e07d..6430e2e 100644 --- a/opensoar/competition/daily_results_page.py +++ b/opensoar/competition/daily_results_page.py @@ -1,18 +1,17 @@ import requests from bs4 import BeautifulSoup import os -import time import requests import operator import os from abc import ABC, abstractmethod from typing import List -import time from bs4 import BeautifulSoup from opensoar.competition.competition_day import CompetitionDay from opensoar.task.task import Task +from opensoar.utilities.retry_utils import web_request_retry class DailyResultsPage(ABC): @@ -37,6 +36,7 @@ def set_igc_directory(self, target_directory, competition_name, plane_class, dat self._igc_directory = os.path.join(target_directory, competition_name, plane_class, date.strftime('%d-%m-%Y')) + @web_request_retry(max_attempts=3) def _get_html_soup(self) -> BeautifulSoup: """ Get a BeautifulSoup object from the URL. @@ -88,6 +88,7 @@ def igc_file_path(self, competition_id: str) -> str: file_name = self.igc_file_name(competition_id) return os.path.join(self._igc_directory, file_name) + @web_request_retry(max_attempts=3) def download_flight(self, igc_url: str, competition_id: str) -> str: """ Download flight and return file_path @@ -106,30 +107,21 @@ def download_flight(self, igc_url: str, competition_id: str) -> str: file_path = self.igc_file_path(competition_id) - # Attempt to download the file - max_retries = 3 - retry_count = 0 - while not os.path.exists(file_path) and retry_count < max_retries: - try: - response = requests.get(igc_url, timeout=30) - response.raise_for_status() # Raise an exception for HTTP errors + if not os.path.exists(file_path): + response = requests.get(igc_url, timeout=30) + response.raise_for_status() # Raise an exception for HTTP errors + + # Write the content to the file + with open(file_path, 'wb') as f: + f.write(response.content) - # Write the content to the file - with open(file_path, 'wb') as f: - f.write(response.content) + # Verify file was created + if not os.path.exists(file_path): + raise FileNotFoundError(f"File was not created at {file_path}") - # Verify file was created - if not os.path.exists(file_path): - raise FileNotFoundError(f"File was not created at {file_path}") - - except (requests.exceptions.RequestException, FileNotFoundError) as e: - print(f"Download attempt {retry_count + 1} failed: {e}") - retry_count += 1 - time.sleep(1) # Longer delay between retries - if not os.path.exists(file_path): - raise RuntimeError(f"Failed to download file from {igc_url} after {max_retries} attempts") + raise RuntimeError(f"Failed to download file from {igc_url}") return file_path From 845073d05337b260f863124948d43a668aa378e8 Mon Sep 17 00:00:00 2001 From: sylvainvdm <61663831+sylvainvdm@users.noreply.github.com> Date: Sun, 25 May 2025 08:35:28 +0200 Subject: [PATCH 7/9] remove obsolete line --- opensoar/competition/daily_results_page.py | 1 - 1 file changed, 1 deletion(-) diff --git a/opensoar/competition/daily_results_page.py b/opensoar/competition/daily_results_page.py index 6430e2e..b8ee019 100644 --- a/opensoar/competition/daily_results_page.py +++ b/opensoar/competition/daily_results_page.py @@ -100,7 +100,6 @@ def download_flight(self, igc_url: str, competition_id: str) -> str: Returns: str: Path to the downloaded file """ - # Make directory if necessary if not os.path.exists(self._igc_directory): os.makedirs(self._igc_directory) From 1f58a862ea98d6aeb35a369f91f26105651370e6 Mon Sep 17 00:00:00 2001 From: sylvainvdm <61663831+sylvainvdm@users.noreply.github.com> Date: Sun, 25 May 2025 08:35:57 +0200 Subject: [PATCH 8/9] remove obsolete line --- opensoar/competition/daily_results_page.py | 1 - 1 file changed, 1 deletion(-) diff --git a/opensoar/competition/daily_results_page.py b/opensoar/competition/daily_results_page.py index b8ee019..7bdd355 100644 --- a/opensoar/competition/daily_results_page.py +++ b/opensoar/competition/daily_results_page.py @@ -106,7 +106,6 @@ def download_flight(self, igc_url: str, competition_id: str) -> str: file_path = self.igc_file_path(competition_id) - if not os.path.exists(file_path): response = requests.get(igc_url, timeout=30) response.raise_for_status() # Raise an exception for HTTP errors From 7060c1edf8e694dfaf5e8a6983332a9f6a92dfce Mon Sep 17 00:00:00 2001 From: Sylvain Date: Sun, 25 May 2025 08:36:30 +0200 Subject: [PATCH 9/9] add retry utils --- opensoar/utilities/retry_utils.py | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 opensoar/utilities/retry_utils.py diff --git a/opensoar/utilities/retry_utils.py b/opensoar/utilities/retry_utils.py new file mode 100644 index 0000000..0b4274a --- /dev/null +++ b/opensoar/utilities/retry_utils.py @@ -0,0 +1,71 @@ +""" +Retry utilities for robust network operations. + +This module provides lightweight retry decorators for handling transient failures +in network requests and file operations. +""" + +import time +import functools +from typing import Callable, Union, Tuple, Type + + +def retry( + max_attempts: int = 3, + delay: float = 1.0, + backoff: float = 2.0, + exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = Exception +): + """ + Lightweight retry decorator for handling transient failures. + + Args: + max_attempts: Maximum number of retry attempts + delay: Initial delay between retries in seconds + backoff: Multiplier for delay after each retry + exceptions: Exception types to catch and retry on + + Returns: + Decorated function with retry logic + + Example: + @retry(max_attempts=3, delay=1.0, exceptions=requests.exceptions.RequestException) + def download_file(url): + response = requests.get(url) + response.raise_for_status() + return response.content + """ + def decorator(func: Callable): + @functools.wraps(func) + def wrapper(*args, **kwargs): + last_exception = None + current_delay = delay + + for attempt in range(max_attempts): + try: + return func(*args, **kwargs) + except exceptions as e: + last_exception = e + if attempt == max_attempts - 1: + raise + + print(f"Attempt {attempt + 1} failed: {e}. Retrying in {current_delay}s...") + time.sleep(current_delay) + current_delay *= backoff + + raise last_exception + + return wrapper + return decorator + + +# Predefined retry decorators for common use cases +def web_request_retry(max_attempts: int = 3): + """Retry decorator specifically for web requests.""" + import requests + return retry( + max_attempts=max_attempts, + delay=1.0, + backoff=2.0, + exceptions=requests.exceptions.RequestException + )