diff --git a/.env.sample b/.env.sample index 9fd2e4b..a1a0ed5 100644 --- a/.env.sample +++ b/.env.sample @@ -10,9 +10,11 @@ LOGGING_LEVEL=DEBUG # ключи для доступа к API # https://apilayer.com/marketplace/geo-api -API_KEY_APILAYER= +API_KEY_APILAYER=ERNjwL3lfk38GxU3LlyH9bqmM3oqb2c2 # https://openweathermap.org/price#weather -API_KEY_OPENWEATHER= +API_KEY_OPENWEATHER=1c9fddce098864eb1395202eed9aabbc +# https://newsapi.org/account +API_KEY_NEWSAPI=04fc313c0d2e48abbe6987bacf7eab66 # время актуальности данных о странах (в секундах) CACHE_TTL_COUNTRY=31_536_000 diff --git a/README.md b/README.md index b731e52..ff86a31 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,12 @@ Install the appropriate software: To access the API, visit the appropriate resources and obtain an access token: - APILayer – Geography API (https://apilayer.com/marketplace/geo-api) - OpenWeather – Weather Free Plan (https://openweathermap.org/price#weather) + - APINews - News API (https://newsapi.org) Set received access tokens as environment variable values (in `.env` file): - `API_KEY_APILAYER` – for APILayer access token - `API_KEY_OPENWEATHER` – for OpenWeather access token + - `API_KEY_NEWSAPI` - for APINews access token 2. Build the container using Docker Compose: ```shell @@ -62,6 +64,7 @@ Install the appropriate software: - `CACHE_TTL_COUNTRY` (country data up-to-date time in seconds) - `CACHE_TTL_CURRENCY_RATES` (currency rates data up-to-date time in seconds) - `CACHE_TTL_WEATHER` (weather data up-to-date time in seconds) + - `CACHE_TTL_NEWS` (news data up-to-date time in seconds) 5. After collecting all the data, you can query the country information by executing the command: ```shell diff --git a/docs/source/index.rst b/docs/source/index.rst index faf8d10..b29272a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,11 +60,13 @@ * APILayer — Geography API (https://apilayer.com/marketplace/geo-api) * OpenWeather – Weather Free Plan (https://openweathermap.org/price#weather) + * APINews - News API (https://newsapi.org) Задайте полученные токены доступа в качестве значений переменных окружения (в файле `.env`): * `API_KEY_APILAYER` – для токена доступа к APILayer * `API_KEY_OPENWEATHER` – для токена доступа к OpenWeather + * `API_KEY_NEWSAPI` - для токена доступа к APINews 2. Соберите Docker-контейнер с помощью Docker Compose: .. code-block:: console @@ -92,6 +94,7 @@ * `CACHE_TTL_COUNTRY` (время актуальности данных о странах) * `CACHE_TTL_CURRENCY_RATES` (время актуальности данных о курсах валют) * `CACHE_TTL_WEATHER` (время актуальности данных о погоде) + * `CACHE_TTL_NEWS` (время актуальности данных о новостях) Значение для этих переменных указывается в секундах (они определяются в файле `.env`). diff --git a/src/clients/news.py b/src/clients/news.py new file mode 100644 index 0000000..e8d7155 --- /dev/null +++ b/src/clients/news.py @@ -0,0 +1,39 @@ +""" +Функции для взаимодействия с внешним новостным сервисом. +""" + +from http import HTTPStatus +from typing import Optional + +import aiohttp + +from clients.base import BaseClient +from logger import trace_config +from settings import API_KEY_NEWSAPI, NEWS_COUNT + + +class NewsClient(BaseClient): + """ + Реализация функций для взаимодействия с внешним новостным сервисом. + """ + + async def get_base_url(self) -> str: + return "https://newsapi.org/v2/everything" + + async def _request(self, endpoint: str) -> Optional[dict]: + async with aiohttp.ClientSession(trace_configs=[trace_config]) as session: + async with session.get(endpoint) as response: + if response.status == HTTPStatus.OK: + return await response.json() + + return None + + async def get_news(self, location: str) -> Optional[dict]: + """ + Получение новостей по локации + :param location: город или страна + :return: данные о городе или стране + """ + return await self._request( + f"{await self.get_base_url()}?q={location}&apiKey={API_KEY_NEWSAPI}&pageSize={NEWS_COUNT}" + ) diff --git a/src/collectors/collector.py b/src/collectors/collector.py index ebadf7e..74ac792 100644 --- a/src/collectors/collector.py +++ b/src/collectors/collector.py @@ -15,18 +15,21 @@ from clients.currency import CurrencyClient from clients.weather import WeatherClient from collectors.base import BaseCollector +from clients.news import NewsClient from collectors.models import ( LocationDTO, CountryDTO, CurrencyRatesDTO, CurrencyInfoDTO, WeatherInfoDTO, + NewsInfoDTO, ) from settings import ( MEDIA_PATH, CACHE_TTL_COUNTRY, CACHE_TTL_CURRENCY_RATES, CACHE_TTL_WEATHER, + CACHE_TTL_NEWS, ) @@ -103,6 +106,9 @@ async def read(cls) -> Optional[list[CountryDTO]]: population=item["population"], subregion=item["subregion"], timezones=item["timezones"], + square=item["area"], + geographical_latitude=item["latitude"], + geographical_longitude=item["longitude"], ) ) @@ -203,8 +209,8 @@ async def read(cls, location: LocationDTO) -> Optional[WeatherInfoDTO]: """ Чтение данных из кэша. - :param location: - :return: + :param location: Название локации + :return: None, если кэш пуст, WeatherInfoDTO иначе """ filename = f"{location.capital}_{location.alpha2code}".lower() @@ -217,13 +223,90 @@ async def read(cls, location: LocationDTO) -> Optional[WeatherInfoDTO]: temp=result["main"]["temp"], pressure=result["main"]["pressure"], humidity=result["main"]["humidity"], + visibility=result["visibility"], wind_speed=result["wind"]["speed"], description=result["weather"][0]["description"], + offset_seconds=result["timezone"], + timezone=result["timezone"], ) return None +class NewsCollector(BaseCollector): + """ + Сбор информации о новостях. + """ + + def __init__(self) -> None: + self.client = NewsClient() + + @staticmethod + async def get_file_path(filename: str = "", **kwargs: Any) -> str: + return f"{MEDIA_PATH}/news/{filename}.json" + + @staticmethod + async def get_cache_ttl() -> int: + return CACHE_TTL_NEWS + + async def collect( + self, locations: FrozenSet[LocationDTO] = frozenset(), **kwargs: Any + ) -> None: + """ + Сбор информации о новостях. + :param locations: Страна или город, для сбора новостей. + :return: + """ + + target_dir_path = f"{MEDIA_PATH}/news" + # если целевой директории еще не существует, то она создается + if not await aiofiles.os.path.exists(target_dir_path): + await aiofiles.os.mkdir(target_dir_path) + + for location in locations: + filename = f"{location.capital}_{location.alpha2code}".lower() + if not await self.cache_invalid(filename=filename): + continue + + # если кэш уже невалиден, то актуализируем его + result = await self.client.get_news(location.capital) + if not result: + continue + + result_str = json.dumps(result) + async with aiofiles.open( + await self.get_file_path(filename), mode="w" + ) as file: + await file.write(result_str) + + @classmethod + async def read(cls, location: LocationDTO) -> list[NewsInfoDTO] | None: + """ + Чтение данных из кэша. + :param location: Страна и/или город для которых нужно получить новости. + :return: + """ + + filename = f"{location.capital}_{location.alpha2code}".lower() + async with aiofiles.open(await cls.get_file_path(filename), mode="r") as file: + content = await file.read() + + result = json.loads(content) + if not result: + return None + return [ + NewsInfoDTO( + source=article["source"]["name"], + title=article["title"], + description=article["description"], + url=article["url"], + published_at=article["publishedAt"], + content=article["content"], + ) + for article in result["articles"] + ] + + class Collectors: @staticmethod async def gather() -> tuple: @@ -238,6 +321,7 @@ def collect() -> None: try: results = loop.run_until_complete(Collectors.gather()) loop.run_until_complete(WeatherCollector().collect(results[1])) + loop.run_until_complete(NewsCollector().collect(results[1])) loop.run_until_complete(loop.shutdown_asyncgens()) finally: diff --git a/src/collectors/models.py b/src/collectors/models.py index 7e36198..cb095b9 100644 --- a/src/collectors/models.py +++ b/src/collectors/models.py @@ -3,6 +3,7 @@ """ from pydantic import Field, BaseModel +from datetime import datetime class HashableBaseModel(BaseModel): @@ -93,6 +94,9 @@ class CountryDTO(BaseModel): timezones=[ "UTC+02:00", ], + square=50453.1, + geographical_latitude=45.23, + geographical_longitude=15.35, ) """ @@ -106,6 +110,9 @@ class CountryDTO(BaseModel): population: int subregion: str timezones: list[str] + square: float | None + geographical_latitude: float | None + geographical_longitude: float | None class CurrencyRatesDTO(BaseModel): @@ -143,11 +150,36 @@ class WeatherInfoDTO(BaseModel): ) """ + offset_seconds: int + timezone: int temp: float pressure: int humidity: int wind_speed: float description: str + visibility: int + + +class NewsInfoDTO(BaseModel): + """ + Модель данных новостей. + .. code-block:: + NewsDTO( + source="CNN", + title="The latest news about the coronavirus pandemic", + description="The latest news about the coronavirus pandemic", + url="https://www.cnn.com/world/live-news/coronavirus-pandemic-09-14-21-intl/index.html", + published_at="2021-09-14T20:00:00Z", + content="The latest news about the coronavirus pandemic" + ) + """ + + source: str + title: str + description: str + url: str + published_at: datetime + content: str class LocationInfoDTO(BaseModel): @@ -201,3 +233,4 @@ class LocationInfoDTO(BaseModel): location: CountryDTO weather: WeatherInfoDTO currency_rates: dict[str, float] + news: list[NewsInfoDTO] | None diff --git a/src/reader.py b/src/reader.py index dd1a74d..d3cb4a8 100644 --- a/src/reader.py +++ b/src/reader.py @@ -8,6 +8,7 @@ from collectors.collector import ( CountryCollector, CurrencyRatesCollector, + NewsCollector, WeatherCollector, ) from collectors.models import ( @@ -15,6 +16,7 @@ CurrencyInfoDTO, LocationDTO, LocationInfoDTO, + NewsInfoDTO, WeatherInfoDTO, ) @@ -33,16 +35,20 @@ async def find(self, location: str) -> Optional[LocationInfoDTO]: """ country = await self.find_country(location) + location_dto = LocationDTO( + capital=country.capital, alpha2code=country.alpha2code + ) if country: weather = await self.get_weather( LocationDTO(capital=country.capital, alpha2code=country.alpha2code) ) currency_rates = await self.get_currency_rates(country.currencies) - + news = await self.get_news(location_dto) return LocationInfoDTO( location=country, weather=weather, currency_rates=currency_rates, + news=news, ) return None @@ -118,3 +124,11 @@ async def _match(search: str, country: CountryDTO) -> bool: return True return False + + async def get_news(self, location: LocationDTO) -> list[NewsInfoDTO] | None: + """ + Получение новостей по стране. + :param location: + :return: + """ + return await NewsCollector.read(location=location) diff --git a/src/renderer.py b/src/renderer.py index 8b90dcc..aba2f22 100644 --- a/src/renderer.py +++ b/src/renderer.py @@ -2,9 +2,10 @@ Функции для формирования выходной информации. """ +import datetime from decimal import ROUND_HALF_UP, Decimal -from collectors.models import LocationInfoDTO +from collectors.models import LocationInfoDTO, NewsInfoDTO class Renderer: @@ -28,15 +29,116 @@ async def render(self) -> tuple[str, ...]: :return: Результат форматирования """ - return ( - f"Страна: {self.location_info.location.name}", - f"Столица: {self.location_info.location.capital}", - f"Регион: {self.location_info.location.subregion}", - f"Языки: {await self._format_languages()}", - f"Население страны: {await self._format_population()} чел.", - f"Курсы валют: {await self._format_currency_rates()}", - f"Погода: {self.location_info.weather.temp} °C", + values = { + "Страна": self.location_info.location.name, + "Столица": self.location_info.location.capital, + "Регион": self.location_info.location.subregion, + "Языки": await self._format_languages(), + "Население страны": await self._format_population(), + "Курсы валют": await self._format_currency_rates(), + "Погода": self.location_info.weather.temp, + "Площадь страны": self.location_info.location.square, + "Широта": self.location_info.location.geographical_latitude, + "Долгота": self.location_info.location.geographical_longitude, + "Время": await self._format_current_time(), + "Часовой пояс": self.location_info.weather.timezone, + "Описание погоды": self.location_info.weather.description, + "Видимость": self.location_info.weather.visibility, + "Влажность": self.location_info.weather.humidity, + "Скорость ветра": self.location_info.weather.wind_speed, + "Давление": self.location_info.weather.pressure, + } + + first_column_width = max(len(key) for key in values) + 1 + second_column_width = max(len(str(value)) for value in values.values()) + 1 + formatted_values = [("-" * (first_column_width + second_column_width + 3))] + formatted_values.extend( + [ + f"|{key:<{first_column_width}}|{value:>{second_column_width}}|" + for key, value in values.items() + ] ) + formatted_values.append("-" * (first_column_width + second_column_width + 3)) + formatted_values.extend( + await self._format_news( + self.location_info.news, first_column_width, second_column_width + ) + ) + return tuple(formatted_values) + + async def _format_news_line( + self, + first_col_name: str, + content: str, + first_column_width: int, + second_column_width: int, + ) -> list[str]: + """ + Форматирование информации о новостях. + :param first_col_name: Название первой колонки. + :param content: Содержимое колонки + :param first_column_width: Ширина первой колонки. + :param second_column_width: Ширина второй колонки. + :return: + """ + content = content.replace("\n", " ").replace("\r", " ") + values = [ + f"|{first_col_name:<{first_column_width}}|{content[:second_column_width]:>{second_column_width}}|" + ] + + values.extend( + [ + f"|{'':<{first_column_width}}|{content[line:line + second_column_width]:<{second_column_width}}|" + for line in range( + second_column_width, len(content), second_column_width + ) + ] + ) + return values + + async def _format_news( + self, + news: list[NewsInfoDTO] | None, + first_column_width: int, + second_column_width: int, + ) -> list[str]: + """ + Форматирование информации о новостях. + :param news: Список новостей. + :param first_column_width: Ширина первой колонки. + :param second_column_width: Ширина второй колонки. + :return: + """ + if news is None: + return [] + values = [] + first_column_names = [ + "Источник", + "Новость", + "Ссылка", + "Дата", + "Описание", + "Текст", + ] + for item in news: + for first_col_name, content in zip( + first_column_names, + [ + item.source, + item.title, + item.url, + item.published_at.strftime("%d.%m.%Y %H:%M"), + item.description, + item.content, + ], + ): + values.extend( + await self._format_news_line( + first_col_name, content, first_column_width, second_column_width + ) + ) + values.append("-" * (first_column_width + second_column_width + 3)) + return values async def _format_languages(self) -> str: """ @@ -60,6 +162,17 @@ async def _format_population(self) -> str: # pylint: disable=C0209 return "{:,}".format(self.location_info.location.population).replace(",", ".") + async def _format_current_time(self) -> str: + """ + Форматирование информации о времени. + :return: + """ + + render_time = datetime.datetime.now() + datetime.timedelta( + seconds=self.location_info.weather.offset_seconds + ) + return render_time.strftime("%X, %x") + async def _format_currency_rates(self) -> str: """ Форматирование информации о курсах валют. diff --git a/src/settings.py b/src/settings.py index 4559f84..940337a 100644 --- a/src/settings.py +++ b/src/settings.py @@ -20,6 +20,7 @@ # ключи для доступа к API API_KEY_APILAYER: Optional[str] = os.getenv("API_KEY_APILAYER") API_KEY_OPENWEATHER: Optional[str] = os.getenv("API_KEY_OPENWEATHER") +API_KEY_NEWSAPI: Optional[str] = os.getenv("API_KEY_NEWSAPI") # время актуальности данных о странах (в секундах), по умолчанию – один год CACHE_TTL_COUNTRY: int = int(os.getenv("CACHE_TTL_COUNTRY", "31_536_000")) @@ -27,3 +28,5 @@ CACHE_TTL_CURRENCY_RATES: int = int(os.getenv("CACHE_TTL_CURRENCY_RATES", "86_400")) # время актуальности данных о погоде (в секундах), по умолчанию ~ три часа CACHE_TTL_WEATHER: int = int(os.getenv("CACHE_TTL_WEATHER", "10_700")) +CACHE_TTL_NEWS: int = int(os.getenv("CACHE_TTL_NEWS", "3_600")) +NEWS_COUNT: int = int(os.getenv("NEWS_COUNT", "5"))