diff --git a/dev-requirements.in b/dev-requirements.in index e96f7d5308..50a9feb82c 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -11,7 +11,7 @@ pre-commit>=2.9.0 pylint>=2.6.0 # pylint < 2.6 doesn't work with isort5 pytest-cov~=3.0.0 pytest-runner -pytest-xdist~=2.4.0 +pytest-xdist~=3.1.0 pytest~=6.2.4 sphinx~=4.3.0 sqlalchemy-stubs diff --git a/dev-requirements.txt b/dev-requirements.txt index 2f98158ee9..5ad427a4e9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,8 +8,6 @@ alabaster==0.7.12 # via sphinx astroid==2.11.2 # via pylint -atomicwrites==1.4.0 - # via pytest attrs==21.4.0 # via # -c requirements.txt @@ -30,6 +28,8 @@ certifi==2021.10.8 # via # -c requirements.txt # requests +cffi==1.15.1 + # via cryptography cfgv==3.3.1 # via pre-commit charset-normalizer==2.0.12 @@ -47,16 +47,13 @@ codacy-coverage==1.3.11 colorama==0.4.4 # via # -c requirements.txt - # click - # pylint - # pytest - # sphinx - # tqdm # twine coverage[toml]==6.2 # via # -r dev-requirements.in # pytest-cov +cryptography==38.0.4 + # via secretstorage dill==0.3.4 # via pylint distlib==0.3.4 @@ -92,6 +89,10 @@ isort==5.10.1 # via # -r dev-requirements.in # pylint +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -143,9 +144,9 @@ pluggy==1.0.0 pre-commit==2.17.0 # via -r dev-requirements.in py==1.11.0 - # via - # pytest - # pytest-forked + # via pytest +pycparser==2.21 + # via cffi pygments==2.11.2 # via # -c requirements.txt @@ -153,7 +154,7 @@ pygments==2.11.2 # sphinx pylint==2.13.2 # via -r dev-requirements.in -pyparsing==3.0.7 +pyparsing==2.4.7 # via # -c requirements.txt # packaging @@ -161,15 +162,12 @@ pytest==6.2.5 # via # -r dev-requirements.in # pytest-cov - # pytest-forked # pytest-xdist pytest-cov==3.0.0 # via -r dev-requirements.in -pytest-forked==1.4.0 - # via pytest-xdist pytest-runner==5.3.2 # via -r dev-requirements.in -pytest-xdist==2.4.0 +pytest-xdist==3.1.0 # via -r dev-requirements.in python-dateutil==2.8.2 # via @@ -199,6 +197,8 @@ rfc3986==1.5.0 # via twine s3transfer==0.5.2 # via boto3 +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # -c requirements.txt @@ -232,11 +232,9 @@ toml==0.10.2 # pytest tomli==1.2.3 # via - # black # coverage # mypy # pep517 - # pylint tqdm==4.63.1 # via twine twine==3.6.0 diff --git a/flexget/components/managed_lists/lists/entry_list/db.py b/flexget/components/managed_lists/lists/entry_list/db.py index 5a2ea8bc39..b9161f153b 100644 --- a/flexget/components/managed_lists/lists/entry_list/db.py +++ b/flexget/components/managed_lists/lists/entry_list/db.py @@ -119,7 +119,7 @@ def _entry_query(self, session, entry): or_( EntryListEntry.title == entry['title'], and_( - EntryListEntry.original_url, + EntryListEntry.original_url.isnot(None), EntryListEntry.original_url == entry['original_url'], ), ), @@ -157,7 +157,9 @@ def add(self, entry): if stored_entry: # Refresh all the fields if we already have this entry logger.debug('refreshing entry {}', entry) - stored_entry.entry = entry + new_entry = Entry(stored_entry.entry) + new_entry.update(entry) + stored_entry.entry = new_entry else: logger.debug('adding entry {} to list {}', entry, self._db_list(session).name) stored_entry = EntryListEntry(entry=entry, entry_list_id=self._db_list(session).id) diff --git a/flexget/components/managed_lists/lists/radarr_list.py b/flexget/components/managed_lists/lists/radarr_list.py index 68881e50ac..cb884df4d2 100644 --- a/flexget/components/managed_lists/lists/radarr_list.py +++ b/flexget/components/managed_lists/lists/radarr_list.py @@ -75,7 +75,7 @@ def request_delete_json(url, headers): def request_post_json(url, headers, data): """Makes a POST request and returns the JSON response""" try: - response = requests.post(url, headers=headers, data=data, timeout=10) + response = requests.post(url, headers=headers, json=data, timeout=10) if response.status_code == 201: return response.json() else: @@ -123,7 +123,7 @@ def __init__(self, api_key, base_url, port=None): if parsed_base_url.port: port = int(parsed_base_url.port) - self.api_url = "%s://%s:%s%s/api/" % ( + self.api_url = "%s://%s:%s%s/api/v3/" % ( parsed_base_url.scheme, parsed_base_url.netloc, port, @@ -132,7 +132,7 @@ def __init__(self, api_key, base_url, port=None): def get_profiles(self): """Gets all profiles""" - request_url = self.api_url + "profile" + request_url = self.api_url + "qualityProfile" headers = self._default_headers() return request_get_json(request_url, headers) @@ -147,7 +147,7 @@ def add_tag(self, label): request_url = self.api_url + "tag" headers = self._default_headers() data = {"label": label} - return request_post_json(request_url, headers, json.dumps(data)) + return request_post_json(request_url, headers, data) def get_movies(self): """Gets all movies""" @@ -220,7 +220,7 @@ def add_movie( data["addOptions"] = add_options try: - json_response = request_post_json(request_url, headers, json.dumps(data)) + json_response = request_post_json(request_url, headers, data) except RadarrRequestError as ex: spec_ex = spec_exception_from_response_ex(ex) if spec_ex: @@ -395,6 +395,7 @@ def add(self, entry): if result: root_folders = self.service.get_root_folders() root_folder_path = root_folders[0]["path"] + add_options = {'searchForMovie': True } if self.config.get('search') else None try: self.service.add_movie( @@ -406,6 +407,7 @@ def add(self, entry): result["tmdbId"], root_folder_path, monitored=self.config.get('monitored', False), + add_options=add_options, tags=self.get_tag_ids(entry), ) logger.verbose('Added movie {} to Radarr list', result['title']) @@ -528,7 +530,7 @@ def _get_movie_entries(self): # Check if we should add quality requirement if self.config.get("include_data"): - movie_profile_id = movie["profileId"] + movie_profile_id = movie["qualityProfileId"] for profile in profiles: profile_id = profile["id"] if profile_id == movie_profile_id: @@ -544,6 +546,7 @@ def _get_movie_entries(self): title=movie["title"], url="", radarr_id=movie["id"], + radarr_added=movie['added'], movie_name=movie["title"], movie_year=movie["year"], ) @@ -619,6 +622,7 @@ class RadarrList: "api_key": {"type": "string"}, "only_monitored": {"type": "boolean", "default": True}, "include_data": {"type": "boolean", "default": False}, + "search": {"type": "boolean", "default": False}, "only_use_cutoff_quality": {"type": "boolean", "default": False}, "monitored": {"type": "boolean", "default": True}, "profile_id": {"type": "integer", "default": 1}, diff --git a/flexget/components/managed_lists/lists/sonarr_list.py b/flexget/components/managed_lists/lists/sonarr_list.py index 28f17105dd..6e7dc87296 100644 --- a/flexget/components/managed_lists/lists/sonarr_list.py +++ b/flexget/components/managed_lists/lists/sonarr_list.py @@ -12,7 +12,7 @@ SERIES_ENDPOINT = 'series' LOOKUP_ENDPOINT = 'series/lookup' -PROFILE_ENDPOINT = 'profile' +PROFILE_ENDPOINT = 'qualityProfile' ROOTFOLDER_ENDPOINT = 'Rootfolder' DELETE_ENDPOINT = 'series/{}' @@ -32,10 +32,11 @@ class SonarrSet(MutableSet): 'include_ended': {'type': 'boolean', 'default': True}, 'only_monitored': {'type': 'boolean', 'default': True}, 'include_data': {'type': 'boolean', 'default': False}, - 'search_missing_episodes': {'type': 'boolean', 'default': True}, + 'search_missing_episodes': {'type': 'string', 'enum': ['all', 'first_season', 'no']}, 'ignore_episodes_without_files': {'type': 'boolean', 'default': False}, 'ignore_episodes_with_files': {'type': 'boolean', 'default': False}, 'profile_id': {'type': 'integer', 'default': 1}, + 'language_id': {'type': 'integer', 'default': 1}, 'season_folder': {'type': 'boolean', 'default': False}, 'monitored': {'type': 'boolean', 'default': True}, 'root_folder_path': {'type': 'string'}, @@ -61,7 +62,7 @@ def _sonarr_request(self, endpoint, term=None, method='get', data=None): base_url = self.config['base_url'] port = self.config['port'] base_path = self.config['base_path'] - url = '{}:{}{}/api/{}'.format(base_url, port, base_path, endpoint) + url = '{}:{}{}/api/v3/{}'.format(base_url, port, base_path, endpoint) headers = {'X-Api-Key': self.config['api_key']} if term: url += '?term={}'.format(term) @@ -147,7 +148,7 @@ def list_entries(self, filters=True): # Checks if to retrieve ended shows if show['status'] == 'ended' and not self.config.get('include_ended'): continue - profile = profiles_dict.get(show['profileId']) + profile = profiles_dict.get(show['qualityProfileId']) if profile: fg_qualities, fg_cutoff = self.quality_requirement_builder(profile) @@ -158,9 +159,10 @@ def list_entries(self, filters=True): tvdb_id=show.get('tvdbId'), tvrage_id=show.get('tvRageId'), tvmaze_id=show.get('tvMazeId'), - imdb_id=show.get('imdbid'), + imdb_id=show.get('imdbId'), slug=show.get('titleSlug'), sonarr_id=show.get('id'), + sonarr_added=show.get('added'), ) if len(fg_qualities) > 1: entry['configure_series_qualities'] = fg_qualities @@ -207,6 +209,7 @@ def add_show(self, entry): # Setting defaults for Sonarr show['profileId'] = self.config.get('profile_id') show['qualityProfileId'] = self.config.get('profile_id') + show['languageProfileId'] = self.config.get('language_id') show['seasonFolder'] = self.config.get('season_folder') show['monitored'] = self.config.get('monitored') show['seriesType'] = self.config.get('series_type') @@ -215,9 +218,18 @@ def add_show(self, entry): show['addOptions'] = { "ignoreEpisodesWithFiles": self.config.get('ignore_episodes_with_files'), "ignoreEpisodesWithoutFiles": self.config.get('ignore_episodes_without_files'), - "searchForMissingEpisodes": self.config.get('search_missing_episodes'), } + if self.config.get('searchForMissingEpisodes') == 'no': + show['addOptions']['searchForMissingEpisodes'] = False + elif self.config.get('searchForMissingEpisodes'): + show['addOptions']['searchForMissingEpisodes'] = True + + if self.config.get('searchForMissingEpisodes') == 'first_season' and len(show['seasons']) > 1: + for season in show['seasons']: + if season['seasonNumber'] > 1: + season['monitored'] = False + logger.debug('adding show {} to sonarr', show) returned_show = self._sonarr_request(SERIES_ENDPOINT, method='post', data=show) return returned_show diff --git a/flexget/components/trakt/api_trakt.py b/flexget/components/trakt/api_trakt.py index 44dd3db6ef..df1be1c9f7 100644 --- a/flexget/components/trakt/api_trakt.py +++ b/flexget/components/trakt/api_trakt.py @@ -63,6 +63,10 @@ def _update_watched_cache(self, cache, media_type, username=None, account=None): cache[media_id] = media cache[media_id]['watched_at'] = dateutil_parse(media['last_watched_at'], ignoretz=True) cache[media_id]['plays'] = media['plays'] + if 'seasons' in cache[media_id]: + for season in cache[media_id]['seasons']: + for episode in season['episodes']: + episode['watched_at'] = dateutil_parse(episode['last_watched_at'], ignoretz=True) def _update_ratings_cache(self, cache, media_type, username=None, account=None): ratings = db.get_user_data( @@ -135,6 +139,12 @@ def lookup_map(self): 'episode': self.is_episode_watched, 'movie': self.is_movie_watched, }, + 'watched_at': { + 'show': self.show_watched_at, + 'season': self.season_watched_at, + 'episode': self.episode_watched_at, + 'movie': self.movie_watched_at, + }, 'collected': { 'show': self.is_show_in_collection, 'season': self.is_season_in_collection, @@ -327,9 +337,10 @@ def is_movie_in_collection(self, trakt_data, title): ) return in_collection - def is_show_watched(self, trakt_data, title): + def show_watched(self, trakt_data, title): cache = user_cache.get_watched_shows(username=self.username, account=self.account) is_watched = False + watched_at = None if trakt_data.id in cache: series = cache[trakt_data.id] # specials are not included @@ -337,34 +348,52 @@ def is_show_watched(self, trakt_data, title): len(s['episodes']) for s in series['seasons'] if s['number'] > 0 ) is_watched = number_of_watched_episodes == trakt_data.aired_episodes + watched_at = cache[trakt_data.id]['watched_at'] logger.debug( 'The result for show entry "{}" is: {}', title, 'Watched' if is_watched else 'Not watched', + watched_at if watched_at else '', ) - return is_watched + return is_watched, watched_at - def is_season_watched(self, trakt_data, title): + def is_show_watched(self, trakt_data, title): + return self.show_watched(trakt_data, title)[0] + + def show_watched_at(self, trakt_data, title): + return self.show_watched(trakt_data, title)[1] + + def season_watched(self, trakt_data, title): cache = user_cache.get_watched_shows(username=self.username, account=self.account) is_watched = False + watched_at = None if trakt_data.show.id in cache: series = cache[trakt_data.show.id] for s in series['seasons']: if trakt_data.number == s['number']: is_watched = True + watched_at = max(e['watched_at'] for e in s['episodes']) break logger.debug( 'The result for season entry "{}" is: {}', title, 'Watched' if is_watched else 'Not watched', + watched_at if watched_at else '', ) - return is_watched + return is_watched, watched_at - def is_episode_watched(self, trakt_data, title): + def is_season_watched(self, trakt_data, title): + return self.season_watched(trakt_data, title)[0] + + def season_watched_at(self, trakt_data, title): + return self.season_watched(trakt_data, title)[1] + + def episode_watched(self, trakt_data, title): cache = user_cache.get_watched_shows(username=self.username, account=self.account) is_watched = False + watched_at = None if trakt_data.show.id in cache: series = cache[trakt_data.show.id] for s in series['seasons']: @@ -372,23 +401,44 @@ def is_episode_watched(self, trakt_data, title): # extract all episode numbers currently in collection for the season number episodes = [ep['number'] for ep in s['episodes']] is_watched = trakt_data.number in episodes + it = (i for i,v in s['episodes'].enumerate() if s['episodes'][i]['number'] == trakt_data.number) + pos = next(it, None) + watched_at = s['episodes'][pos]['watched_at'] if pos else None break logger.debug( 'The result for episode entry "{}" is: {}', title, 'Watched' if is_watched else 'Not watched', + watched_at if watched_at else '', ) - return is_watched + return is_watched, watched_at - def is_movie_watched(self, trakt_data, title): + def is_episode_watched(self, trakt_data, title): + return self.episode_watched(trakt_data, title)[0] + + def episode_watched_at(self, trakt_data, title): + return self.episode_watched(trakt_data, title)[1] + + def movie_watched(self, trakt_data, title): cache = user_cache.get_watched_movies(username=self.username, account=self.account) - is_watched = trakt_data.id in cache + is_watched = False + watched_at = None + if trakt_data.id in cache: + is_watched = True + watched_at = cache[trakt_data.id]['watched_at'] logger.debug( - 'The result for movie entry "{}" is: {}', + 'The result for movie entry "{}" is: {} {}', title, 'Watched' if is_watched else 'Not watched', + watched_at if watched_at else '', ) - return is_watched + return is_watched, watched_at + + def is_movie_watched(self, trakt_data, title): + return self.movie_watched(trakt_data, title)[0] + + def movie_watched_at(self, trakt_data, title): + return self.movie_watched(trakt_data, title)[1] def show_user_ratings(self, trakt_data, title): cache = user_cache.get_show_user_ratings(username=self.username, account=self.account) diff --git a/flexget/components/trakt/trakt_list.py b/flexget/components/trakt/trakt_list.py index b88a19ff5b..c538fc100b 100644 --- a/flexget/components/trakt/trakt_list.py +++ b/flexget/components/trakt/trakt_list.py @@ -63,6 +63,8 @@ def generate_episode_title(item): 'tmdb_id': 'movie.ids.tmdb', 'trakt_movie_id': 'movie.ids.trakt', 'trakt_movie_slug': 'movie.ids.slug', + 'trakt_listed_at': 'listed_at', + 'trakt_collected_at': 'collected_at', }, 'show': { 'title': generate_show_title, @@ -75,6 +77,8 @@ def generate_episode_title(item): 'tmdb_id': 'show.ids.tmdb', 'trakt_show_id': 'show.ids.trakt', 'trakt_show_slug': 'show.ids.slug', + 'trakt_listed_at': 'listed_at', + 'trakt_last_collected_at': 'last_collected_at', }, 'episode': { 'title': generate_episode_title, @@ -91,6 +95,8 @@ def generate_episode_title(item): 'trakt_show_id': 'show.ids.trakt', 'trakt_show_slug': 'show.ids.slug', 'trakt_ep_name': 'episode.title', + 'trakt_listed_at': 'listed_at', + 'trakt_collected_at': 'collected_at', }, } diff --git a/flexget/components/trakt/trakt_lookup.py b/flexget/components/trakt/trakt_lookup.py index deb6c83a27..da1c99427c 100644 --- a/flexget/components/trakt/trakt_lookup.py +++ b/flexget/components/trakt/trakt_lookup.py @@ -221,6 +221,7 @@ def add_lazy_fields(entry: entry.Entry, lazy_lookup_name: str, media_type: str) user_data_fields = { 'collected': 'trakt_collected', 'watched': 'trakt_watched', + 'watched_at': 'trakt_watched_at', 'ratings': { 'show': 'trakt_series_user_rating', 'season': 'trakt_season_user_rating', @@ -348,6 +349,7 @@ def on_task_metainfo(self, task, config): if media_type: add_lazy_user_fields(entry, 'collected', media_type=media_type, **credentials) add_lazy_user_fields(entry, 'watched', media_type=media_type, **credentials) + add_lazy_user_fields(entry, 'watched_at', media_type=media_type, **credentials) if is_show(entry): add_lazy_user_fields(entry, 'ratings', media_type='show', **credentials) if is_season(entry): diff --git a/flexget/manager.py b/flexget/manager.py index a0f5096bb9..0e29d0aba9 100644 --- a/flexget/manager.py +++ b/flexget/manager.py @@ -268,7 +268,7 @@ def has_lock(self) -> bool: def execute( self, options: Union[dict, argparse.Namespace] = None, - priority: int = 1, + priority: int = None, suppress_warnings: Sequence[str] = None, ) -> List[Tuple[str, str, threading.Event]]: """ @@ -321,13 +321,14 @@ def execute( finished_events = [] for task_name in task_names: + task_priority = priority if priority else self.config['tasks'][task_name].get('priority', 1) task = Task( self, task_name, options=options, output=get_console_output(), session_id=flexget.log.get_log_session_id(), - priority=priority, + priority=task_priority, suppress_warnings=suppress_warnings, ) self.task_queue.put(task) diff --git a/flexget/plugins/modify/manipulate.py b/flexget/plugins/modify/manipulate.py index ad0f773dc6..b5c13ee078 100644 --- a/flexget/plugins/modify/manipulate.py +++ b/flexget/plugins/modify/manipulate.py @@ -109,6 +109,13 @@ def process(self, entry, jobs): from_field = field if 'from' in config: from_field = config['from'] + if config.get('remove'): + try: + del entry[field] + modified = True + except KeyError: + pass + continue field_value = entry.get(from_field) logger.debug( 'field: `{}` from_field: `{}` field_value: `{}`', @@ -116,11 +123,6 @@ def process(self, entry, jobs): from_field, field_value, ) - if config.get('remove'): - if field in entry: - del entry[field] - modified = True - continue if 'extract' in config: if not field_value: logger.warning('Cannot extract, field `{}` is not present', from_field) diff --git a/flexget/plugins/output/dump.py b/flexget/plugins/output/dump.py index 2e038c0d1f..a60b1040c1 100644 --- a/flexget/plugins/output/dump.py +++ b/flexget/plugins/output/dump.py @@ -97,7 +97,7 @@ class OutputDump: @plugin.priority(0) def on_task_output(self, task, config): - if not config and task.options.dump_entries is None: + if not config or task.options.dump_entries is None: return eval_lazy = 'eval' in task.options.dump_entries