From 753465b5f48e31f2ec214b12fa720e6dc8267f42 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 12 Aug 2025 13:06:43 +0000 Subject: [PATCH] Remove the `parse_response` parameter for the `call` method --- src/apify_client/_http_client.py | 41 +------------------ src/apify_client/_utils.py | 33 ++++++++++++++- .../clients/resource_clients/dataset.py | 4 -- .../resource_clients/key_value_store.py | 15 +++---- .../clients/resource_clients/log.py | 4 -- tests/unit/test_client_errors.py | 4 +- uv.lock | 2 +- 7 files changed, 45 insertions(+), 58 deletions(-) diff --git a/src/apify_client/_http_client.py b/src/apify_client/_http_client.py index 74d98553..9ac76af2 100644 --- a/src/apify_client/_http_client.py +++ b/src/apify_client/_http_client.py @@ -10,9 +10,9 @@ from typing import TYPE_CHECKING, Any import httpx -from apify_shared.utils import ignore_docs, is_content_type_json, is_content_type_text, is_content_type_xml +from apify_shared.utils import ignore_docs -from apify_client._errors import ApifyApiError, InvalidResponseBodyError, is_retryable_error +from apify_client._errors import ApifyApiError, is_retryable_error from apify_client._logging import log_context, logger_name from apify_client._statistics import Statistics from apify_client._utils import retry_with_exp_backoff, retry_with_exp_backoff_async @@ -64,25 +64,6 @@ def __init__( self.stats = stats or Statistics() - @staticmethod - def _maybe_parse_response(response: httpx.Response) -> Any: - if response.status_code == HTTPStatus.NO_CONTENT: - return None - - content_type = '' - if 'content-type' in response.headers: - content_type = response.headers['content-type'].split(';')[0].strip() - - try: - if is_content_type_json(content_type): - return response.json() - elif is_content_type_xml(content_type) or is_content_type_text(content_type): # noqa: RET505 - return response.text - else: - return response.content - except ValueError as err: - raise InvalidResponseBodyError(response) from err - @staticmethod def _parse_params(params: dict | None) -> dict | None: if params is None: @@ -143,7 +124,6 @@ def call( data: Any = None, json: JSONSerializable | None = None, stream: bool | None = None, - parse_response: bool | None = True, timeout_secs: int | None = None, ) -> httpx.Response: log_context.method.set(method) @@ -151,9 +131,6 @@ def call( self.stats.calls += 1 - if stream and parse_response: - raise ValueError('Cannot stream response and parse it at the same time!') - headers, params, content = self._prepare_request_call(headers, params, data, json) httpx_client = self.httpx_client @@ -190,11 +167,6 @@ def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response: # If response status is < 300, the request was successful, and we can return the result if response.status_code < 300: # noqa: PLR2004 logger.debug('Request successful', extra={'status_code': response.status_code}) - if not stream: - _maybe_parsed_body = ( - self._maybe_parse_response(response) if parse_response else response.content - ) - setattr(response, '_maybe_parsed_body', _maybe_parsed_body) # noqa: B010 return response @@ -239,7 +211,6 @@ async def call( data: Any = None, json: JSONSerializable | None = None, stream: bool | None = None, - parse_response: bool | None = True, timeout_secs: int | None = None, ) -> httpx.Response: log_context.method.set(method) @@ -247,9 +218,6 @@ async def call( self.stats.calls += 1 - if stream and parse_response: - raise ValueError('Cannot stream response and parse it at the same time!') - headers, params, content = self._prepare_request_call(headers, params, data, json) httpx_async_client = self.httpx_async_client @@ -283,11 +251,6 @@ async def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response # If response status is < 300, the request was successful, and we can return the result if response.status_code < 300: # noqa: PLR2004 logger.debug('Request successful', extra={'status_code': response.status_code}) - if not stream: - _maybe_parsed_body = ( - self._maybe_parse_response(response) if parse_response else response.content - ) - setattr(response, '_maybe_parsed_body', _maybe_parsed_body) # noqa: B010 return response diff --git a/src/apify_client/_utils.py b/src/apify_client/_utils.py index 512d492b..0f12238e 100644 --- a/src/apify_client/_utils.py +++ b/src/apify_client/_utils.py @@ -9,11 +9,21 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any, TypeVar, cast -from apify_shared.utils import is_file_or_bytes, maybe_extract_enum_member_value +from apify_shared.utils import ( + is_content_type_json, + is_content_type_text, + is_content_type_xml, + is_file_or_bytes, + maybe_extract_enum_member_value, +) + +from apify_client._errors import InvalidResponseBodyError if TYPE_CHECKING: from collections.abc import Awaitable + from httpx import Response + from apify_client._errors import ApifyApiError PARSE_DATE_FIELDS_MAX_DEPTH = 3 @@ -149,3 +159,24 @@ def encode_key_value_store_record_value(value: Any, content_type: str | None = N value = json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False, default=str).encode('utf-8') return (value, content_type) + + +def maybe_parse_response(response: Response) -> Any: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + + content_type = '' + if 'content-type' in response.headers: + content_type = response.headers['content-type'].split(';')[0].strip() + + try: + if is_content_type_json(content_type): + response_data = response.json() + elif is_content_type_xml(content_type) or is_content_type_text(content_type): + response_data = response.text + else: + response_data = response.content + except ValueError as err: + raise InvalidResponseBodyError(response) from err + else: + return response_data diff --git a/src/apify_client/clients/resource_clients/dataset.py b/src/apify_client/clients/resource_clients/dataset.py index 368eef0b..130b5a09 100644 --- a/src/apify_client/clients/resource_clients/dataset.py +++ b/src/apify_client/clients/resource_clients/dataset.py @@ -420,7 +420,6 @@ def get_items_as_bytes( url=self._url('items'), method='GET', params=request_params, - parse_response=False, ) return response.content @@ -516,7 +515,6 @@ def stream_items( method='GET', params=request_params, stream=True, - parse_response=False, ) yield response finally: @@ -877,7 +875,6 @@ async def get_items_as_bytes( url=self._url('items'), method='GET', params=request_params, - parse_response=False, ) return response.content @@ -973,7 +970,6 @@ async def stream_items( method='GET', params=request_params, stream=True, - parse_response=False, ) yield response finally: diff --git a/src/apify_client/clients/resource_clients/key_value_store.py b/src/apify_client/clients/resource_clients/key_value_store.py index 7100d475..fcbeddab 100644 --- a/src/apify_client/clients/resource_clients/key_value_store.py +++ b/src/apify_client/clients/resource_clients/key_value_store.py @@ -8,7 +8,12 @@ from apify_shared.utils import filter_out_none_values_recursively, ignore_docs, parse_date_fields from apify_client._errors import ApifyApiError -from apify_client._utils import catch_not_found_or_throw, encode_key_value_store_record_value, pluck_data +from apify_client._utils import ( + catch_not_found_or_throw, + encode_key_value_store_record_value, + maybe_parse_response, + pluck_data, +) from apify_client.clients.base import ResourceClient, ResourceClientAsync if TYPE_CHECKING: @@ -146,7 +151,7 @@ def get_record(self, key: str, *, as_bytes: bool = False, as_file: bool = False) return { 'key': key, - 'value': response._maybe_parsed_body, # type: ignore[attr-defined] # noqa: SLF001 + 'value': maybe_parse_response(response), 'content_type': response.headers['content-type'], } @@ -196,7 +201,6 @@ def get_record_as_bytes(self, key: str) -> dict | None: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, ) return { @@ -228,7 +232,6 @@ def stream_record(self, key: str) -> Iterator[dict | None]: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, stream=True, ) @@ -393,7 +396,7 @@ async def get_record(self, key: str) -> dict | None: return { 'key': key, - 'value': response._maybe_parsed_body, # type: ignore[attr-defined] # noqa: SLF001 + 'value': maybe_parse_response(response), 'content_type': response.headers['content-type'], } @@ -443,7 +446,6 @@ async def get_record_as_bytes(self, key: str) -> dict | None: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, ) return { @@ -475,7 +477,6 @@ async def stream_record(self, key: str) -> AsyncIterator[dict | None]: url=self._url(f'records/{key}'), method='GET', params=self._params(), - parse_response=False, stream=True, ) diff --git a/src/apify_client/clients/resource_clients/log.py b/src/apify_client/clients/resource_clients/log.py index bca8b07a..6abfdd53 100644 --- a/src/apify_client/clients/resource_clients/log.py +++ b/src/apify_client/clients/resource_clients/log.py @@ -76,7 +76,6 @@ def get_as_bytes(self, *, raw: bool = False) -> bytes | None: url=self.url, method='GET', params=self._params(raw=raw), - parse_response=False, ) return response.content # noqa: TRY300 @@ -105,7 +104,6 @@ def stream(self, *, raw: bool = False) -> Iterator[httpx.Response | None]: method='GET', params=self._params(stream=True, raw=raw), stream=True, - parse_response=False, ) yield response @@ -166,7 +164,6 @@ async def get_as_bytes(self, *, raw: bool = False) -> bytes | None: url=self.url, method='GET', params=self._params(raw=raw), - parse_response=False, ) return response.content # noqa: TRY300 @@ -195,7 +192,6 @@ async def stream(self, *, raw: bool = False) -> AsyncIterator[httpx.Response | N method='GET', params=self._params(stream=True, raw=raw), stream=True, - parse_response=False, ) yield response diff --git a/tests/unit/test_client_errors.py b/tests/unit/test_client_errors.py index ca410ce4..394f7851 100644 --- a/tests/unit/test_client_errors.py +++ b/tests/unit/test_client_errors.py @@ -92,7 +92,7 @@ def test_client_apify_api_error_streamed(httpserver: HTTPServer) -> None: httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler) with pytest.raises(ApifyApiError) as e: - client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False) + client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True) assert e.value.message == error['error']['message'] assert e.value.type == error['error']['type'] @@ -108,7 +108,7 @@ async def test_async_client_apify_api_error_streamed(httpserver: HTTPServer) -> httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler) with pytest.raises(ApifyApiError) as e: - await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False) + await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True) assert e.value.message == error['error']['message'] assert e.value.type == error['error']['type'] diff --git a/uv.lock b/uv.lock index 4e297b08..26e7a5df 100644 --- a/uv.lock +++ b/uv.lock @@ -23,7 +23,7 @@ wheels = [ [[package]] name = "apify-client" -version = "1.12.2" +version = "1.12.3" source = { editable = "." } dependencies = [ { name = "apify-shared" },