From 2cfbad562f431f69443f2e565ae673dce6d671f7 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 19 Aug 2025 17:04:27 +0200 Subject: [PATCH 1/5] Add wrapper methods to allow using unique key only --- .../clients/resource_clients/request_queue.py | 143 +++++++++++++++++- uv.lock | 4 +- 2 files changed, 143 insertions(+), 4 deletions(-) diff --git a/src/apify_client/clients/resource_clients/request_queue.py b/src/apify_client/clients/resource_clients/request_queue.py index 14e57113..ff2cbe32 100644 --- a/src/apify_client/clients/resource_clients/request_queue.py +++ b/src/apify_client/clients/resource_clients/request_queue.py @@ -4,7 +4,10 @@ import json as jsonlib import logging import math +import re +from base64 import b64encode from collections.abc import Iterable +from hashlib import sha256 from queue import Queue from typing import TYPE_CHECKING, Any, TypedDict @@ -46,6 +49,32 @@ class BatchAddRequestsResult(TypedDict): unprocessedRequests: list[dict] +def unique_key_to_request_id(unique_key: str, *, request_id_length: int = 15) -> str: + """Generate a deterministic request ID based on a unique key. + + Args: + unique_key: The unique key to convert into a request ID. + request_id_length: The length of the request ID. + + Returns: + A URL-safe, truncated request ID based on the unique key. + """ + if not unique_key: + raise ValueError('unique_key must not be empty') + + # Encode the unique key and compute its SHA-256 hash + hashed_key = sha256(unique_key.encode('utf-8')).digest() + + # Encode the hash in base64 and decode it to get a string + base64_encoded = b64encode(hashed_key).decode('utf-8') + + # Remove characters that are not URL-safe ('+', '/', or '=') + url_safe_key = re.sub(r'(\+|\/|=)', '', base64_encoded) + + # Truncate the key to the desired length + return url_safe_key[:request_id_length] + + class RequestQueueClient(ResourceClient): """Sub-client for manipulating a single request queue.""" @@ -194,6 +223,19 @@ def get_request(self, request_id: str) -> dict | None: return None + def get_request_by_unique_key(self, request_unique_key: str) -> dict | None: + """Retrieve a request from the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/get-request + + Args: + request_unique_key: unique key of the request to retrieve. + + Returns: + The retrieved request, or None, if it did not exist. + """ + return self.get_request(unique_key_to_request_id(request_unique_key)) + def update_request(self, request: dict, *, forefront: bool | None = None) -> dict: """Update a request in the queue. @@ -206,7 +248,7 @@ def update_request(self, request: dict, *, forefront: bool | None = None) -> dic Returns: The updated request. """ - request_id = request['id'] + request_id = request.get('id', unique_key_to_request_id(request.get('unique_key', ''))) request_params = self._params(forefront=forefront, clientKey=self.client_key) @@ -239,6 +281,16 @@ def delete_request(self, request_id: str) -> None: timeout_secs=_SMALL_TIMEOUT, ) + def delete_request_by_unique_key(self, request_unique_key: str) -> None: + """Delete a request from the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/delete-request + + Args: + request_unique_key: Unique key of the request to delete. + """ + return self.delete_request(unique_key_to_request_id(request_unique_key)) + def prolong_request_lock( self, request_id: str, @@ -266,6 +318,26 @@ def prolong_request_lock( return parse_date_fields(pluck_data(jsonlib.loads(response.text))) + def prolong_request_lock_by_unique_key( + self, + request_unique_key: str, + *, + forefront: bool | None = None, + lock_secs: int, + ) -> dict: + """Prolong the lock on a request. + + https://docs.apify.com/api/v2#/reference/request-queues/request-lock/prolong-request-lock + + Args: + request_unique_key: ID of the request to prolong the lock. + forefront: Whether to put the request in the beginning or the end of the queue after lock expires. + lock_secs: By how much to prolong the lock, in seconds. + """ + return self.prolong_request_lock( + unique_key_to_request_id(request_unique_key), forefront=forefront, lock_secs=lock_secs + ) + def delete_request_lock(self, request_id: str, *, forefront: bool | None = None) -> None: """Delete the lock on a request. @@ -284,6 +356,17 @@ def delete_request_lock(self, request_id: str, *, forefront: bool | None = None) timeout_secs=_SMALL_TIMEOUT, ) + def delete_request_lock_by_unique_key(self, request_unique_key: str, *, forefront: bool | None = None) -> None: + """Delete the lock on a request. + + https://docs.apify.com/api/v2#/reference/request-queues/request-lock/delete-request-lock + + Args: + request_unique_key: ID of the request to delete the lock. + forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted. + """ + return self.delete_request_lock(unique_key_to_request_id(request_unique_key), forefront=forefront) + def batch_add_requests( self, requests: list[dict], @@ -574,6 +657,19 @@ async def get_request(self, request_id: str) -> dict | None: return None + async def get_request_by_unique_key(self, request_unique_key: str) -> dict | None: + """Retrieve a request from the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/get-request + + Args: + request_unique_key: unique key of the request to retrieve. + + Returns: + The retrieved request, or None, if it did not exist. + """ + return await self.get_request(unique_key_to_request_id(request_unique_key)) + async def update_request(self, request: dict, *, forefront: bool | None = None) -> dict: """Update a request in the queue. @@ -586,7 +682,7 @@ async def update_request(self, request: dict, *, forefront: bool | None = None) Returns: The updated request. """ - request_id = request['id'] + request_id = request.get('id', unique_key_to_request_id(request.get('unique_key', ''))) request_params = self._params(forefront=forefront, clientKey=self.client_key) @@ -617,6 +713,16 @@ async def delete_request(self, request_id: str) -> None: timeout_secs=_SMALL_TIMEOUT, ) + async def delete_request_by_unique_key(self, request_unique_key: str) -> None: + """Delete a request from the queue. + + https://docs.apify.com/api/v2#/reference/request-queues/request/delete-request + + Args: + request_unique_key: Unique key of the request to delete. + """ + return await self.delete_request(unique_key_to_request_id(request_unique_key)) + async def prolong_request_lock( self, request_id: str, @@ -644,6 +750,26 @@ async def prolong_request_lock( return parse_date_fields(pluck_data(jsonlib.loads(response.text))) + async def prolong_request_lock_by_unique_key( + self, + request_unique_key: str, + *, + forefront: bool | None = None, + lock_secs: int, + ) -> dict: + """Prolong the lock on a request. + + https://docs.apify.com/api/v2#/reference/request-queues/request-lock/prolong-request-lock + + Args: + request_unique_key: ID of the request to prolong the lock. + forefront: Whether to put the request in the beginning or the end of the queue after lock expires. + lock_secs: By how much to prolong the lock, in seconds. + """ + return await self.prolong_request_lock( + unique_key_to_request_id(request_unique_key), forefront=forefront, lock_secs=lock_secs + ) + async def delete_request_lock( self, request_id: str, @@ -667,6 +793,19 @@ async def delete_request_lock( timeout_secs=_SMALL_TIMEOUT, ) + async def delete_request_lock_by_unique_key( + self, request_unique_key: str, *, forefront: bool | None = None + ) -> None: + """Delete the lock on a request. + + https://docs.apify.com/api/v2#/reference/request-queues/request-lock/delete-request-lock + + Args: + request_unique_key: ID of the request to delete the lock. + forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted. + """ + return await self.delete_request_lock(unique_key_to_request_id(request_unique_key), forefront=forefront) + async def _batch_add_requests_worker( self, queue: asyncio.Queue[Iterable[dict]], diff --git a/uv.lock b/uv.lock index 70e0a525..7df1d39a 100644 --- a/uv.lock +++ b/uv.lock @@ -1,10 +1,10 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" [[package]] name = "apify-client" -version = "2.0.0" +version = "2.0.1" source = { editable = "." } dependencies = [ { name = "apify-shared" }, From 2ebd75c406e8836a27558b3cb3050868bb0471c4 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 19 Aug 2025 17:40:13 +0200 Subject: [PATCH 2/5] Update to uniqueKey --- src/apify_client/clients/resource_clients/request_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apify_client/clients/resource_clients/request_queue.py b/src/apify_client/clients/resource_clients/request_queue.py index ff2cbe32..19cb5776 100644 --- a/src/apify_client/clients/resource_clients/request_queue.py +++ b/src/apify_client/clients/resource_clients/request_queue.py @@ -248,7 +248,7 @@ def update_request(self, request: dict, *, forefront: bool | None = None) -> dic Returns: The updated request. """ - request_id = request.get('id', unique_key_to_request_id(request.get('unique_key', ''))) + request_id = request.get('id', unique_key_to_request_id(request.get('uniqueKey', ''))) request_params = self._params(forefront=forefront, clientKey=self.client_key) @@ -682,7 +682,7 @@ async def update_request(self, request: dict, *, forefront: bool | None = None) Returns: The updated request. """ - request_id = request.get('id', unique_key_to_request_id(request.get('unique_key', ''))) + request_id = request.get('id', unique_key_to_request_id(request.get('uniqueKey', ''))) request_params = self._params(forefront=forefront, clientKey=self.client_key) From 3729faca3e55db9858b89edb1d9730aa11039979 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 20 Aug 2025 08:47:12 +0200 Subject: [PATCH 3/5] Migrate tests --- .../clients/resource_clients/request_queue.py | 3 -- tests/integration/test_request_queue.py | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/apify_client/clients/resource_clients/request_queue.py b/src/apify_client/clients/resource_clients/request_queue.py index 19cb5776..c6ebbd59 100644 --- a/src/apify_client/clients/resource_clients/request_queue.py +++ b/src/apify_client/clients/resource_clients/request_queue.py @@ -59,9 +59,6 @@ def unique_key_to_request_id(unique_key: str, *, request_id_length: int = 15) -> Returns: A URL-safe, truncated request ID based on the unique key. """ - if not unique_key: - raise ValueError('unique_key must not be empty') - # Encode the unique key and compute its SHA-256 hash hashed_key = sha256(unique_key.encode('utf-8')).digest() diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py index 64759e47..ba1e52e1 100644 --- a/tests/integration/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -2,8 +2,12 @@ from typing import TYPE_CHECKING +import pytest + from integration.integration_test_utils import random_resource_name, random_string +from apify_client.clients.resource_clients.request_queue import unique_key_to_request_id + if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync @@ -113,3 +117,38 @@ async def test_request_batch_operations(self, apify_client_async: ApifyClientAsy assert len(requests_in_queue2['items']) == 25 - len(delete_response['processedRequests']) await queue.delete() + + +def test_unique_key_to_request_id_length() -> None: + unique_key = 'exampleKey123' + request_id = unique_key_to_request_id(unique_key, request_id_length=15) + assert len(request_id) == 15, 'Request ID should have the correct length.' + + +def test_unique_key_to_request_id_consistency() -> None: + unique_key = 'consistentKey' + request_id_1 = unique_key_to_request_id(unique_key) + request_id_2 = unique_key_to_request_id(unique_key) + assert request_id_1 == request_id_2, 'The same unique key should generate consistent request IDs.' + + +@pytest.mark.parametrize( + ('unique_key', 'expected_request_id'), + [ + ('abc', 'ungWv48BzpBQUDe'), + ('uniqueKey', 'xiWPs083cree7mH'), + ('', '47DEQpj8HBSaTIm'), + ('测试中文', 'lKPdJkdvw8MXEUp'), + ('test+/=', 'XZRQjhoG0yjfnYD'), + ], + ids=[ + 'basic_abc', + 'keyword_uniqueKey', + 'empty_string', + 'non_ascii_characters', + 'url_unsafe_characters', + ], +) +def test_unique_key_to_request_id_matches_known_values(unique_key: str, expected_request_id: str) -> None: + request_id = unique_key_to_request_id(unique_key) + assert request_id == expected_request_id, f'Unique key "{unique_key}" should produce the expected request ID.' From 9b7853ba91822f3ae52fce97dce1037f5c31aef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Proch=C3=A1zka?= Date: Wed, 20 Aug 2025 10:08:45 +0200 Subject: [PATCH 4/5] Update request_queue.py Update docstrings --- .../clients/resource_clients/request_queue.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/apify_client/clients/resource_clients/request_queue.py b/src/apify_client/clients/resource_clients/request_queue.py index c6ebbd59..049fff18 100644 --- a/src/apify_client/clients/resource_clients/request_queue.py +++ b/src/apify_client/clients/resource_clients/request_queue.py @@ -226,7 +226,7 @@ def get_request_by_unique_key(self, request_unique_key: str) -> dict | None: https://docs.apify.com/api/v2#/reference/request-queues/request/get-request Args: - request_unique_key: unique key of the request to retrieve. + request_unique_key: Unique key of the request to retrieve. Returns: The retrieved request, or None, if it did not exist. @@ -327,7 +327,7 @@ def prolong_request_lock_by_unique_key( https://docs.apify.com/api/v2#/reference/request-queues/request-lock/prolong-request-lock Args: - request_unique_key: ID of the request to prolong the lock. + request_unique_key: Unique key of the request to prolong the lock. forefront: Whether to put the request in the beginning or the end of the queue after lock expires. lock_secs: By how much to prolong the lock, in seconds. """ @@ -359,7 +359,7 @@ def delete_request_lock_by_unique_key(self, request_unique_key: str, *, forefron https://docs.apify.com/api/v2#/reference/request-queues/request-lock/delete-request-lock Args: - request_unique_key: ID of the request to delete the lock. + request_unique_key: Unique key of the request to delete the lock. forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted. """ return self.delete_request_lock(unique_key_to_request_id(request_unique_key), forefront=forefront) @@ -660,7 +660,7 @@ async def get_request_by_unique_key(self, request_unique_key: str) -> dict | Non https://docs.apify.com/api/v2#/reference/request-queues/request/get-request Args: - request_unique_key: unique key of the request to retrieve. + request_unique_key: Unique key of the request to retrieve. Returns: The retrieved request, or None, if it did not exist. @@ -759,7 +759,7 @@ async def prolong_request_lock_by_unique_key( https://docs.apify.com/api/v2#/reference/request-queues/request-lock/prolong-request-lock Args: - request_unique_key: ID of the request to prolong the lock. + request_unique_key: Unique key of the request to prolong the lock. forefront: Whether to put the request in the beginning or the end of the queue after lock expires. lock_secs: By how much to prolong the lock, in seconds. """ @@ -798,7 +798,7 @@ async def delete_request_lock_by_unique_key( https://docs.apify.com/api/v2#/reference/request-queues/request-lock/delete-request-lock Args: - request_unique_key: ID of the request to delete the lock. + request_unique_key: Unique key of the request to delete the lock. forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted. """ return await self.delete_request_lock(unique_key_to_request_id(request_unique_key), forefront=forefront) From 0b0d3ca13c7ddc06cd15a1476c401e4a179f962f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 20 Aug 2025 11:53:38 +0200 Subject: [PATCH 5/5] Review comments --- .../clients/resource_clients/request_queue.py | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/src/apify_client/clients/resource_clients/request_queue.py b/src/apify_client/clients/resource_clients/request_queue.py index 049fff18..095fff8a 100644 --- a/src/apify_client/clients/resource_clients/request_queue.py +++ b/src/apify_client/clients/resource_clients/request_queue.py @@ -220,18 +220,18 @@ def get_request(self, request_id: str) -> dict | None: return None - def get_request_by_unique_key(self, request_unique_key: str) -> dict | None: + def get_request_by_unique_key(self, unique_key: str) -> dict | None: """Retrieve a request from the queue. https://docs.apify.com/api/v2#/reference/request-queues/request/get-request Args: - request_unique_key: Unique key of the request to retrieve. + unique_key: Unique key of the request to retrieve. Returns: The retrieved request, or None, if it did not exist. """ - return self.get_request(unique_key_to_request_id(request_unique_key)) + return self.get_request(unique_key_to_request_id(unique_key)) def update_request(self, request: dict, *, forefront: bool | None = None) -> dict: """Update a request in the queue. @@ -278,15 +278,15 @@ def delete_request(self, request_id: str) -> None: timeout_secs=_SMALL_TIMEOUT, ) - def delete_request_by_unique_key(self, request_unique_key: str) -> None: + def delete_request_by_unique_key(self, unique_key: str) -> None: """Delete a request from the queue. https://docs.apify.com/api/v2#/reference/request-queues/request/delete-request Args: - request_unique_key: Unique key of the request to delete. + unique_key: Unique key of the request to delete. """ - return self.delete_request(unique_key_to_request_id(request_unique_key)) + return self.delete_request(unique_key_to_request_id(unique_key)) def prolong_request_lock( self, @@ -317,7 +317,7 @@ def prolong_request_lock( def prolong_request_lock_by_unique_key( self, - request_unique_key: str, + unique_key: str, *, forefront: bool | None = None, lock_secs: int, @@ -327,13 +327,11 @@ def prolong_request_lock_by_unique_key( https://docs.apify.com/api/v2#/reference/request-queues/request-lock/prolong-request-lock Args: - request_unique_key: Unique key of the request to prolong the lock. + unique_key: Unique key of the request to prolong the lock. forefront: Whether to put the request in the beginning or the end of the queue after lock expires. lock_secs: By how much to prolong the lock, in seconds. """ - return self.prolong_request_lock( - unique_key_to_request_id(request_unique_key), forefront=forefront, lock_secs=lock_secs - ) + return self.prolong_request_lock(unique_key_to_request_id(unique_key), forefront=forefront, lock_secs=lock_secs) def delete_request_lock(self, request_id: str, *, forefront: bool | None = None) -> None: """Delete the lock on a request. @@ -353,16 +351,16 @@ def delete_request_lock(self, request_id: str, *, forefront: bool | None = None) timeout_secs=_SMALL_TIMEOUT, ) - def delete_request_lock_by_unique_key(self, request_unique_key: str, *, forefront: bool | None = None) -> None: + def delete_request_lock_by_unique_key(self, unique_key: str, *, forefront: bool | None = None) -> None: """Delete the lock on a request. https://docs.apify.com/api/v2#/reference/request-queues/request-lock/delete-request-lock Args: - request_unique_key: Unique key of the request to delete the lock. + unique_key: Unique key of the request to delete the lock. forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted. """ - return self.delete_request_lock(unique_key_to_request_id(request_unique_key), forefront=forefront) + return self.delete_request_lock(unique_key_to_request_id(unique_key), forefront=forefront) def batch_add_requests( self, @@ -654,18 +652,18 @@ async def get_request(self, request_id: str) -> dict | None: return None - async def get_request_by_unique_key(self, request_unique_key: str) -> dict | None: + async def get_request_by_unique_key(self, unique_key: str) -> dict | None: """Retrieve a request from the queue. https://docs.apify.com/api/v2#/reference/request-queues/request/get-request Args: - request_unique_key: Unique key of the request to retrieve. + unique_key: Unique key of the request to retrieve. Returns: The retrieved request, or None, if it did not exist. """ - return await self.get_request(unique_key_to_request_id(request_unique_key)) + return await self.get_request(unique_key_to_request_id(unique_key)) async def update_request(self, request: dict, *, forefront: bool | None = None) -> dict: """Update a request in the queue. @@ -710,15 +708,15 @@ async def delete_request(self, request_id: str) -> None: timeout_secs=_SMALL_TIMEOUT, ) - async def delete_request_by_unique_key(self, request_unique_key: str) -> None: + async def delete_request_by_unique_key(self, unique_key: str) -> None: """Delete a request from the queue. https://docs.apify.com/api/v2#/reference/request-queues/request/delete-request Args: - request_unique_key: Unique key of the request to delete. + unique_key: Unique key of the request to delete. """ - return await self.delete_request(unique_key_to_request_id(request_unique_key)) + return await self.delete_request(unique_key_to_request_id(unique_key)) async def prolong_request_lock( self, @@ -749,7 +747,7 @@ async def prolong_request_lock( async def prolong_request_lock_by_unique_key( self, - request_unique_key: str, + unique_key: str, *, forefront: bool | None = None, lock_secs: int, @@ -759,12 +757,12 @@ async def prolong_request_lock_by_unique_key( https://docs.apify.com/api/v2#/reference/request-queues/request-lock/prolong-request-lock Args: - request_unique_key: Unique key of the request to prolong the lock. + unique_key: Unique key of the request to prolong the lock. forefront: Whether to put the request in the beginning or the end of the queue after lock expires. lock_secs: By how much to prolong the lock, in seconds. """ return await self.prolong_request_lock( - unique_key_to_request_id(request_unique_key), forefront=forefront, lock_secs=lock_secs + unique_key_to_request_id(unique_key), forefront=forefront, lock_secs=lock_secs ) async def delete_request_lock( @@ -790,18 +788,16 @@ async def delete_request_lock( timeout_secs=_SMALL_TIMEOUT, ) - async def delete_request_lock_by_unique_key( - self, request_unique_key: str, *, forefront: bool | None = None - ) -> None: + async def delete_request_lock_by_unique_key(self, unique_key: str, *, forefront: bool | None = None) -> None: """Delete the lock on a request. https://docs.apify.com/api/v2#/reference/request-queues/request-lock/delete-request-lock Args: - request_unique_key: Unique key of the request to delete the lock. + unique_key: Unique key of the request to delete the lock. forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted. """ - return await self.delete_request_lock(unique_key_to_request_id(request_unique_key), forefront=forefront) + return await self.delete_request_lock(unique_key_to_request_id(unique_key), forefront=forefront) async def _batch_add_requests_worker( self,