Request requires 1 credits but only 0 credits are available ERROR - async download using smbclient #312
Description
From time to time I get this error: "Request requires 1 credits but only 0 credits are available"
I'm trying to download multiple files in parallel (async). It occurs with 3 files each +500MB. Any pointers/ideas how I could fix this? Thanks!
These are my 2 code files:
"""Samba downloader module."""
import asyncio
from dataclasses import dataclass
from pathlib import Path
from pydantic import FileUrl
from .samba.smb_creds import SmbCreds, get_creds
from .samba.smb_reader import SmbReader
@dataclass
class SmbDownloadItem:
"""SMB download item."""
src_url: FileUrl
dst_path: Path
def __str__(self) -> str:
"""Return the string representation of the object.
:return: The string representation.
:rtype: str
"""
return f"src: '{str(self.src_url)}', dst: '{str(self.dst_path)}'"
class SmbDownloader:
"""SMB downloader class."""
def __init__(
self,
download_items: list[SmbDownloadItem],
smb_creds: SmbCreds | None = None,
):
"""Initialize the samba reader.
You can inject the SMB credentials as an argument or use the default.
In case of Windows, the current user's credentials are used.
:param smb_creds: SMB credentials
:ptype smb_creds: SmbCreds
"""
self.download_items = download_items
self.smb_creds = smb_creds or get_creds()
self.smb_reader = SmbReader(smb_creds=self.smb_creds)
self.download_errors = []
def get_download_tasks(self) -> list:
"""Get the download tasks.
:return: The download tasks.
:rtype: list
"""
tasks = []
for item in self.download_items:
tasks.append(
asyncio.to_thread(self.smb_reader.download, item.src_url, item.dst_path)
)
return tasks
async def download(self):
"""Download the files.
When finished, the download errors are stored in the download_errors list.
"""
tasks = self.get_download_tasks()
results = await asyncio.gather(*tasks, return_exceptions=True)
for result in results:
if isinstance(result, Exception):
self.download_errors.append(result)
"""Samba reader module."""
import logging
import shutil
from pathlib import Path
import smbclient
from pydantic import FileUrl
from smbprotocol.exceptions import SMBOSError
from .constants import SMB_CHUNK_SIZE
from .exceptions import SmbDownloadError
from .samba.smb_creds import SmbCreds, get_creds
from .samba.smb_path_parts import SmbPathParts
logger = logging.getLogger(__name__)
class SmbReader:
"""Samba reader class."""
def __init__(
self,
smb_creds: SmbCreds | None = None,
):
"""Initialize the samba reader.
You can inject the SMB credentials as an argument or use the default.
In case of Windows, the current user's credentials are used.
:param smb_creds: SMB credentials
:ptype smb_creds: SmbCreds
"""
self.smb_creds = smb_creds or get_creds()
self._registered_servers = set()
if self.smb_creds is not None:
smbclient.ClientConfig(
username=self.smb_creds.username,
password=self.smb_creds.password,
domain=self.smb_creds.domain,
)
def _register_session(self, smb_url_parts: SmbPathParts):
"""Register an SMB session.
:param smb_url_parts: The SMB URL parts.
:type smb_url_parts: SmbPathParts
"""
if smb_url_parts.server_name in self._registered_servers:
return # Already registered
if self.smb_creds is None:
smbclient.register_session(server=smb_url_parts.server_name)
else:
smbclient.register_session(
server=smb_url_parts.server_name,
username=self.smb_creds.username,
password=self.smb_creds.password,
)
self._registered_servers.add(smb_url_parts.server_name)
def download(self, source_url: FileUrl, destination_path: Path):
"""Download the file from an SMB share to a local directory.
To avoid lock-handles on the SMB share, the 'share_access' parameter is set to
'r'. This allows other handles to be opened with read access.
We don't set 'w' and 'd' because we want to lock the file for writing.
:param source_url: The file URL (e.g., file://fileserver/share/path/to/file.txt)
:param destination_path: The full destination path to save the file.
:raises SmbDownloadError: If the download fails.
"""
smb_url_parts = SmbPathParts.from_file_url(source_url)
self._register_session(smb_url_parts)
destination_path.parent.mkdir(parents=True, exist_ok=True)
logger.info(f"Downloading '{str(source_url)}' to '{str(destination_path)}' ...")
try:
with smbclient.open_file(
smb_url_parts.unc_path, mode="rb", share_access="r"
) as remote_file, destination_path.open("wb") as local_file:
shutil.copyfileobj(remote_file, local_file, length=SMB_CHUNK_SIZE)
except Exception as e:
logger.debug(
f"Failed to download file '{str(source_url)}' to "
f"'{str(destination_path)}' : {e}"
)
raise SmbDownloadError(source_url, destination_path, e) from e
def exists(self, source_url: FileUrl) -> bool:
"""Check if the file exists on the SMB share.
:param source_url: The file URL (e.g., file://fileserver/share/path/to/file.txt)
:return: True if the file exists, False otherwise.
"""
smb_url_parts = SmbPathParts.from_file_url(source_url)
self._register_session(smb_url_parts)
try:
smbclient.stat(smb_url_parts.unc_path)
except SMBOSError as e:
logger.debug(f"File does not exist: {e}")
return False
return True