Skip to content

添加简单域名选择逻辑 #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
broqdev wants to merge 2 commits into
base: master
Choose a base branch
from
Open

添加简单域名选择逻辑 #247

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
152dff1
select url by retry count instead of pure randomness
broqdev May 22, 2025
bb28d0f
remove uncessary code
broqdev May 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion bilix/download/base_downloader_part.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

__all__ = ['BaseDownloaderPart']

domain_scores = {}


class BaseDownloaderPart(BaseDownloader):
"""Base Async http Content-Range Downloader"""
Expand Down Expand Up @@ -199,6 +201,7 @@ async def get_file(self, url_or_urls: Union[str, Iterable[str]], path: Union[Pat

async def _get_file_part(self, urls: List[str], path: Path, part_range: Tuple[int, int],
task_id) -> Path:
global domain_scores
start, end = part_range
part_path = path.with_name(f'{path.name}.{part_range[0]}-{part_range[1]}')
exist, part_path = path_check(part_path)
Expand All @@ -208,10 +211,20 @@ async def _get_file_part(self, urls: List[str], path: Path, part_range: Tuple[in
await self.progress.update(task_id, advance=downloaded)
if start > end:
return part_path # skip already finished
url_idx = random.randint(0, len(urls) - 1)

for times in range(1 + self.stream_retry):
# find domain with min score in domain_scores
domains = [urlparse(url_).netloc for url_ in urls]
domain_idx = min(range(len(domains)), key=lambda i: domain_scores.get(domains[i], 0))
url_idx = domain_idx
# update domain_scores
domain_min_score = domain_scores.get(domains[domain_idx], 0)
for domain in domain_scores.keys():
domain_scores[domain] = domain_scores[domain] - domain_min_score

try:
# parse domain from url
domain = urlparse(urls[url_idx]).netloc
async with \
self.client.stream("GET", urls[url_idx], follow_redirects=True,
headers={'Range': f'bytes={start}-{end}'}) as r, \
Expand All @@ -227,6 +240,7 @@ async def _get_file_part(self, urls: List[str], path: Path, part_range: Tuple[in
await self._check_speed(len(chunk))
break
except (httpx.HTTPStatusError, httpx.TransportError):
domain_scores[domain] = domain_scores.get(domain, 0) + 1
continue
else:
raise Exception(f"STREAM 超过重复次数 {part_path.name}")
Expand Down