From c29123f2152238564cd812842e59a2bdaa1978c7 Mon Sep 17 00:00:00 2001 From: letterkey <459994620@qq.com> Date: Tue, 3 Feb 2026 17:21:37 +0800 Subject: [PATCH 1/3] =?UTF-8?q?[=E4=BB=BB=E5=8A=A1=E8=AF=A6=E6=83=85]-http?= =?UTF-8?q?s://github.com/934050259/BrowserCluster/issues/5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/src/views/Tasks.vue | 3 ++- app/core/browser.py | 3 ++- app/core/scraper.py | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/admin/src/views/Tasks.vue b/admin/src/views/Tasks.vue index 5ab03dc..ab4b7bb 100644 --- a/admin/src/views/Tasks.vue +++ b/admin/src/views/Tasks.vue @@ -492,9 +492,10 @@

完整 URL: {{ req.url }}

状态码: {{ req.status }}

+

Headers: {{ req.headers }}

响应内容: - 
{{ formatJSON(req.content) }}
+ 
{{ formatJSON(req.body) }}
diff --git a/app/core/browser.py b/app/core/browser.py index cf87a0b..6272279 100644 --- a/app/core/browser.py +++ b/app/core/browser.py @@ -6,6 +6,7 @@ import asyncio import logging import sys +import tempfile import threading import time from typing import Optional @@ -101,13 +102,13 @@ async def get_browser(self) -> Browser: settings.browser_type, playwright.chromium ) - # 启动浏览器参数 launch_args = [] # 反检测参数 if settings.stealth_mode: launch_args.extend([ + '--disable-web-security', "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", diff --git a/app/core/scraper.py b/app/core/scraper.py index f4753a2..9e23820 100644 --- a/app/core/scraper.py +++ b/app/core/scraper.py @@ -13,7 +13,6 @@ from app.core.browser import browser_manager from app.core.config import settings - class Scraper: """网页抓取器""" @@ -52,7 +51,7 @@ async def scrape( # 创建浏览器上下文参数 context_options = { "java_script_enabled": True, - "user_agent": user_agent + # "user_agent": user_agent } if proxy_config: @@ -108,8 +107,8 @@ async def scrape( await page.set_viewport_size(params["viewport"]) # 注入反检测脚本 - if params.get("stealth", settings.stealth_mode): - await Stealth().apply_stealth_async(page) + # if params.get("stealth", settings.stealth_mode): + # await Stealth().apply_stealth_async(page) # 设置接口拦截 intercept_apis = params.get("intercept_apis", []) From 0ef953d25bd8be39812840f4028ba0849b2d55a8 Mon Sep 17 00:00:00 2001 From: letterkey <459994620@qq.com> Date: Wed, 4 Feb 2026 17:13:55 +0800 Subject: [PATCH 2/3] =?UTF-8?q?[=E4=BB=BB=E5=8A=A1=E7=AE=A1=E7=90=86]-?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BB=BB=E5=8A=A1=E5=BF=AB=E9=80=9F=E5=A4=8D?= =?UTF-8?q?=E5=88=B6/=E4=BB=BB=E5=8A=A1=E8=AE=BE=E7=BD=AE=E9=A2=9D?= =?UTF-8?q?=E5=A4=96=E7=AD=89=E5=BE=85=E6=97=B6=E9=97=B4(=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E6=8B=A6=E6=88=AA=E6=8E=A5=E5=8F=A3=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E6=8B=A6=E6=88=AA=E5=88=B0=E7=9A=84=E9=97=AE=E9=A2=98)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/src/views/Tasks.vue | 34 ++++------------------------------ app/core/scraper.py | 3 +-- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/admin/src/views/Tasks.vue b/admin/src/views/Tasks.vue index e46e18d..9ad0115 100644 --- a/admin/src/views/Tasks.vue +++ b/admin/src/views/Tasks.vue @@ -544,6 +544,10 @@ + + + +
@@ -637,17 +641,6 @@ - -
API 拦截配置
- - -
支持通配符 * 和正则表达式
-
@@ -1312,7 +1305,6 @@ const submitMode = ref('single') // 'single' | 'batch' const batchMode = ref('text') // 'text' | 'file' const batchUrlsText = ref('') const batchFileUrls = ref([]) -const interceptApisText = ref('') // 解析配置相关状态 const llmFieldOptions = [ @@ -1638,13 +1630,6 @@ const copyTask = async (task) => { ] } - // 处理API拦截配置 - if (scrapeForm.value.params.intercept_apis && Array.isArray(scrapeForm.value.params.intercept_apis)) { - interceptApisText.value = scrapeForm.value.params.intercept_apis.join('\n') - } else { - interceptApisText.value = '' - } - activeConfigTab.value = 'basic' showScrapeDialog.value = true ElMessage.success('任务配置已复制到新建任务表单') @@ -1774,16 +1759,6 @@ const submitTask = async () => { if (!data.params.proxy.username) delete data.params.proxy.username if (!data.params.proxy.password) delete data.params.proxy.password } - - // 处理API拦截配置 - if (interceptApisText.value.trim()) { - data.params.intercept_apis = interceptApisText.value - .split('\n') - .map(line => line.trim()) - .filter(line => line) - } else if (!data.params.intercept_apis || data.params.intercept_apis.length === 0) { - data.params.intercept_apis = null - } if (data.params.cookies) { const cookieVal = data.params.cookies.trim() @@ -1883,7 +1858,6 @@ const resetForm = () => { { field: 'title', path: '//h1' }, { field: 'content', path: "//div[@class='article-body']" } ] - interceptApisText.value = '' lastCheckedDomain = '' matchedRules.value = [] activeConfigTab.value = 'basic' diff --git a/app/core/scraper.py b/app/core/scraper.py index 25d8578..d457661 100644 --- a/app/core/scraper.py +++ b/app/core/scraper.py @@ -320,12 +320,11 @@ async def route_handler(route, request): # 检查请求 URL 是否匹配任何拦截模式 request_url = request.url matched_pattern = None - for pattern in api_patterns: if url_matches_pattern(request_url, pattern): matched_pattern = pattern break - + print("【拦截URL】- "+ request_url) if matched_pattern: # 拦截请求，获取响应 try: From c89eeb4a0d323931d5150f281fd19dfde57706e5 Mon Sep 17 00:00:00 2001 From: letterkey <459994620@qq.com> Date: Wed, 4 Feb 2026 17:28:23 +0800 Subject: [PATCH 3/3] =?UTF-8?q?[=E4=BB=BB=E5=8A=A1=E7=AE=A1=E7=90=86]-?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BB=BB=E5=8A=A1=E5=BF=AB=E9=80=9F=E5=A4=8D?= =?UTF-8?q?=E5=88=B6/=E4=BB=BB=E5=8A=A1=E8=AE=BE=E7=BD=AE=E9=A2=9D?= =?UTF-8?q?=E5=A4=96=E7=AD=89=E5=BE=85=E6=97=B6=E9=97=B4(=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E6=8B=A6=E6=88=AA=E6=8E=A5=E5=8F=A3=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E6=8B=A6=E6=88=AA=E5=88=B0=E7=9A=84=E9=97=AE=E9=A2=98)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/scraper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/core/scraper.py b/app/core/scraper.py index d457661..09fde83 100644 --- a/app/core/scraper.py +++ b/app/core/scraper.py @@ -147,8 +147,8 @@ async def scrape( await page.set_viewport_size(params["viewport"]) # 注入反检测脚本 - # if params.get("stealth", settings.stealth_mode): - # await Stealth().apply_stealth_async(page) + if params.get("stealth", settings.stealth_mode): + await Stealth().apply_stealth_async(page) # 设置接口拦截 intercept_apis = params.get("intercept_apis", []) @@ -324,7 +324,7 @@ async def route_handler(route, request): if url_matches_pattern(request_url, pattern): matched_pattern = pattern break - print("【拦截URL】- "+ request_url) + if matched_pattern: # 拦截请求，获取响应 try: