feat: 浏览器扩展支持交互式页面操作 (browser_act)

在原有只读抓取 (browser_fetch_page) 基础上,新增 observe→act 循环,
让 Agent 能在用户的 Chrome 中实际操作网页:点击/输入/滚动/等待/页内跳转。

后端:
- 新工具 browser_act,按 session 复用标签页,返回带编号的可交互元素清单+正文
- browser_context 任务队列扩展 kind/session_id/action 与 observation 结果
- /api/browser-context 路由下发 act 任务、接收 observation
- BrowserContextConfig 增加 actions_enabled 等开关,tools.py 两级门控
- 接线 app.py 路由挂载、app_config.py 配置字段、config.yaml 工具注册

前端:
- ChatPage 处理「发送当前页」回跳的 browser_context query 参数并自动提交

扩展 (browser-extension/deer-web-clipper):
- shared.js 自包含注入函数 deerObservePage / deerPerformAction
- background.js 按 kind 分发任务,session→tab 映射存 chrome.storage.session

测试:tests/test_browser_act.py(队列往返 + 工具校验 + 模拟扩展成功路径,8 passed)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-08 13:47:00 +08:00
1 parent 0001f86826
commit d4a71ce384
24 files changed
+4499 -162

No files matched your search

@@ -29,9 +29,10 @@ from app.gateway.routers import (
ai_writing,
admin_users,
agents,
artifacts,
assistants_compat,
auth,
artifacts,
assistants_compat,
auth,
browser_context,
channels,
checkpoint_migration,
feedback,
@@ -1197,11 +1198,14 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
app.include_router(artifacts.router)
# Uploads API is mounted at /api/threads/{thread_id}/uploads
app.include_router(uploads.router)
# Thread cleanup API is mounted at /api/threads/{thread_id}
app.include_router(threads.router)
# Uploads API is mounted at /api/threads/{thread_id}/uploads
app.include_router(uploads.router)
# Optional Chrome-extension page capture API is mounted at /api/browser-context
app.include_router(browser_context.router)
# Thread cleanup API is mounted at /api/threads/{thread_id}
app.include_router(threads.router)
# Agents API is mounted at /api/agents
app.include_router(agents.router)
@@ -0,0 +1,590 @@
"""Browser page context capture endpoints.
This router is intentionally isolated behind ``config.yaml``:
browser_context:
enabled: true
When disabled, mutating endpoints behave as if the integration is unavailable.
"""
from __future__ import annotations
import asyncio
import logging
import re
from datetime import UTC, datetime
from typing import Any, Literal
from urllib.parse import urlencode
from uuid import uuid4
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_config, get_thread_store
from app.gateway.routers.thread_runs import RunCreateRequest
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
from app.gateway.routers.uploads import _make_file_sandbox_writable
from app.gateway.services import start_run
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
from deerflow.config.app_config import AppConfig
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
from deerflow.uploads.manager import (
UnsafeUploadPathError,
ensure_uploads_dir,
normalize_filename,
open_upload_file_no_symlink,
upload_artifact_url,
upload_virtual_path,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/browser-context", tags=["browser-context"])
DEFAULT_BROWSER_CONTEXT_PROMPT = "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002"
class BrowserContextStatusResponse(BaseModel):
enabled: bool
max_content_chars: int
max_title_chars: int
max_url_chars: int
task_result_timeout_seconds: int
task_poll_interval_seconds: float
task_lease_seconds: int
tool_result_max_chars: int
class BrowserContextCaptureRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Existing or caller-generated zncm thread id")
title: str | None = Field(default=None, description="Current browser page title")
url: str | None = Field(default=None, description="Current browser page URL")
content: str = Field(description="Captured page content, preferably Markdown")
content_type: Literal["markdown", "text"] = Field(default="markdown")
prompt: str | None = Field(default=None, description="Optional first question to ask in zncm after capture")
class BrowserContextFileInfo(BaseModel):
filename: str
size: int
path: str
virtual_path: str
artifact_url: str
class BrowserContextCaptureResponse(BaseModel):
success: bool
thread_id: str
file: BrowserContextFileInfo
chat_path: str
chat_url: str
browser_context_token: str
prompt: str
truncated: bool = False
message: str
class BrowserContextAskRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Existing zncm thread id for follow-up questions")
title: str | None = Field(default=None, description="Current browser page title")
url: str | None = Field(default=None, description="Current browser page URL")
content: str = Field(description="Captured page content, preferably Markdown")
content_type: Literal["markdown", "text"] = Field(default="markdown")
question: str = Field(description="Question asked from the in-page assistant")
class BrowserContextAskResponse(BaseModel):
success: bool
thread_id: str
run_id: str
answer: str
truncated: bool = False
message: str
class BrowserContextTaskPayload(BaseModel):
id: str
kind: str = "fetch"
url: str
instruction: str = ""
session_id: str = ""
action: dict[str, Any] | None = None
created_at: str
updated_at: str
max_content_chars: int
class BrowserContextTaskNextResponse(BaseModel):
task: BrowserContextTaskPayload | None = None
class BrowserContextTaskResultRequest(BaseModel):
kind: str = Field(default="fetch", description="Task kind the extension processed: fetch or act")
title: str | None = Field(default=None, description="Captured page title")
url: str | None = Field(default=None, description="Final page URL after redirects")
content: str | None = Field(default=None, description="Captured page content")
content_type: Literal["markdown", "text"] = Field(default="markdown")
observation: dict[str, Any] | None = Field(default=None, description="Structured observation for an act step (elements, text, status)")
error: str | None = Field(default=None, description="Extension-side task error")
metadata: dict[str, Any] = Field(default_factory=dict)
class BrowserContextTaskResultResponse(BaseModel):
success: bool
task_id: str
status: str
truncated: bool = False
def _settings(config: AppConfig) -> BrowserContextConfig:
value = getattr(config, "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _require_enabled(config: AppConfig) -> BrowserContextConfig:
settings = _settings(config)
if not settings.enabled:
raise HTTPException(status_code=404, detail="Browser context integration is disabled")
return settings
def _trim(value: str | None, limit: int) -> str:
if not value:
return ""
return value.strip()[:limit]
def _limit_utf8_bytes(value: str, max_bytes: int) -> str:
text = value
while len(text.encode("utf-8")) > max_bytes:
text = text[:-1]
return text
def _filename_part(value: str, fallback: str, *, max_bytes: int) -> str:
text = value.strip() or fallback
# Keep unicode word chars so Chinese titles remain readable, while removing
# path separators and shell-hostile punctuation.
text = re.sub(r"[^\w.-]+", "-", text, flags=re.UNICODE).strip("-._")
text = _limit_utf8_bytes(text, max_bytes)
return text or fallback
def _build_markdown(
*,
title: str,
url: str,
content: str,
content_type: str,
captured_at: str,
truncated: bool,
) -> str:
lines = [
f"# {title or 'Captured Web Page'}",
"",
f"- Source URL: {url or '(unknown)'}",
f"- Captured at: {captured_at}",
]
if truncated:
lines.append("- Note: Content was truncated by the browser_context.max_content_chars limit.")
lines.extend(["", "---", ""])
if content_type == "markdown":
lines.append(content.strip())
else:
lines.append("```text")
lines.append(content.strip())
lines.append("```")
lines.append("")
return "\n".join(lines)
def _build_ask_prompt(
*,
title: str,
url: str,
content: str,
content_type: str,
captured_at: str,
truncated: bool,
) -> str:
note = "页面内容已按配置截断,请只基于可见内容回答。" if truncated else "页面内容如下。"
body = content.strip()
if content_type == "text":
body = f"```text\n{body}\n```"
return "\n\n".join(
[
"你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。",
"如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
f"页面标题:{title or '未知标题'}",
f"页面 URL:{url or '未知 URL'}",
f"读取时间:{captured_at}",
f"说明:{note}",
"当前网页内容:",
body,
]
)
def _message_type(message: Any) -> str:
if isinstance(message, dict):
return str(message.get("type") or message.get("role") or "").lower()
return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower()
def _content_to_text(content: Any) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for item in content:
if isinstance(item, str):
parts.append(item)
elif isinstance(item, dict):
if isinstance(item.get("text"), str):
parts.append(item["text"])
elif item.get("type") == "text" and isinstance(item.get("content"), str):
parts.append(item["content"])
return "\n".join(part.strip() for part in parts if part and part.strip()).strip()
return ""
def _latest_ai_answer(values: dict[str, Any]) -> str:
messages = values.get("messages")
if not isinstance(messages, list):
return ""
for message in reversed(messages):
if _message_type(message) not in {"ai", "assistant"}:
continue
answer = _content_to_text(message.get("content") if isinstance(message, dict) else getattr(message, "content", None))
if answer:
return answer
return ""
async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_path, data: bytes) -> None:
sandbox_provider = get_sandbox_provider()
if bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)):
return
sandbox_id = sandbox_provider.acquire(thread_id)
sandbox = sandbox_provider.get(sandbox_id)
if sandbox is None:
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
_make_file_sandbox_writable(file_path)
sandbox.update_file(virtual_path, data)
@router.get("/status", response_model=BrowserContextStatusResponse)
async def get_browser_context_status(
config: AppConfig = Depends(get_config),
) -> BrowserContextStatusResponse:
settings = _settings(config)
return BrowserContextStatusResponse(
enabled=settings.enabled,
max_content_chars=settings.max_content_chars,
max_title_chars=settings.max_title_chars,
max_url_chars=settings.max_url_chars,
task_result_timeout_seconds=settings.task_result_timeout_seconds,
task_poll_interval_seconds=settings.task_poll_interval_seconds,
task_lease_seconds=settings.task_lease_seconds,
tool_result_max_chars=settings.tool_result_max_chars,
)
@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse)
@require_permission("runs", "create")
async def claim_browser_context_task(
request: Request,
worker_id: str | None = None,
config: AppConfig = Depends(get_config),
) -> BrowserContextTaskNextResponse:
settings = _require_enabled(config)
prune_browser_tasks()
task = claim_next_browser_task(
user_id=get_effective_user_id(),
worker_id=worker_id or "chrome-extension",
lease_seconds=settings.task_lease_seconds,
)
if task is None:
return BrowserContextTaskNextResponse(task=None)
return BrowserContextTaskNextResponse(task=BrowserContextTaskPayload.model_validate(task))
@router.post("/tasks/{task_id}/result", response_model=BrowserContextTaskResultResponse)
@require_permission("runs", "create")
async def submit_browser_context_task_result(
task_id: str,
body: BrowserContextTaskResultRequest,
request: Request,
config: AppConfig = Depends(get_config),
) -> BrowserContextTaskResultResponse:
settings = _require_enabled(config)
error = _trim(body.error, 4000)
content = body.content or ""
is_act = (body.kind or "fetch") == "act"
if not error and not is_act and not content.strip():
raise HTTPException(status_code=400, detail="Captured page content is empty")
if not error and is_act and not isinstance(body.observation, dict):
raise HTTPException(status_code=400, detail="Act task result requires an observation")
task = complete_browser_task(
task_id=task_id,
user_id=get_effective_user_id(),
title=_trim(body.title, settings.max_title_chars),
url=_trim(body.url, settings.max_url_chars),
content=content,
content_type=body.content_type,
error=error,
metadata=body.metadata,
observation=body.observation,
)
if task is None:
raise HTTPException(status_code=404, detail=f"Browser task {task_id} not found")
result = task.get("result") or {}
return BrowserContextTaskResultResponse(
success=True,
task_id=task_id,
status=str(task.get("status") or "unknown"),
truncated=bool(result.get("truncated")),
)
@router.post("/capture", response_model=BrowserContextCaptureResponse)
@require_permission("threads", "write")
async def capture_browser_context(
body: BrowserContextCaptureRequest,
request: Request,
config: AppConfig = Depends(get_config),
) -> BrowserContextCaptureResponse:
settings = _require_enabled(config)
raw_content = (body.content or "").strip()
if not raw_content:
raise HTTPException(status_code=400, detail="Captured page content is empty")
title = _trim(body.title, settings.max_title_chars)
url = _trim(body.url, settings.max_url_chars)
truncated = len(raw_content) > settings.max_content_chars
content = raw_content[: settings.max_content_chars] if truncated else raw_content
requested_thread_id = (body.thread_id or "").strip() or None
if requested_thread_id:
thread_store = get_thread_store(request)
allowed = await thread_store.check_access(
requested_thread_id,
get_effective_user_id(),
require_existing=False,
)
if not allowed:
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
thread_response = await create_thread(
ThreadCreateRequest(
thread_id=requested_thread_id,
metadata={
"source": "browser_context",
"browser_context_url": url,
"browser_context_title": title,
},
),
request,
)
thread_id = thread_response.thread_id
captured_at = datetime.now(UTC).isoformat()
markdown = _build_markdown(
title=title,
url=url,
content=content,
content_type=body.content_type,
captured_at=captured_at,
truncated=truncated,
)
data = markdown.encode("utf-8")
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
prefix = _filename_part(settings.filename_prefix, "webpage", max_bytes=40)
title_part = _filename_part(title, "page", max_bytes=120)
filename = normalize_filename(f"{prefix}-{timestamp}-{title_part}.md")
try:
uploads_dir = ensure_uploads_dir(thread_id)
file_path, fh = open_upload_file_no_symlink(uploads_dir, filename)
with fh:
fh.write(data)
except UnsafeUploadPathError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
logger.exception("Failed to save browser context capture for thread %s", thread_id)
raise HTTPException(status_code=500, detail="Failed to save captured page") from exc
virtual_path = upload_virtual_path(filename)
await _sync_to_sandbox_if_needed(thread_id, virtual_path, file_path, data)
prompt = (body.prompt or "").strip() or DEFAULT_BROWSER_CONTEXT_PROMPT
token = uuid4().hex
route_base = settings.route_base.rstrip("/") or "/page/workspace"
chat_path = f"{route_base}/chats/{thread_id}"
query = urlencode(
{
"browser_context": token,
"bc_filename": filename,
"bc_size": str(len(data)),
"bc_path": virtual_path,
"bc_prompt": prompt,
}
)
chat_url = f"#{chat_path}?{query}"
return BrowserContextCaptureResponse(
success=True,
thread_id=thread_id,
file=BrowserContextFileInfo(
filename=filename,
size=len(data),
path=virtual_path,
virtual_path=virtual_path,
artifact_url=upload_artifact_url(thread_id, filename),
),
chat_path=chat_path,
chat_url=chat_url,
browser_context_token=token,
prompt=prompt,
truncated=truncated,
message="Captured page saved as a zncm upload",
)
@router.post("/ask", response_model=BrowserContextAskResponse)
@require_permission("runs", "create")
async def ask_browser_context(
body: BrowserContextAskRequest,
request: Request,
config: AppConfig = Depends(get_config),
) -> BrowserContextAskResponse:
settings = _require_enabled(config)
raw_content = (body.content or "").strip()
if not raw_content:
raise HTTPException(status_code=400, detail="Captured page content is empty")
question = (body.question or "").strip()
if not question:
raise HTTPException(status_code=400, detail="Question is empty")
title = _trim(body.title, settings.max_title_chars)
url = _trim(body.url, settings.max_url_chars)
content_limit = max(1_000, min(settings.max_content_chars, settings.tool_result_max_chars))
truncated = len(raw_content) > content_limit
content = raw_content[:content_limit] if truncated else raw_content
requested_thread_id = (body.thread_id or "").strip() or None
if requested_thread_id:
thread_store = get_thread_store(request)
allowed = await thread_store.check_access(
requested_thread_id,
get_effective_user_id(),
require_existing=False,
)
if not allowed:
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
thread_response = await create_thread(
ThreadCreateRequest(
thread_id=requested_thread_id,
assistant_id="lead_agent",
metadata={
"source": "browser_context_page_assistant",
"browser_context_url": url,
"browser_context_title": title,
},
),
request,
)
thread_id = thread_response.thread_id
captured_at = datetime.now(UTC).isoformat()
is_follow_up = requested_thread_id is not None
if is_follow_up:
prompt_prefix = "\n\n".join(
[
"你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。",
"如果本会话里的网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
f"当前页面标题:{title or '未知标题'}",
f"当前页面 URL:{url or '未知 URL'}",
f"本轮提问时间:{captured_at}",
]
)
else:
prompt_prefix = _build_ask_prompt(
title=title,
url=url,
content=content,
content_type=body.content_type,
captured_at=captured_at,
truncated=truncated,
)
run_body = RunCreateRequest(
assistant_id="lead_agent",
input={
"messages": [
{
"type": "human",
"content": question,
"additional_kwargs": {
"prompt_prefix": prompt_prefix,
},
}
]
},
metadata={
"source": "browser_context_page_assistant",
"browser_context_url": url,
"browser_context_title": title,
"browser_context_truncated": truncated,
},
stream_mode=["values"],
on_disconnect="continue",
multitask_strategy="reject",
)
record = await start_run(run_body, thread_id, request)
if record.task is not None:
try:
await record.task
except asyncio.CancelledError:
pass
checkpointer = get_checkpointer(request)
values: dict[str, Any] = {}
try:
checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
if checkpoint_tuple is not None:
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
except Exception:
logger.exception("Failed to fetch browser context answer for run %s", record.run_id)
answer = _latest_ai_answer(values)
if not answer:
answer = record.error or "zncm did not return an answer."
return BrowserContextAskResponse(
success=bool(answer),
thread_id=thread_id,
run_id=record.run_id,
answer=answer,
truncated=truncated,
message="zncm page assistant answer completed",
)
+110 -81
View File
@@ -10,90 +10,90 @@ gateway:
port: 8001
models:
- name: deepseek-chat
display_name: DeepSeek Chat
use: langchain_openai:ChatOpenAI
model: deepseek-chat
api_key: sk-1520028668cb4cc388ed8315ff1e856e
base_url: https://api.deepseek.com
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: false
supports_thinking: true
# - name: deepseek-chat
# display_name: DeepSeek Chat
# use: langchain_openai:ChatOpenAI
# model: deepseek-chat
# api_key: sk-1520028668cb4cc388ed8315ff1e856e
# base_url: https://api.deepseek.com
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: false
# supports_thinking: true
- name: deepseek-v4-flash
display_name: DeepSeek V4 Flash
use: langchain_openai:ChatOpenAI
model: deepseek-v4-flash
api_key: sk-e1c22b7931dc446f8b67e57878edd105
base_url: https://api.deepseek.com
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: false
supports_thinking: true
# - name: deepseek-v4-flash
# display_name: DeepSeek V4 Flash
# use: langchain_openai:ChatOpenAI
# model: deepseek-v4-flash
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
# base_url: https://api.deepseek.com
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: false
# supports_thinking: true
- name: deepseek-v4-pro
display_name: DeepSeek V4 Pro
# - name: deepseek-v4-pro
# display_name: DeepSeek V4 Pro
# use: langchain_openai:ChatOpenAI
# model: deepseek-v4-pro
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
# base_url: https://api.deepseek.com
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: false
# supports_thinking: true
# - name: MiniMax-M2.7
# display_name: MiniMax-M2.7
# use: langchain_openai:ChatOpenAI
# model: MiniMax-M2.7
# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
# base_url: https://api.minimaxi.com/v1
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
# - name: glm-5.1
# display_name: glm-5.1
# use: langchain_openai:ChatOpenAI
# model: glm-5.1
# api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR
# base_url: https://open.bigmodel.cn/api/paas/v4/
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
# - name: deepseek-ai/DeepSeek-OCR
# display_name: deepseek-ai/DeepSeek-OCR
# use: langchain_openai:ChatOpenAI
# model: deepseek-ai/DeepSeek-OCR
# api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
# base_url: https://api.siliconflow.cn/v1/
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
# - name: zai-org/GLM-5-FP8
# display_name: zai-org/GLM-5-FP8
# use: langchain_openai:ChatOpenAI
# model: zai-org/GLM-5-FP8
# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
# base_url: https://api.us-west-2.modal.direct/v1
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
- name: deepseek-ai/DeepSeek-V4-Flash
display_name: deepseek-ai/DeepSeek-V4-Flash
use: langchain_openai:ChatOpenAI
model: deepseek-v4-pro
api_key: sk-e1c22b7931dc446f8b67e57878edd105
base_url: https://api.deepseek.com
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: false
supports_thinking: true
- name: MiniMax-M2.7
display_name: MiniMax-M2.7
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.7
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com/v1
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: glm-5.1
display_name: glm-5.1
use: langchain_openai:ChatOpenAI
model: glm-5.1
api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR
base_url: https://open.bigmodel.cn/api/paas/v4/
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: deepseek-ai/DeepSeek-OCR
display_name: deepseek-ai/DeepSeek-OCR
use: langchain_openai:ChatOpenAI
model: deepseek-ai/DeepSeek-OCR
api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
base_url: https://api.siliconflow.cn/v1/
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: zai-org/GLM-5-FP8
display_name: zai-org/GLM-5-FP8
use: langchain_openai:ChatOpenAI
model: zai-org/GLM-5-FP8
api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
base_url: https://api.us-west-2.modal.direct/v1
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: Qwen/Qwen3.6-27B
display_name: Qwen/Qwen3.6-27B
use: langchain_openai:ChatOpenAI
model: Qwen/Qwen3.6-27B
model: deepseek-ai/DeepSeek-V4-Flash
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
base_url: https://api.siliconflow.cn/v1/
max_tokens: 4096
@@ -122,6 +122,26 @@ uploads:
auto_convert_documents: true
pdf_converter: auto
# Chrome 插件「当前网页发送到 Deer」入口。默认关闭,打开后才启用
# /api/browser-context/capture。它只保存当前页 Markdown 到线程 uploads,
# 不会改变普通聊天、上传或工具调用流程。
browser_context:
enabled: true
max_content_chars: 200000
max_title_chars: 300
max_url_chars: 2048
filename_prefix: webpage
route_base: /page/workspace
task_result_timeout_seconds: 60
task_poll_interval_seconds: 1
task_lease_seconds: 120
tool_result_max_chars: 60000
# 交互式页面操作(browser_act:点击/输入/滚动/等待/跳转)。关闭后只保留只读抓取。
actions_enabled: true
action_result_timeout_seconds: 45
max_observed_elements: 150
observation_text_max_chars: 12000
title:
enabled: true
max_words: 6
@@ -139,6 +159,7 @@ title:
tool_groups:
- name: web
- name: browser
- name: file:read
- name: file:write
- name: bash
@@ -154,6 +175,14 @@ tools:
use: deerflow.community.jina_ai.tools:web_fetch_tool
timeout: 15
- name: browser_fetch_page
group: browser
use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
- name: browser_act
group: browser
use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
- name: ls
group: file:read
use: deerflow.sandbox.tools:ls_tool
@@ -0,0 +1,2 @@
"""Browser-backed page context helpers."""
@@ -0,0 +1,303 @@
"""Small file-backed task queue for browser page fetch requests."""
from __future__ import annotations
import asyncio
import json
import logging
import re
import threading
import time
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import uuid4
from deerflow.config.runtime_paths import runtime_home
logger = logging.getLogger(__name__)
_LOCK = threading.RLock()
_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$")
_TERMINAL_STATUSES = {"done", "error", "expired"}
def tasks_dir() -> Path:
"""Return the browser-context task directory, creating it if needed."""
path = runtime_home() / "browser-context" / "tasks"
path.mkdir(parents=True, exist_ok=True)
return path
def _now() -> datetime:
return datetime.now(UTC)
def _iso(value: datetime | None = None) -> str:
return (value or _now()).isoformat()
def _parse_iso(value: Any) -> datetime | None:
if not isinstance(value, str) or not value:
return None
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def _task_path(task_id: str) -> Path | None:
if not _TASK_ID_RE.fullmatch(task_id or ""):
return None
return tasks_dir() / f"{task_id}.json"
def _read_task(path: Path) -> dict[str, Any] | None:
try:
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
except FileNotFoundError:
return None
except Exception:
logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True)
return None
return data if isinstance(data, dict) else None
def _write_task(path: Path, data: dict[str, Any]) -> None:
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
with tmp.open("w", encoding="utf-8") as fh:
json.dump(data, fh, ensure_ascii=False, separators=(",", ":"))
tmp.replace(path)
def _public_task(data: dict[str, Any]) -> dict[str, Any]:
return {
"id": data.get("id"),
"kind": data.get("kind") or "fetch",
"url": data.get("url") or "",
"instruction": data.get("instruction") or "",
"session_id": data.get("session_id") or "",
"action": data.get("action") or None,
"created_at": data.get("created_at") or "",
"updated_at": data.get("updated_at") or "",
"max_content_chars": int(data.get("max_content_chars") or 200_000),
}
def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]:
if data.get("status") in _TERMINAL_STATUSES:
return data
expires_at = _parse_iso(data.get("expires_at"))
if expires_at is not None and now >= expires_at:
data = dict(data)
data["status"] = "expired"
data["error"] = "Browser task expired before the extension returned a result."
data["updated_at"] = _iso(now)
_write_task(path, data)
return data
def prune_browser_tasks(max_age_seconds: int = 86_400) -> int:
"""Delete old terminal tasks and return the number removed."""
cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds)))
removed = 0
with _LOCK:
for path in tasks_dir().glob("*.json"):
data = _read_task(path)
if not data or data.get("status") not in _TERMINAL_STATUSES:
continue
updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at"))
if updated_at is None or updated_at < cutoff:
try:
path.unlink()
removed += 1
except OSError:
logger.debug("Failed to delete old browser task: %s", path, exc_info=True)
return removed
def create_browser_task(
*,
user_id: str,
url: str,
instruction: str = "",
timeout_seconds: int = 60,
max_content_chars: int = 200_000,
metadata: dict[str, Any] | None = None,
kind: str = "fetch",
session_id: str = "",
action: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Create a browser task for the current user's extension to pick up.
``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or
``"act"`` for an interactive observe/act step driven by ``action`` within a
long-lived ``session_id`` (a tab kept open by the extension).
"""
now = _now()
task_id = uuid4().hex
task = {
"id": task_id,
"user_id": str(user_id or "default"),
"status": "pending",
"kind": kind if kind in {"fetch", "act"} else "fetch",
"url": url,
"instruction": instruction or "",
"session_id": str(session_id or ""),
"action": action or None,
"created_at": _iso(now),
"updated_at": _iso(now),
"expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))),
"max_content_chars": max(1_000, int(max_content_chars)),
"worker_id": None,
"claimed_at": None,
"result": None,
"error": None,
"metadata": metadata or {},
}
path = tasks_dir() / f"{task_id}.json"
with _LOCK:
_write_task(path, task)
return task
def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None:
"""Read a task if it belongs to the requested user."""
path = _task_path(task_id)
if path is None:
return None
with _LOCK:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
return None
return _expire_if_needed(path, data, _now())
def claim_next_browser_task(
*,
user_id: str,
worker_id: str,
lease_seconds: int = 120,
) -> dict[str, Any] | None:
"""Claim the oldest pending task for a user."""
now = _now()
user_id = str(user_id or "default")
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
lease_seconds = max(10, int(lease_seconds))
with _LOCK:
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
for path in paths:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != user_id:
continue
data = _expire_if_needed(path, data, now)
status = data.get("status")
if status == "running":
updated_at = _parse_iso(data.get("updated_at")) or now
if (now - updated_at).total_seconds() <= lease_seconds:
continue
status = "pending"
data = dict(data)
data["status"] = status
if status != "pending":
continue
data = dict(data)
data["status"] = "running"
data["worker_id"] = worker_id
data["claimed_at"] = _iso(now)
data["updated_at"] = _iso(now)
_write_task(path, data)
return _public_task(data)
return None
def complete_browser_task(
*,
task_id: str,
user_id: str,
title: str = "",
url: str = "",
content: str = "",
content_type: str = "markdown",
error: str = "",
metadata: dict[str, Any] | None = None,
observation: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Mark a browser task complete or failed.
For ``act`` tasks the extension returns a structured ``observation`` (final
URL/title, interactive element list, page text) instead of a captured
markdown document; it is stored alongside the result verbatim.
"""
path = _task_path(task_id)
if path is None:
return None
now = _now()
with _LOCK:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
return None
data = dict(data)
if error:
data["status"] = "error"
data["error"] = str(error).strip()[:4000]
data["result"] = None
else:
raw_content = str(content or "").strip()
max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000))
truncated = len(raw_content) > max_content_chars
data["status"] = "done"
data["error"] = None
data["result"] = {
"title": str(title or "").strip()[:500],
"url": str(url or data.get("url") or "").strip()[:4096],
"content": raw_content[:max_content_chars] if truncated else raw_content,
"content_type": content_type if content_type in {"markdown", "text"} else "markdown",
"truncated": truncated,
"observation": observation if isinstance(observation, dict) else None,
"captured_at": _iso(now),
"metadata": metadata or {},
}
data["updated_at"] = _iso(now)
_write_task(path, data)
return data
async def wait_for_browser_task_result(
*,
task_id: str,
user_id: str,
timeout_seconds: int,
poll_interval_seconds: float = 1.0,
) -> dict[str, Any] | None:
"""Wait for a browser task to reach a terminal state."""
deadline = time.monotonic() + max(1, int(timeout_seconds))
interval = max(0.25, float(poll_interval_seconds))
while True:
task = get_browser_task(task_id=task_id, user_id=user_id)
if task is None:
return None
if task.get("status") in _TERMINAL_STATUSES:
return task
if time.monotonic() >= deadline:
return task
await asyncio.sleep(interval)
@@ -9,8 +9,9 @@ from dotenv import load_dotenv
from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
@@ -132,7 +133,8 @@ class AppConfig(BaseModel):
gateway: GatewayConfig = Field(default_factory=GatewayConfig, description="Gateway FastAPI listening port (used by intra-process loopback)")
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
@classmethod
def resolve_config_path(cls, config_path: str | None = None) -> Path:
@@ -0,0 +1,74 @@
"""Configuration for browser page context capture."""
from pydantic import BaseModel, Field
class BrowserContextConfig(BaseModel):
"""Controls the optional Chrome-extension page capture integration."""
enabled: bool = Field(
default=False,
description="Enable /api/browser-context endpoints for Chrome extension page capture.",
)
max_content_chars: int = Field(
default=200_000,
ge=1_000,
description="Maximum captured page text stored per request. Longer content is truncated.",
)
max_title_chars: int = Field(
default=300,
ge=20,
description="Maximum title length accepted from the browser extension.",
)
max_url_chars: int = Field(
default=2_048,
ge=100,
description="Maximum source URL length accepted from the browser extension.",
)
filename_prefix: str = Field(
default="webpage",
description="Prefix used for Markdown files created from captured pages.",
)
route_base: str = Field(
default="/page/workspace",
description="Frontend route base used when returning chat paths to the extension.",
)
task_result_timeout_seconds: int = Field(
default=60,
ge=5,
description="Maximum time the browser_fetch_page tool waits for the extension result.",
)
task_poll_interval_seconds: float = Field(
default=1.0,
ge=0.25,
description="How often the backend tool checks for a returned browser task result.",
)
task_lease_seconds: int = Field(
default=120,
ge=10,
description="How long a claimed browser task is reserved before another extension may retry it.",
)
tool_result_max_chars: int = Field(
default=60_000,
ge=1_000,
description="Maximum page characters returned from browser_fetch_page into the model context.",
)
actions_enabled: bool = Field(
default=True,
description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).",
)
action_result_timeout_seconds: int = Field(
default=45,
ge=5,
description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.",
)
max_observed_elements: int = Field(
default=150,
ge=10,
description="Maximum number of interactive elements returned in a browser_act observation.",
)
observation_text_max_chars: int = Field(
default=12_000,
ge=500,
description="Maximum page text characters returned alongside a browser_act observation.",
)
@@ -0,0 +1,225 @@
"""Interactive browser action tool backed by the user's Chrome extension.
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
*observe -> act* loop on a live page the extension keeps open:
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
2. The extension performs the action on the session tab, then re-scans the page
and returns an *observation*: the final URL/title, an indexed list of
interactive elements, and the visible page text.
3. The model reads the observation, picks the next element by its index, and
calls ``browser_act`` again with the same ``session_id``.
The element ``index`` values are only valid for the most recent observation of a
session; always act on indices from the latest result.
"""
from __future__ import annotations
from typing import Any
from urllib.parse import urlparse
from uuid import uuid4
from langchain.tools import tool
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
from deerflow.config import get_app_config
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime.user_context import get_effective_user_id
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
_NEEDS_INDEX = {"click", "type"}
def _settings() -> BrowserContextConfig:
value = getattr(get_app_config(), "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _validate_url(raw_url: str) -> str | None:
url = (raw_url or "").strip()
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
return url
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
if not elements:
return "(no interactive elements detected)"
lines: list[str] = []
for el in elements:
idx = el.get("index")
tag = str(el.get("tag") or "?")
el_type = str(el.get("type") or "").strip()
label = str(el.get("label") or "").strip().replace("\n", " ")
if len(label) > 120:
label = label[:120] + "…"
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
extra = ""
if el.get("value"):
value = str(el["value"]).strip().replace("\n", " ")
extra = f' value="{value[:60]}"'
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
return "\n".join(lines)
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
observation = observation or {}
status = str(observation.get("action_status") or "ok")
message = str(observation.get("message") or "").strip()
url = str(observation.get("url") or "").strip() or "(unknown)"
title = str(observation.get("title") or "").strip() or "(untitled)"
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
text = str(observation.get("text") or "").strip()
text_limit = max(500, int(settings.observation_text_max_chars))
text_truncated = len(text) > text_limit
if text_truncated:
text = text[:text_limit]
parts = [
f"Browser session: {session_id}",
f"Last action: {action} -> {status}",
]
if message:
parts.append(f"Note: {message}")
parts.extend(
[
f"URL: {url}",
f"Title: {title}",
"",
"Interactive elements (use the [index] with click/type):",
_format_elements(elements),
"",
"Page text:",
text or "(empty page text)",
]
)
if text_truncated:
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
parts.append(
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
"When the task is finished, call browser_act with action=\"close\" to release the tab."
)
return "\n".join(parts)
@tool("browser_act", parse_docstring=True)
async def browser_act_tool(
action: str,
session_id: str = "",
url: str = "",
index: int = -1,
text: str = "",
submit: bool = False,
wait_for: str = "",
wait_seconds: int = 0,
) -> str:
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
Use this to operate a real web page on the user's behalf — log in, fill forms,
click through navigation, and read pages that need the user's session or local
network. Start every task with ``action="navigate"`` and a URL to open a tab
and get the first observation, then keep calling this tool with the same
``session_id``, choosing elements by the ``index`` shown in the observation.
Call ``action="close"`` when done to free the tab.
Args:
action: One of navigate, click, type, scroll, wait, extract, observe, close.
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
url: Target http(s) URL. Required for navigate (and to start a new session).
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
text: Text to enter for the type action.
submit: For type, press Enter / submit the form after typing.
wait_for: For wait, a CSS selector to wait for before observing.
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
"""
settings = _settings()
if not settings.enabled:
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
if not settings.actions_enabled:
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
action = (action or "").strip().lower()
if action not in _ACTIONS:
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
session_id = (session_id or "").strip()
safe_url = _validate_url(url) if url else None
if action == "navigate" and safe_url is None:
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
if not session_id:
if action != "navigate":
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
session_id = uuid4().hex
if action in _NEEDS_INDEX and index < 0:
return f"browser_act {action} requires a non-negative element index from the latest observation."
if action == "type" and not text:
return "browser_act type requires the text to enter."
action_payload: dict[str, Any] = {"type": action}
if safe_url is not None:
action_payload["url"] = safe_url
if index >= 0:
action_payload["index"] = int(index)
if text:
action_payload["text"] = text
if submit:
action_payload["submit"] = True
if wait_for.strip():
action_payload["wait_for"] = wait_for.strip()[:300]
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
requested_timeout = int(wait_seconds or configured_timeout)
timeout = min(max(5, requested_timeout), configured_timeout)
user_id = get_effective_user_id()
task = create_browser_task(
user_id=user_id,
url=safe_url or "",
instruction="",
kind="act",
session_id=session_id,
action=action_payload,
timeout_seconds=timeout + int(settings.task_lease_seconds),
max_content_chars=settings.max_content_chars,
metadata={"source": "browser_act_tool"},
)
completed = await wait_for_browser_task_result(
task_id=task["id"],
user_id=user_id,
timeout_seconds=timeout,
poll_interval_seconds=settings.task_poll_interval_seconds,
)
if completed is None:
return f"Browser task {task['id']} was not found after it was created."
status = completed.get("status")
if status == "error":
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
if status == "expired":
return (
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
)
if status != "done":
return (
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
)
if action == "close":
return f"browser_act session {session_id}: tab closed."
result = completed.get("result") or {}
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
return _format_observation(action, observation, session_id, settings)
@@ -0,0 +1,117 @@
"""Browser extension backed page fetch tool."""
from __future__ import annotations
from urllib.parse import urlparse
from langchain.tools import tool
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
from deerflow.config import get_app_config
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime.user_context import get_effective_user_id
def _settings() -> BrowserContextConfig:
value = getattr(get_app_config(), "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _validate_url(raw_url: str) -> str | None:
url = (raw_url or "").strip()
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
return url
@tool("browser_fetch_page", parse_docstring=True)
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
Use this when the user explicitly asks to use the browser extension or asks
for data from a page that may require their browser session, login state, or
local network access. The URL must be an exact http(s) URL provided by the
user. If the user did not provide a URL, ask for the URL instead of guessing.
Args:
url: Exact http(s) URL to open in the user's Chrome browser.
question: Optional instruction describing what data the user wants from the page.
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
"""
settings = _settings()
if not settings.enabled:
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
safe_url = _validate_url(url)
if safe_url is None:
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
requested_timeout = int(wait_seconds or configured_timeout)
timeout = min(max(5, requested_timeout), configured_timeout)
user_id = get_effective_user_id()
task = create_browser_task(
user_id=user_id,
url=safe_url,
instruction=(question or "").strip(),
timeout_seconds=timeout + int(settings.task_lease_seconds),
max_content_chars=settings.max_content_chars,
metadata={"source": "browser_fetch_page_tool"},
)
completed = await wait_for_browser_task_result(
task_id=task["id"],
user_id=user_id,
timeout_seconds=timeout,
poll_interval_seconds=settings.task_poll_interval_seconds,
)
if completed is None:
return f"Browser task {task['id']} was not found after it was created."
status = completed.get("status")
if status == "error":
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
if status == "expired":
return (
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
)
if status != "done":
return (
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
)
result = completed.get("result") or {}
content = str(result.get("content") or "").strip()
tool_limit = max(1_000, int(settings.tool_result_max_chars))
tool_truncated = len(content) > tool_limit
if tool_truncated:
content = content[:tool_limit]
notes: list[str] = []
if result.get("truncated"):
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
if tool_truncated:
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
parts = [
"Browser page fetched through the user's Chrome extension.",
f"Task ID: {task['id']}",
f"Title: {result.get('title') or '(untitled)'}",
f"URL: {result.get('url') or safe_url}",
]
if question:
parts.extend(["", "Requested extraction:", question.strip()])
if notes:
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
parts.extend(["", "Page content:", content or "(empty page text)"])
return "\n".join(parts)
@@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry
logger = logging.getLogger(__name__)
BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool"
BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool"
BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE}
BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"}
BUILTIN_TOOLS = [
present_file_tool,
ask_clarification_tool,
@@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool:
return False
def _is_browser_context_tool(tool: object) -> bool:
"""Return True if the tool config represents the browser extension bridge."""
use = getattr(tool, "use", None)
name = getattr(tool, "name", None)
return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
def get_available_tools(
groups: list[str] | None = None,
include_mcp: bool = True,
@@ -64,6 +77,18 @@ def get_available_tools(
config = app_config or get_app_config()
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
browser_context_config = getattr(config, "browser_context", None)
if not getattr(browser_context_config, "enabled", False):
tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
elif not getattr(browser_context_config, "actions_enabled", True):
# browser_context is on, but interactive page actions are disabled — keep
# the read-only browser_fetch_page tool, drop only browser_act.
tool_configs = [
tool
for tool in tool_configs
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
]
# Do not expose host bash by default when LocalSandboxProvider is active.
if not is_host_bash_allowed(config):
tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)]
@@ -0,0 +1,158 @@
"""Tests for the interactive browser_act tool and its act-task queue support."""
from __future__ import annotations
import asyncio
import pytest
from deerflow.browser_context import tasks as task_queue
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.tools.builtins import browser_act_tool as act_mod
@pytest.fixture(autouse=True)
def _isolated_task_home(tmp_path, monkeypatch):
"""Point the file-backed task queue at a throwaway directory."""
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
yield
def _enable_actions(monkeypatch, *, enabled=True, actions_enabled=True):
cfg = BrowserContextConfig(
enabled=enabled,
actions_enabled=actions_enabled,
action_result_timeout_seconds=5,
task_poll_interval_seconds=0.25,
task_lease_seconds=30,
)
monkeypatch.setattr(act_mod, "_settings", lambda: cfg)
return cfg
# --- task queue layer -------------------------------------------------------
def test_act_task_round_trip_carries_session_and_observation():
task = task_queue.create_browser_task(
user_id="u1",
url="https://example.com",
kind="act",
session_id="sess-1",
action={"type": "navigate", "url": "https://example.com"},
)
assert task["kind"] == "act"
assert task["session_id"] == "sess-1"
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1")
assert claimed is not None
assert claimed["kind"] == "act"
assert claimed["session_id"] == "sess-1"
assert claimed["action"] == {"type": "navigate", "url": "https://example.com"}
observation = {
"action_status": "ok",
"url": "https://example.com/home",
"title": "Home",
"elements": [{"index": 0, "tag": "button", "label": "Login"}],
"text": "welcome",
}
done = task_queue.complete_browser_task(
task_id=task["id"],
user_id="u1",
observation=observation,
)
assert done is not None
assert done["status"] == "done"
assert done["result"]["observation"] == observation
def test_fetch_task_default_kind_is_fetch():
task = task_queue.create_browser_task(user_id="u1", url="https://example.com")
assert task["kind"] == "fetch"
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1")
assert claimed["kind"] == "fetch"
assert claimed["session_id"] == ""
assert claimed["action"] is None
# --- tool validation --------------------------------------------------------
async def _invoke(**kwargs) -> str:
return await act_mod.browser_act_tool.ainvoke(kwargs)
@pytest.mark.asyncio
async def test_browser_act_disabled_when_actions_off(monkeypatch):
_enable_actions(monkeypatch, enabled=True, actions_enabled=False)
out = await _invoke(action="navigate", url="https://example.com")
assert "actions_enabled is false" in out
@pytest.mark.asyncio
async def test_browser_act_unknown_action(monkeypatch):
_enable_actions(monkeypatch)
out = await _invoke(action="frobnicate")
assert "unknown action" in out
@pytest.mark.asyncio
async def test_browser_act_navigate_requires_url(monkeypatch):
_enable_actions(monkeypatch)
out = await _invoke(action="navigate")
assert "requires an exact http(s) URL" in out
@pytest.mark.asyncio
async def test_browser_act_non_navigate_needs_session(monkeypatch):
_enable_actions(monkeypatch)
out = await _invoke(action="click", index=0)
assert "no session_id yet" in out
@pytest.mark.asyncio
async def test_browser_act_click_requires_index(monkeypatch):
_enable_actions(monkeypatch)
out = await _invoke(action="click", session_id="sess-1")
assert "requires a non-negative element index" in out
# --- tool success path (simulated extension) --------------------------------
@pytest.mark.asyncio
async def test_browser_act_returns_formatted_observation(monkeypatch):
_enable_actions(monkeypatch)
async def fake_extension():
# Wait for the tool to create the pending act task, then complete it.
for _ in range(40):
claimed = task_queue.claim_next_browser_task(user_id="default", worker_id="ext")
if claimed is not None:
task_queue.complete_browser_task(
task_id=claimed["id"],
user_id="default",
observation={
"action_status": "ok",
"url": "https://example.com/home",
"title": "Home Page",
"elements": [
{"index": 0, "tag": "input", "type": "search", "label": "Search", "value": ""},
{"index": 1, "tag": "button", "label": "Go"},
],
"text": "Welcome to the home page.",
},
)
return
await asyncio.sleep(0.1)
ext = asyncio.create_task(fake_extension())
out = await _invoke(action="navigate", url="https://example.com")
await ext
assert "Browser session:" in out
assert "Home Page" in out
assert "[0] input[search]" in out
assert "[1] button" in out
assert "Welcome to the home page." in out
+69 -69
View File
@@ -1,76 +1,76 @@
config_version: 8
log_level: info
models:
- name: deepseek-chat
display_name: DeepSeek Chat
# - name: deepseek-chat
# display_name: DeepSeek Chat
# use: langchain_openai:ChatOpenAI
# model: deepseek-chat
# api_key: sk-9808fb2375534112ac275541aec14b9b
# base_url: https://api.deepseek.com
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: false
# supports_thinking: true
# - name: zai-org/GLM-5-FP8
# display_name: zai-org/GLM-5-FP8
# use: langchain_openai:ChatOpenAI
# model: zai-org/GLM-5-FP8
# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
# base_url: https://api.us-west-2.modal.direct/v1
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
# - name: deepseek-v4-pro
# display_name: DeepSeek V4 Pro
# use: langchain_openai:ChatOpenAI
# model: deepseek-v4-pro
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
# base_url: https://api.deepseek.com
# request_timeout: 600.0
# max_retries: 2
# max_tokens: 4096
# temperature: 1.0
# supports_vision: false
# supports_thinking: true
# - name: minimax-m2.7
# display_name: MiniMax M2.7
# use: langchain_openai:ChatOpenAI
# model: MiniMax-M2.7
# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
# base_url: https://api.minimaxi.com
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
# - name: minimax-m2.5
# display_name: MiniMax M2.5
# use: langchain_openai:ChatOpenAI
# model: MiniMax-M2.5
# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
# base_url: https://api.minimaxi.com
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
# - name: deepseek-ai/DeepSeek-OCR
# display_name: deepseek-ai/DeepSeek-OCR
# use: langchain_openai:ChatOpenAI
# model: deepseek-ai/DeepSeek-OCR
# api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
# base_url: https://api.siliconflow.cn/v1/
# max_tokens: 4096
# temperature: 1.0
# supports_vision: true
# supports_thinking: true
- name: deepseek-ai/DeepSeek-V4-Flash
display_name: deepseek-ai/DeepSeek-V4-Flash
use: langchain_openai:ChatOpenAI
model: deepseek-chat
api_key: sk-9808fb2375534112ac275541aec14b9b
base_url: https://api.deepseek.com
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: false
supports_thinking: true
- name: zai-org/GLM-5-FP8
display_name: zai-org/GLM-5-FP8
use: langchain_openai:ChatOpenAI
model: zai-org/GLM-5-FP8
api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
base_url: https://api.us-west-2.modal.direct/v1
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: deepseek-v4-pro
display_name: DeepSeek V4 Pro
use: langchain_openai:ChatOpenAI
model: deepseek-v4-pro
api_key: sk-e1c22b7931dc446f8b67e57878edd105
base_url: https://api.deepseek.com
request_timeout: 600.0
max_retries: 2
max_tokens: 4096
temperature: 1.0
supports_vision: false
supports_thinking: true
- name: minimax-m2.7
display_name: MiniMax M2.7
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.7
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: minimax-m2.5
display_name: MiniMax M2.5
use: langchain_openai:ChatOpenAI
model: MiniMax-M2.5
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
base_url: https://api.minimaxi.com
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: deepseek-ai/DeepSeek-OCR
display_name: deepseek-ai/DeepSeek-OCR
use: langchain_openai:ChatOpenAI
model: deepseek-ai/DeepSeek-OCR
api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
base_url: https://api.siliconflow.cn/v1/
max_tokens: 4096
temperature: 1.0
supports_vision: true
supports_thinking: true
- name: Qwen/Qwen3.6-27B
display_name: Qwen/Qwen3.6-27B
use: langchain_openai:ChatOpenAI
model: Qwen/Qwen3.6-27B
model: deepseek-ai/DeepSeek-V4-Flash
api_key: sk-qwwwvjompnzxqhndctlwmmomvqdgaqqqgxvibfxchsikvgod
base_url: https://api.siliconflow.cn/v1/
max_tokens: 4096