feat: 浏览器扩展支持交互式页面操作 (browser_act)
在原有只读抓取 (browser_fetch_page) 基础上,新增 observe→act 循环, 让 Agent 能在用户的 Chrome 中实际操作网页:点击/输入/滚动/等待/页内跳转。 后端: - 新工具 browser_act,按 session 复用标签页,返回带编号的可交互元素清单+正文 - browser_context 任务队列扩展 kind/session_id/action 与 observation 结果 - /api/browser-context 路由下发 act 任务、接收 observation - BrowserContextConfig 增加 actions_enabled 等开关,tools.py 两级门控 - 接线 app.py 路由挂载、app_config.py 配置字段、config.yaml 工具注册 前端: - ChatPage 处理「发送当前页」回跳的 browser_context query 参数并自动提交 扩展 (browser-extension/deer-web-clipper): - shared.js 自包含注入函数 deerObservePage / deerPerformAction - background.js 按 kind 分发任务,session→tab 映射存 chrome.storage.session 测试:tests/test_browser_act.py(队列往返 + 工具校验 + 模拟扩展成功路径,8 passed) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0001f86826
commit
d4a71ce384
24 files changed
+4499
-162
No files matched your search
@@ -29,9 +29,10 @@ from app.gateway.routers import (
|
||||
ai_writing,
|
||||
admin_users,
|
||||
agents,
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
browser_context,
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
feedback,
|
||||
@@ -1197,11 +1198,14 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
|
||||
app.include_router(artifacts.router)
|
||||
|
||||
# Uploads API is mounted at /api/threads/{thread_id}/uploads
|
||||
app.include_router(uploads.router)
|
||||
|
||||
# Thread cleanup API is mounted at /api/threads/{thread_id}
|
||||
app.include_router(threads.router)
|
||||
# Uploads API is mounted at /api/threads/{thread_id}/uploads
|
||||
app.include_router(uploads.router)
|
||||
|
||||
# Optional Chrome-extension page capture API is mounted at /api/browser-context
|
||||
app.include_router(browser_context.router)
|
||||
|
||||
# Thread cleanup API is mounted at /api/threads/{thread_id}
|
||||
app.include_router(threads.router)
|
||||
|
||||
# Agents API is mounted at /api/agents
|
||||
app.include_router(agents.router)
|
||||
|
||||
@@ -0,0 +1,590 @@
|
||||
"""Browser page context capture endpoints.
|
||||
|
||||
This router is intentionally isolated behind ``config.yaml``:
|
||||
|
||||
browser_context:
|
||||
enabled: true
|
||||
|
||||
When disabled, mutating endpoints behave as if the integration is unavailable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import urlencode
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer, get_config, get_thread_store
|
||||
from app.gateway.routers.thread_runs import RunCreateRequest
|
||||
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
|
||||
from app.gateway.routers.uploads import _make_file_sandbox_writable
|
||||
from app.gateway.services import start_run
|
||||
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
|
||||
from deerflow.uploads.manager import (
|
||||
UnsafeUploadPathError,
|
||||
ensure_uploads_dir,
|
||||
normalize_filename,
|
||||
open_upload_file_no_symlink,
|
||||
upload_artifact_url,
|
||||
upload_virtual_path,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/browser-context", tags=["browser-context"])
|
||||
DEFAULT_BROWSER_CONTEXT_PROMPT = "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002"
|
||||
|
||||
|
||||
class BrowserContextStatusResponse(BaseModel):
|
||||
enabled: bool
|
||||
max_content_chars: int
|
||||
max_title_chars: int
|
||||
max_url_chars: int
|
||||
task_result_timeout_seconds: int
|
||||
task_poll_interval_seconds: float
|
||||
task_lease_seconds: int
|
||||
tool_result_max_chars: int
|
||||
|
||||
|
||||
class BrowserContextCaptureRequest(BaseModel):
|
||||
thread_id: str | None = Field(default=None, description="Existing or caller-generated zncm thread id")
|
||||
title: str | None = Field(default=None, description="Current browser page title")
|
||||
url: str | None = Field(default=None, description="Current browser page URL")
|
||||
content: str = Field(description="Captured page content, preferably Markdown")
|
||||
content_type: Literal["markdown", "text"] = Field(default="markdown")
|
||||
prompt: str | None = Field(default=None, description="Optional first question to ask in zncm after capture")
|
||||
|
||||
|
||||
class BrowserContextFileInfo(BaseModel):
|
||||
filename: str
|
||||
size: int
|
||||
path: str
|
||||
virtual_path: str
|
||||
artifact_url: str
|
||||
|
||||
|
||||
class BrowserContextCaptureResponse(BaseModel):
|
||||
success: bool
|
||||
thread_id: str
|
||||
file: BrowserContextFileInfo
|
||||
chat_path: str
|
||||
chat_url: str
|
||||
browser_context_token: str
|
||||
prompt: str
|
||||
truncated: bool = False
|
||||
message: str
|
||||
|
||||
|
||||
class BrowserContextAskRequest(BaseModel):
|
||||
thread_id: str | None = Field(default=None, description="Existing zncm thread id for follow-up questions")
|
||||
title: str | None = Field(default=None, description="Current browser page title")
|
||||
url: str | None = Field(default=None, description="Current browser page URL")
|
||||
content: str = Field(description="Captured page content, preferably Markdown")
|
||||
content_type: Literal["markdown", "text"] = Field(default="markdown")
|
||||
question: str = Field(description="Question asked from the in-page assistant")
|
||||
|
||||
|
||||
class BrowserContextAskResponse(BaseModel):
|
||||
success: bool
|
||||
thread_id: str
|
||||
run_id: str
|
||||
answer: str
|
||||
truncated: bool = False
|
||||
message: str
|
||||
|
||||
|
||||
class BrowserContextTaskPayload(BaseModel):
|
||||
id: str
|
||||
kind: str = "fetch"
|
||||
url: str
|
||||
instruction: str = ""
|
||||
session_id: str = ""
|
||||
action: dict[str, Any] | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
max_content_chars: int
|
||||
|
||||
|
||||
class BrowserContextTaskNextResponse(BaseModel):
|
||||
task: BrowserContextTaskPayload | None = None
|
||||
|
||||
|
||||
class BrowserContextTaskResultRequest(BaseModel):
|
||||
kind: str = Field(default="fetch", description="Task kind the extension processed: fetch or act")
|
||||
title: str | None = Field(default=None, description="Captured page title")
|
||||
url: str | None = Field(default=None, description="Final page URL after redirects")
|
||||
content: str | None = Field(default=None, description="Captured page content")
|
||||
content_type: Literal["markdown", "text"] = Field(default="markdown")
|
||||
observation: dict[str, Any] | None = Field(default=None, description="Structured observation for an act step (elements, text, status)")
|
||||
error: str | None = Field(default=None, description="Extension-side task error")
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class BrowserContextTaskResultResponse(BaseModel):
|
||||
success: bool
|
||||
task_id: str
|
||||
status: str
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
def _settings(config: AppConfig) -> BrowserContextConfig:
|
||||
value = getattr(config, "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _require_enabled(config: AppConfig) -> BrowserContextConfig:
|
||||
settings = _settings(config)
|
||||
if not settings.enabled:
|
||||
raise HTTPException(status_code=404, detail="Browser context integration is disabled")
|
||||
return settings
|
||||
|
||||
|
||||
def _trim(value: str | None, limit: int) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
return value.strip()[:limit]
|
||||
|
||||
|
||||
def _limit_utf8_bytes(value: str, max_bytes: int) -> str:
|
||||
text = value
|
||||
while len(text.encode("utf-8")) > max_bytes:
|
||||
text = text[:-1]
|
||||
return text
|
||||
|
||||
|
||||
def _filename_part(value: str, fallback: str, *, max_bytes: int) -> str:
|
||||
text = value.strip() or fallback
|
||||
# Keep unicode word chars so Chinese titles remain readable, while removing
|
||||
# path separators and shell-hostile punctuation.
|
||||
text = re.sub(r"[^\w.-]+", "-", text, flags=re.UNICODE).strip("-._")
|
||||
text = _limit_utf8_bytes(text, max_bytes)
|
||||
return text or fallback
|
||||
|
||||
|
||||
def _build_markdown(
|
||||
*,
|
||||
title: str,
|
||||
url: str,
|
||||
content: str,
|
||||
content_type: str,
|
||||
captured_at: str,
|
||||
truncated: bool,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"# {title or 'Captured Web Page'}",
|
||||
"",
|
||||
f"- Source URL: {url or '(unknown)'}",
|
||||
f"- Captured at: {captured_at}",
|
||||
]
|
||||
if truncated:
|
||||
lines.append("- Note: Content was truncated by the browser_context.max_content_chars limit.")
|
||||
lines.extend(["", "---", ""])
|
||||
|
||||
if content_type == "markdown":
|
||||
lines.append(content.strip())
|
||||
else:
|
||||
lines.append("```text")
|
||||
lines.append(content.strip())
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _build_ask_prompt(
|
||||
*,
|
||||
title: str,
|
||||
url: str,
|
||||
content: str,
|
||||
content_type: str,
|
||||
captured_at: str,
|
||||
truncated: bool,
|
||||
) -> str:
|
||||
note = "页面内容已按配置截断,请只基于可见内容回答。" if truncated else "页面内容如下。"
|
||||
body = content.strip()
|
||||
if content_type == "text":
|
||||
body = f"```text\n{body}\n```"
|
||||
return "\n\n".join(
|
||||
[
|
||||
"你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。",
|
||||
"如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
|
||||
f"页面标题:{title or '未知标题'}",
|
||||
f"页面 URL:{url or '未知 URL'}",
|
||||
f"读取时间:{captured_at}",
|
||||
f"说明:{note}",
|
||||
"当前网页内容:",
|
||||
body,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _message_type(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("type") or message.get("role") or "").lower()
|
||||
return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower()
|
||||
|
||||
|
||||
def _content_to_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if isinstance(item.get("text"), str):
|
||||
parts.append(item["text"])
|
||||
elif item.get("type") == "text" and isinstance(item.get("content"), str):
|
||||
parts.append(item["content"])
|
||||
return "\n".join(part.strip() for part in parts if part and part.strip()).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _latest_ai_answer(values: dict[str, Any]) -> str:
|
||||
messages = values.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return ""
|
||||
for message in reversed(messages):
|
||||
if _message_type(message) not in {"ai", "assistant"}:
|
||||
continue
|
||||
answer = _content_to_text(message.get("content") if isinstance(message, dict) else getattr(message, "content", None))
|
||||
if answer:
|
||||
return answer
|
||||
return ""
|
||||
|
||||
|
||||
async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_path, data: bytes) -> None:
|
||||
sandbox_provider = get_sandbox_provider()
|
||||
if bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)):
|
||||
return
|
||||
sandbox_id = sandbox_provider.acquire(thread_id)
|
||||
sandbox = sandbox_provider.get(sandbox_id)
|
||||
if sandbox is None:
|
||||
raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
|
||||
_make_file_sandbox_writable(file_path)
|
||||
sandbox.update_file(virtual_path, data)
|
||||
|
||||
|
||||
@router.get("/status", response_model=BrowserContextStatusResponse)
|
||||
async def get_browser_context_status(
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextStatusResponse:
|
||||
settings = _settings(config)
|
||||
return BrowserContextStatusResponse(
|
||||
enabled=settings.enabled,
|
||||
max_content_chars=settings.max_content_chars,
|
||||
max_title_chars=settings.max_title_chars,
|
||||
max_url_chars=settings.max_url_chars,
|
||||
task_result_timeout_seconds=settings.task_result_timeout_seconds,
|
||||
task_poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
task_lease_seconds=settings.task_lease_seconds,
|
||||
tool_result_max_chars=settings.tool_result_max_chars,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse)
|
||||
@require_permission("runs", "create")
|
||||
async def claim_browser_context_task(
|
||||
request: Request,
|
||||
worker_id: str | None = None,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextTaskNextResponse:
|
||||
settings = _require_enabled(config)
|
||||
prune_browser_tasks()
|
||||
task = claim_next_browser_task(
|
||||
user_id=get_effective_user_id(),
|
||||
worker_id=worker_id or "chrome-extension",
|
||||
lease_seconds=settings.task_lease_seconds,
|
||||
)
|
||||
if task is None:
|
||||
return BrowserContextTaskNextResponse(task=None)
|
||||
return BrowserContextTaskNextResponse(task=BrowserContextTaskPayload.model_validate(task))
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/result", response_model=BrowserContextTaskResultResponse)
|
||||
@require_permission("runs", "create")
|
||||
async def submit_browser_context_task_result(
|
||||
task_id: str,
|
||||
body: BrowserContextTaskResultRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextTaskResultResponse:
|
||||
settings = _require_enabled(config)
|
||||
error = _trim(body.error, 4000)
|
||||
content = body.content or ""
|
||||
is_act = (body.kind or "fetch") == "act"
|
||||
if not error and not is_act and not content.strip():
|
||||
raise HTTPException(status_code=400, detail="Captured page content is empty")
|
||||
if not error and is_act and not isinstance(body.observation, dict):
|
||||
raise HTTPException(status_code=400, detail="Act task result requires an observation")
|
||||
|
||||
task = complete_browser_task(
|
||||
task_id=task_id,
|
||||
user_id=get_effective_user_id(),
|
||||
title=_trim(body.title, settings.max_title_chars),
|
||||
url=_trim(body.url, settings.max_url_chars),
|
||||
content=content,
|
||||
content_type=body.content_type,
|
||||
error=error,
|
||||
metadata=body.metadata,
|
||||
observation=body.observation,
|
||||
)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail=f"Browser task {task_id} not found")
|
||||
|
||||
result = task.get("result") or {}
|
||||
return BrowserContextTaskResultResponse(
|
||||
success=True,
|
||||
task_id=task_id,
|
||||
status=str(task.get("status") or "unknown"),
|
||||
truncated=bool(result.get("truncated")),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/capture", response_model=BrowserContextCaptureResponse)
|
||||
@require_permission("threads", "write")
|
||||
async def capture_browser_context(
|
||||
body: BrowserContextCaptureRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextCaptureResponse:
|
||||
settings = _require_enabled(config)
|
||||
|
||||
raw_content = (body.content or "").strip()
|
||||
if not raw_content:
|
||||
raise HTTPException(status_code=400, detail="Captured page content is empty")
|
||||
|
||||
title = _trim(body.title, settings.max_title_chars)
|
||||
url = _trim(body.url, settings.max_url_chars)
|
||||
truncated = len(raw_content) > settings.max_content_chars
|
||||
content = raw_content[: settings.max_content_chars] if truncated else raw_content
|
||||
|
||||
requested_thread_id = (body.thread_id or "").strip() or None
|
||||
if requested_thread_id:
|
||||
thread_store = get_thread_store(request)
|
||||
allowed = await thread_store.check_access(
|
||||
requested_thread_id,
|
||||
get_effective_user_id(),
|
||||
require_existing=False,
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
|
||||
|
||||
thread_response = await create_thread(
|
||||
ThreadCreateRequest(
|
||||
thread_id=requested_thread_id,
|
||||
metadata={
|
||||
"source": "browser_context",
|
||||
"browser_context_url": url,
|
||||
"browser_context_title": title,
|
||||
},
|
||||
),
|
||||
request,
|
||||
)
|
||||
thread_id = thread_response.thread_id
|
||||
|
||||
captured_at = datetime.now(UTC).isoformat()
|
||||
markdown = _build_markdown(
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
content_type=body.content_type,
|
||||
captured_at=captured_at,
|
||||
truncated=truncated,
|
||||
)
|
||||
data = markdown.encode("utf-8")
|
||||
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
prefix = _filename_part(settings.filename_prefix, "webpage", max_bytes=40)
|
||||
title_part = _filename_part(title, "page", max_bytes=120)
|
||||
filename = normalize_filename(f"{prefix}-{timestamp}-{title_part}.md")
|
||||
|
||||
try:
|
||||
uploads_dir = ensure_uploads_dir(thread_id)
|
||||
file_path, fh = open_upload_file_no_symlink(uploads_dir, filename)
|
||||
with fh:
|
||||
fh.write(data)
|
||||
except UnsafeUploadPathError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to save browser context capture for thread %s", thread_id)
|
||||
raise HTTPException(status_code=500, detail="Failed to save captured page") from exc
|
||||
|
||||
virtual_path = upload_virtual_path(filename)
|
||||
await _sync_to_sandbox_if_needed(thread_id, virtual_path, file_path, data)
|
||||
|
||||
prompt = (body.prompt or "").strip() or DEFAULT_BROWSER_CONTEXT_PROMPT
|
||||
token = uuid4().hex
|
||||
route_base = settings.route_base.rstrip("/") or "/page/workspace"
|
||||
chat_path = f"{route_base}/chats/{thread_id}"
|
||||
query = urlencode(
|
||||
{
|
||||
"browser_context": token,
|
||||
"bc_filename": filename,
|
||||
"bc_size": str(len(data)),
|
||||
"bc_path": virtual_path,
|
||||
"bc_prompt": prompt,
|
||||
}
|
||||
)
|
||||
chat_url = f"#{chat_path}?{query}"
|
||||
|
||||
return BrowserContextCaptureResponse(
|
||||
success=True,
|
||||
thread_id=thread_id,
|
||||
file=BrowserContextFileInfo(
|
||||
filename=filename,
|
||||
size=len(data),
|
||||
path=virtual_path,
|
||||
virtual_path=virtual_path,
|
||||
artifact_url=upload_artifact_url(thread_id, filename),
|
||||
),
|
||||
chat_path=chat_path,
|
||||
chat_url=chat_url,
|
||||
browser_context_token=token,
|
||||
prompt=prompt,
|
||||
truncated=truncated,
|
||||
message="Captured page saved as a zncm upload",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ask", response_model=BrowserContextAskResponse)
|
||||
@require_permission("runs", "create")
|
||||
async def ask_browser_context(
|
||||
body: BrowserContextAskRequest,
|
||||
request: Request,
|
||||
config: AppConfig = Depends(get_config),
|
||||
) -> BrowserContextAskResponse:
|
||||
settings = _require_enabled(config)
|
||||
|
||||
raw_content = (body.content or "").strip()
|
||||
if not raw_content:
|
||||
raise HTTPException(status_code=400, detail="Captured page content is empty")
|
||||
|
||||
question = (body.question or "").strip()
|
||||
if not question:
|
||||
raise HTTPException(status_code=400, detail="Question is empty")
|
||||
|
||||
title = _trim(body.title, settings.max_title_chars)
|
||||
url = _trim(body.url, settings.max_url_chars)
|
||||
content_limit = max(1_000, min(settings.max_content_chars, settings.tool_result_max_chars))
|
||||
truncated = len(raw_content) > content_limit
|
||||
content = raw_content[:content_limit] if truncated else raw_content
|
||||
|
||||
requested_thread_id = (body.thread_id or "").strip() or None
|
||||
if requested_thread_id:
|
||||
thread_store = get_thread_store(request)
|
||||
allowed = await thread_store.check_access(
|
||||
requested_thread_id,
|
||||
get_effective_user_id(),
|
||||
require_existing=False,
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
|
||||
|
||||
thread_response = await create_thread(
|
||||
ThreadCreateRequest(
|
||||
thread_id=requested_thread_id,
|
||||
assistant_id="lead_agent",
|
||||
metadata={
|
||||
"source": "browser_context_page_assistant",
|
||||
"browser_context_url": url,
|
||||
"browser_context_title": title,
|
||||
},
|
||||
),
|
||||
request,
|
||||
)
|
||||
thread_id = thread_response.thread_id
|
||||
captured_at = datetime.now(UTC).isoformat()
|
||||
is_follow_up = requested_thread_id is not None
|
||||
if is_follow_up:
|
||||
prompt_prefix = "\n\n".join(
|
||||
[
|
||||
"你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。",
|
||||
"如果本会话里的网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
|
||||
f"当前页面标题:{title or '未知标题'}",
|
||||
f"当前页面 URL:{url or '未知 URL'}",
|
||||
f"本轮提问时间:{captured_at}",
|
||||
]
|
||||
)
|
||||
else:
|
||||
prompt_prefix = _build_ask_prompt(
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
content_type=body.content_type,
|
||||
captured_at=captured_at,
|
||||
truncated=truncated,
|
||||
)
|
||||
|
||||
run_body = RunCreateRequest(
|
||||
assistant_id="lead_agent",
|
||||
input={
|
||||
"messages": [
|
||||
{
|
||||
"type": "human",
|
||||
"content": question,
|
||||
"additional_kwargs": {
|
||||
"prompt_prefix": prompt_prefix,
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
metadata={
|
||||
"source": "browser_context_page_assistant",
|
||||
"browser_context_url": url,
|
||||
"browser_context_title": title,
|
||||
"browser_context_truncated": truncated,
|
||||
},
|
||||
stream_mode=["values"],
|
||||
on_disconnect="continue",
|
||||
multitask_strategy="reject",
|
||||
)
|
||||
|
||||
record = await start_run(run_body, thread_id, request)
|
||||
if record.task is not None:
|
||||
try:
|
||||
await record.task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
checkpointer = get_checkpointer(request)
|
||||
values: dict[str, Any] = {}
|
||||
try:
|
||||
checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
if checkpoint_tuple is not None:
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
||||
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch browser context answer for run %s", record.run_id)
|
||||
|
||||
answer = _latest_ai_answer(values)
|
||||
if not answer:
|
||||
answer = record.error or "zncm did not return an answer."
|
||||
|
||||
return BrowserContextAskResponse(
|
||||
success=bool(answer),
|
||||
thread_id=thread_id,
|
||||
run_id=record.run_id,
|
||||
answer=answer,
|
||||
truncated=truncated,
|
||||
message="zncm page assistant answer completed",
|
||||
)
|
||||
@@ -10,90 +10,90 @@ gateway:
|
||||
port: 8001
|
||||
|
||||
models:
|
||||
- name: deepseek-chat
|
||||
display_name: DeepSeek Chat
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-chat
|
||||
api_key: sk-1520028668cb4cc388ed8315ff1e856e
|
||||
base_url: https://api.deepseek.com
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: false
|
||||
supports_thinking: true
|
||||
# - name: deepseek-chat
|
||||
# display_name: DeepSeek Chat
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-chat
|
||||
# api_key: sk-1520028668cb4cc388ed8315ff1e856e
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: false
|
||||
# supports_thinking: true
|
||||
|
||||
- name: deepseek-v4-flash
|
||||
display_name: DeepSeek V4 Flash
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-v4-flash
|
||||
api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
base_url: https://api.deepseek.com
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: false
|
||||
supports_thinking: true
|
||||
# - name: deepseek-v4-flash
|
||||
# display_name: DeepSeek V4 Flash
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-v4-flash
|
||||
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: false
|
||||
# supports_thinking: true
|
||||
|
||||
- name: deepseek-v4-pro
|
||||
display_name: DeepSeek V4 Pro
|
||||
# - name: deepseek-v4-pro
|
||||
# display_name: DeepSeek V4 Pro
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-v4-pro
|
||||
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: false
|
||||
# supports_thinking: true
|
||||
# - name: MiniMax-M2.7
|
||||
# display_name: MiniMax-M2.7
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: MiniMax-M2.7
|
||||
# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
# base_url: https://api.minimaxi.com/v1
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: glm-5.1
|
||||
# display_name: glm-5.1
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: glm-5.1
|
||||
# api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR
|
||||
# base_url: https://open.bigmodel.cn/api/paas/v4/
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: deepseek-ai/DeepSeek-OCR
|
||||
# display_name: deepseek-ai/DeepSeek-OCR
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-ai/DeepSeek-OCR
|
||||
# api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
|
||||
# base_url: https://api.siliconflow.cn/v1/
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: zai-org/GLM-5-FP8
|
||||
# display_name: zai-org/GLM-5-FP8
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: zai-org/GLM-5-FP8
|
||||
# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
|
||||
# base_url: https://api.us-west-2.modal.direct/v1
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
- name: deepseek-ai/DeepSeek-V4-Flash
|
||||
display_name: deepseek-ai/DeepSeek-V4-Flash
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-v4-pro
|
||||
api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
base_url: https://api.deepseek.com
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: false
|
||||
supports_thinking: true
|
||||
- name: MiniMax-M2.7
|
||||
display_name: MiniMax-M2.7
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.7
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com/v1
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: glm-5.1
|
||||
display_name: glm-5.1
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: glm-5.1
|
||||
api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR
|
||||
base_url: https://open.bigmodel.cn/api/paas/v4/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: deepseek-ai/DeepSeek-OCR
|
||||
display_name: deepseek-ai/DeepSeek-OCR
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-ai/DeepSeek-OCR
|
||||
api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: zai-org/GLM-5-FP8
|
||||
display_name: zai-org/GLM-5-FP8
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: zai-org/GLM-5-FP8
|
||||
api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
|
||||
base_url: https://api.us-west-2.modal.direct/v1
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: Qwen/Qwen3.6-27B
|
||||
display_name: Qwen/Qwen3.6-27B
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: Qwen/Qwen3.6-27B
|
||||
model: deepseek-ai/DeepSeek-V4-Flash
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
@@ -122,6 +122,26 @@ uploads:
|
||||
auto_convert_documents: true
|
||||
pdf_converter: auto
|
||||
|
||||
# Chrome 插件「当前网页发送到 Deer」入口。默认关闭,打开后才启用
|
||||
# /api/browser-context/capture。它只保存当前页 Markdown 到线程 uploads,
|
||||
# 不会改变普通聊天、上传或工具调用流程。
|
||||
browser_context:
|
||||
enabled: true
|
||||
max_content_chars: 200000
|
||||
max_title_chars: 300
|
||||
max_url_chars: 2048
|
||||
filename_prefix: webpage
|
||||
route_base: /page/workspace
|
||||
task_result_timeout_seconds: 60
|
||||
task_poll_interval_seconds: 1
|
||||
task_lease_seconds: 120
|
||||
tool_result_max_chars: 60000
|
||||
# 交互式页面操作(browser_act:点击/输入/滚动/等待/跳转)。关闭后只保留只读抓取。
|
||||
actions_enabled: true
|
||||
action_result_timeout_seconds: 45
|
||||
max_observed_elements: 150
|
||||
observation_text_max_chars: 12000
|
||||
|
||||
title:
|
||||
enabled: true
|
||||
max_words: 6
|
||||
@@ -139,6 +159,7 @@ title:
|
||||
|
||||
tool_groups:
|
||||
- name: web
|
||||
- name: browser
|
||||
- name: file:read
|
||||
- name: file:write
|
||||
- name: bash
|
||||
@@ -154,6 +175,14 @@ tools:
|
||||
use: deerflow.community.jina_ai.tools:web_fetch_tool
|
||||
timeout: 15
|
||||
|
||||
- name: browser_fetch_page
|
||||
group: browser
|
||||
use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
|
||||
|
||||
- name: browser_act
|
||||
group: browser
|
||||
use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
|
||||
|
||||
- name: ls
|
||||
group: file:read
|
||||
use: deerflow.sandbox.tools:ls_tool
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Browser-backed page context helpers."""
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Small file-backed task queue for browser page fetch requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LOCK = threading.RLock()
|
||||
_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$")
|
||||
_TERMINAL_STATUSES = {"done", "error", "expired"}
|
||||
|
||||
|
||||
def tasks_dir() -> Path:
|
||||
"""Return the browser-context task directory, creating it if needed."""
|
||||
|
||||
path = runtime_home() / "browser-context" / "tasks"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None = None) -> str:
|
||||
return (value or _now()).isoformat()
|
||||
|
||||
|
||||
def _parse_iso(value: Any) -> datetime | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _task_path(task_id: str) -> Path | None:
|
||||
if not _TASK_ID_RE.fullmatch(task_id or ""):
|
||||
return None
|
||||
return tasks_dir() / f"{task_id}.json"
|
||||
|
||||
|
||||
def _read_task(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except Exception:
|
||||
logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True)
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _write_task(path: Path, data: dict[str, Any]) -> None:
|
||||
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
|
||||
with tmp.open("w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, separators=(",", ":"))
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _public_task(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": data.get("id"),
|
||||
"kind": data.get("kind") or "fetch",
|
||||
"url": data.get("url") or "",
|
||||
"instruction": data.get("instruction") or "",
|
||||
"session_id": data.get("session_id") or "",
|
||||
"action": data.get("action") or None,
|
||||
"created_at": data.get("created_at") or "",
|
||||
"updated_at": data.get("updated_at") or "",
|
||||
"max_content_chars": int(data.get("max_content_chars") or 200_000),
|
||||
}
|
||||
|
||||
|
||||
def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]:
|
||||
if data.get("status") in _TERMINAL_STATUSES:
|
||||
return data
|
||||
expires_at = _parse_iso(data.get("expires_at"))
|
||||
if expires_at is not None and now >= expires_at:
|
||||
data = dict(data)
|
||||
data["status"] = "expired"
|
||||
data["error"] = "Browser task expired before the extension returned a result."
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return data
|
||||
|
||||
|
||||
def prune_browser_tasks(max_age_seconds: int = 86_400) -> int:
|
||||
"""Delete old terminal tasks and return the number removed."""
|
||||
|
||||
cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds)))
|
||||
removed = 0
|
||||
with _LOCK:
|
||||
for path in tasks_dir().glob("*.json"):
|
||||
data = _read_task(path)
|
||||
if not data or data.get("status") not in _TERMINAL_STATUSES:
|
||||
continue
|
||||
updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at"))
|
||||
if updated_at is None or updated_at < cutoff:
|
||||
try:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
logger.debug("Failed to delete old browser task: %s", path, exc_info=True)
|
||||
return removed
|
||||
|
||||
|
||||
def create_browser_task(
|
||||
*,
|
||||
user_id: str,
|
||||
url: str,
|
||||
instruction: str = "",
|
||||
timeout_seconds: int = 60,
|
||||
max_content_chars: int = 200_000,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
kind: str = "fetch",
|
||||
session_id: str = "",
|
||||
action: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a browser task for the current user's extension to pick up.
|
||||
|
||||
``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or
|
||||
``"act"`` for an interactive observe/act step driven by ``action`` within a
|
||||
long-lived ``session_id`` (a tab kept open by the extension).
|
||||
"""
|
||||
|
||||
now = _now()
|
||||
task_id = uuid4().hex
|
||||
task = {
|
||||
"id": task_id,
|
||||
"user_id": str(user_id or "default"),
|
||||
"status": "pending",
|
||||
"kind": kind if kind in {"fetch", "act"} else "fetch",
|
||||
"url": url,
|
||||
"instruction": instruction or "",
|
||||
"session_id": str(session_id or ""),
|
||||
"action": action or None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))),
|
||||
"max_content_chars": max(1_000, int(max_content_chars)),
|
||||
"worker_id": None,
|
||||
"claimed_at": None,
|
||||
"result": None,
|
||||
"error": None,
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
path = tasks_dir() / f"{task_id}.json"
|
||||
with _LOCK:
|
||||
_write_task(path, task)
|
||||
return task
|
||||
|
||||
|
||||
def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
"""Read a task if it belongs to the requested user."""
|
||||
|
||||
path = _task_path(task_id)
|
||||
if path is None:
|
||||
return None
|
||||
with _LOCK:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
|
||||
return None
|
||||
return _expire_if_needed(path, data, _now())
|
||||
|
||||
|
||||
def claim_next_browser_task(
|
||||
*,
|
||||
user_id: str,
|
||||
worker_id: str,
|
||||
lease_seconds: int = 120,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Claim the oldest pending task for a user."""
|
||||
|
||||
now = _now()
|
||||
user_id = str(user_id or "default")
|
||||
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
|
||||
lease_seconds = max(10, int(lease_seconds))
|
||||
|
||||
with _LOCK:
|
||||
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||
for path in paths:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != user_id:
|
||||
continue
|
||||
|
||||
data = _expire_if_needed(path, data, now)
|
||||
status = data.get("status")
|
||||
if status == "running":
|
||||
updated_at = _parse_iso(data.get("updated_at")) or now
|
||||
if (now - updated_at).total_seconds() <= lease_seconds:
|
||||
continue
|
||||
status = "pending"
|
||||
data = dict(data)
|
||||
data["status"] = status
|
||||
|
||||
if status != "pending":
|
||||
continue
|
||||
|
||||
data = dict(data)
|
||||
data["status"] = "running"
|
||||
data["worker_id"] = worker_id
|
||||
data["claimed_at"] = _iso(now)
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return _public_task(data)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def complete_browser_task(
|
||||
*,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
title: str = "",
|
||||
url: str = "",
|
||||
content: str = "",
|
||||
content_type: str = "markdown",
|
||||
error: str = "",
|
||||
metadata: dict[str, Any] | None = None,
|
||||
observation: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Mark a browser task complete or failed.
|
||||
|
||||
For ``act`` tasks the extension returns a structured ``observation`` (final
|
||||
URL/title, interactive element list, page text) instead of a captured
|
||||
markdown document; it is stored alongside the result verbatim.
|
||||
"""
|
||||
|
||||
path = _task_path(task_id)
|
||||
if path is None:
|
||||
return None
|
||||
|
||||
now = _now()
|
||||
with _LOCK:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
|
||||
return None
|
||||
|
||||
data = dict(data)
|
||||
if error:
|
||||
data["status"] = "error"
|
||||
data["error"] = str(error).strip()[:4000]
|
||||
data["result"] = None
|
||||
else:
|
||||
raw_content = str(content or "").strip()
|
||||
max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000))
|
||||
truncated = len(raw_content) > max_content_chars
|
||||
data["status"] = "done"
|
||||
data["error"] = None
|
||||
data["result"] = {
|
||||
"title": str(title or "").strip()[:500],
|
||||
"url": str(url or data.get("url") or "").strip()[:4096],
|
||||
"content": raw_content[:max_content_chars] if truncated else raw_content,
|
||||
"content_type": content_type if content_type in {"markdown", "text"} else "markdown",
|
||||
"truncated": truncated,
|
||||
"observation": observation if isinstance(observation, dict) else None,
|
||||
"captured_at": _iso(now),
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return data
|
||||
|
||||
|
||||
async def wait_for_browser_task_result(
|
||||
*,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
timeout_seconds: int,
|
||||
poll_interval_seconds: float = 1.0,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Wait for a browser task to reach a terminal state."""
|
||||
|
||||
deadline = time.monotonic() + max(1, int(timeout_seconds))
|
||||
interval = max(0.25, float(poll_interval_seconds))
|
||||
while True:
|
||||
task = get_browser_task(task_id=task_id, user_id=user_id)
|
||||
if task is None:
|
||||
return None
|
||||
if task.get("status") in _TERMINAL_STATUSES:
|
||||
return task
|
||||
if time.monotonic() >= deadline:
|
||||
return task
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
@@ -9,8 +9,9 @@ from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.gateway_config import GatewayConfig
|
||||
@@ -132,7 +133,8 @@ class AppConfig(BaseModel):
|
||||
gateway: GatewayConfig = Field(default_factory=GatewayConfig, description="Gateway FastAPI listening port (used by intra-process loopback)")
|
||||
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
|
||||
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
|
||||
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
|
||||
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
|
||||
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
|
||||
|
||||
@classmethod
|
||||
def resolve_config_path(cls, config_path: str | None = None) -> Path:
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Configuration for browser page context capture."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class BrowserContextConfig(BaseModel):
|
||||
"""Controls the optional Chrome-extension page capture integration."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable /api/browser-context endpoints for Chrome extension page capture.",
|
||||
)
|
||||
max_content_chars: int = Field(
|
||||
default=200_000,
|
||||
ge=1_000,
|
||||
description="Maximum captured page text stored per request. Longer content is truncated.",
|
||||
)
|
||||
max_title_chars: int = Field(
|
||||
default=300,
|
||||
ge=20,
|
||||
description="Maximum title length accepted from the browser extension.",
|
||||
)
|
||||
max_url_chars: int = Field(
|
||||
default=2_048,
|
||||
ge=100,
|
||||
description="Maximum source URL length accepted from the browser extension.",
|
||||
)
|
||||
filename_prefix: str = Field(
|
||||
default="webpage",
|
||||
description="Prefix used for Markdown files created from captured pages.",
|
||||
)
|
||||
route_base: str = Field(
|
||||
default="/page/workspace",
|
||||
description="Frontend route base used when returning chat paths to the extension.",
|
||||
)
|
||||
task_result_timeout_seconds: int = Field(
|
||||
default=60,
|
||||
ge=5,
|
||||
description="Maximum time the browser_fetch_page tool waits for the extension result.",
|
||||
)
|
||||
task_poll_interval_seconds: float = Field(
|
||||
default=1.0,
|
||||
ge=0.25,
|
||||
description="How often the backend tool checks for a returned browser task result.",
|
||||
)
|
||||
task_lease_seconds: int = Field(
|
||||
default=120,
|
||||
ge=10,
|
||||
description="How long a claimed browser task is reserved before another extension may retry it.",
|
||||
)
|
||||
tool_result_max_chars: int = Field(
|
||||
default=60_000,
|
||||
ge=1_000,
|
||||
description="Maximum page characters returned from browser_fetch_page into the model context.",
|
||||
)
|
||||
actions_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).",
|
||||
)
|
||||
action_result_timeout_seconds: int = Field(
|
||||
default=45,
|
||||
ge=5,
|
||||
description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.",
|
||||
)
|
||||
max_observed_elements: int = Field(
|
||||
default=150,
|
||||
ge=10,
|
||||
description="Maximum number of interactive elements returned in a browser_act observation.",
|
||||
)
|
||||
observation_text_max_chars: int = Field(
|
||||
default=12_000,
|
||||
ge=500,
|
||||
description="Maximum page text characters returned alongside a browser_act observation.",
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""Interactive browser action tool backed by the user's Chrome extension.
|
||||
|
||||
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
|
||||
*observe -> act* loop on a live page the extension keeps open:
|
||||
|
||||
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
|
||||
2. The extension performs the action on the session tab, then re-scans the page
|
||||
and returns an *observation*: the final URL/title, an indexed list of
|
||||
interactive elements, and the visible page text.
|
||||
3. The model reads the observation, picks the next element by its index, and
|
||||
calls ``browser_act`` again with the same ``session_id``.
|
||||
|
||||
The element ``index`` values are only valid for the most recent observation of a
|
||||
session; always act on indices from the latest result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
|
||||
_NEEDS_INDEX = {"click", "type"}
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
|
||||
if not elements:
|
||||
return "(no interactive elements detected)"
|
||||
lines: list[str] = []
|
||||
for el in elements:
|
||||
idx = el.get("index")
|
||||
tag = str(el.get("tag") or "?")
|
||||
el_type = str(el.get("type") or "").strip()
|
||||
label = str(el.get("label") or "").strip().replace("\n", " ")
|
||||
if len(label) > 120:
|
||||
label = label[:120] + "…"
|
||||
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
|
||||
extra = ""
|
||||
if el.get("value"):
|
||||
value = str(el["value"]).strip().replace("\n", " ")
|
||||
extra = f' value="{value[:60]}"'
|
||||
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
|
||||
observation = observation or {}
|
||||
status = str(observation.get("action_status") or "ok")
|
||||
message = str(observation.get("message") or "").strip()
|
||||
url = str(observation.get("url") or "").strip() or "(unknown)"
|
||||
title = str(observation.get("title") or "").strip() or "(untitled)"
|
||||
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
|
||||
text = str(observation.get("text") or "").strip()
|
||||
text_limit = max(500, int(settings.observation_text_max_chars))
|
||||
text_truncated = len(text) > text_limit
|
||||
if text_truncated:
|
||||
text = text[:text_limit]
|
||||
|
||||
parts = [
|
||||
f"Browser session: {session_id}",
|
||||
f"Last action: {action} -> {status}",
|
||||
]
|
||||
if message:
|
||||
parts.append(f"Note: {message}")
|
||||
parts.extend(
|
||||
[
|
||||
f"URL: {url}",
|
||||
f"Title: {title}",
|
||||
"",
|
||||
"Interactive elements (use the [index] with click/type):",
|
||||
_format_elements(elements),
|
||||
"",
|
||||
"Page text:",
|
||||
text or "(empty page text)",
|
||||
]
|
||||
)
|
||||
if text_truncated:
|
||||
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
|
||||
parts.append(
|
||||
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
|
||||
"When the task is finished, call browser_act with action=\"close\" to release the tab."
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@tool("browser_act", parse_docstring=True)
|
||||
async def browser_act_tool(
|
||||
action: str,
|
||||
session_id: str = "",
|
||||
url: str = "",
|
||||
index: int = -1,
|
||||
text: str = "",
|
||||
submit: bool = False,
|
||||
wait_for: str = "",
|
||||
wait_seconds: int = 0,
|
||||
) -> str:
|
||||
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
|
||||
|
||||
Use this to operate a real web page on the user's behalf — log in, fill forms,
|
||||
click through navigation, and read pages that need the user's session or local
|
||||
network. Start every task with ``action="navigate"`` and a URL to open a tab
|
||||
and get the first observation, then keep calling this tool with the same
|
||||
``session_id``, choosing elements by the ``index`` shown in the observation.
|
||||
Call ``action="close"`` when done to free the tab.
|
||||
|
||||
Args:
|
||||
action: One of navigate, click, type, scroll, wait, extract, observe, close.
|
||||
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
|
||||
url: Target http(s) URL. Required for navigate (and to start a new session).
|
||||
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
|
||||
text: Text to enter for the type action.
|
||||
submit: For type, press Enter / submit the form after typing.
|
||||
wait_for: For wait, a CSS selector to wait for before observing.
|
||||
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
|
||||
if not settings.actions_enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
|
||||
|
||||
action = (action or "").strip().lower()
|
||||
if action not in _ACTIONS:
|
||||
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
|
||||
|
||||
session_id = (session_id or "").strip()
|
||||
safe_url = _validate_url(url) if url else None
|
||||
|
||||
if action == "navigate" and safe_url is None:
|
||||
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
if not session_id:
|
||||
if action != "navigate":
|
||||
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
|
||||
session_id = uuid4().hex
|
||||
|
||||
if action in _NEEDS_INDEX and index < 0:
|
||||
return f"browser_act {action} requires a non-negative element index from the latest observation."
|
||||
if action == "type" and not text:
|
||||
return "browser_act type requires the text to enter."
|
||||
|
||||
action_payload: dict[str, Any] = {"type": action}
|
||||
if safe_url is not None:
|
||||
action_payload["url"] = safe_url
|
||||
if index >= 0:
|
||||
action_payload["index"] = int(index)
|
||||
if text:
|
||||
action_payload["text"] = text
|
||||
if submit:
|
||||
action_payload["submit"] = True
|
||||
if wait_for.strip():
|
||||
action_payload["wait_for"] = wait_for.strip()[:300]
|
||||
|
||||
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url or "",
|
||||
instruction="",
|
||||
kind="act",
|
||||
session_id=session_id,
|
||||
action=action_payload,
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_act_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
|
||||
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
if action == "close":
|
||||
return f"browser_act session {session_id}: tab closed."
|
||||
|
||||
result = completed.get("result") or {}
|
||||
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
|
||||
return _format_observation(action, observation, session_id, settings)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""Browser extension backed page fetch tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
@tool("browser_fetch_page", parse_docstring=True)
|
||||
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
|
||||
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
|
||||
|
||||
Use this when the user explicitly asks to use the browser extension or asks
|
||||
for data from a page that may require their browser session, login state, or
|
||||
local network access. The URL must be an exact http(s) URL provided by the
|
||||
user. If the user did not provide a URL, ask for the URL instead of guessing.
|
||||
|
||||
Args:
|
||||
url: Exact http(s) URL to open in the user's Chrome browser.
|
||||
question: Optional instruction describing what data the user wants from the page.
|
||||
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
|
||||
|
||||
safe_url = _validate_url(url)
|
||||
if safe_url is None:
|
||||
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url,
|
||||
instruction=(question or "").strip(),
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_fetch_page_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
|
||||
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
result = completed.get("result") or {}
|
||||
content = str(result.get("content") or "").strip()
|
||||
tool_limit = max(1_000, int(settings.tool_result_max_chars))
|
||||
tool_truncated = len(content) > tool_limit
|
||||
if tool_truncated:
|
||||
content = content[:tool_limit]
|
||||
|
||||
notes: list[str] = []
|
||||
if result.get("truncated"):
|
||||
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
|
||||
if tool_truncated:
|
||||
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
|
||||
|
||||
parts = [
|
||||
"Browser page fetched through the user's Chrome extension.",
|
||||
f"Task ID: {task['id']}",
|
||||
f"Title: {result.get('title') or '(untitled)'}",
|
||||
f"URL: {result.get('url') or safe_url}",
|
||||
]
|
||||
if question:
|
||||
parts.extend(["", "Requested extraction:", question.strip()])
|
||||
if notes:
|
||||
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
|
||||
parts.extend(["", "Page content:", content or "(empty page text)"])
|
||||
return "\n".join(parts)
|
||||
|
||||
@@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool"
|
||||
BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool"
|
||||
BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE}
|
||||
BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"}
|
||||
|
||||
BUILTIN_TOOLS = [
|
||||
present_file_tool,
|
||||
ask_clarification_tool,
|
||||
@@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _is_browser_context_tool(tool: object) -> bool:
|
||||
"""Return True if the tool config represents the browser extension bridge."""
|
||||
|
||||
use = getattr(tool, "use", None)
|
||||
name = getattr(tool, "name", None)
|
||||
return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
|
||||
|
||||
|
||||
def get_available_tools(
|
||||
groups: list[str] | None = None,
|
||||
include_mcp: bool = True,
|
||||
@@ -64,6 +77,18 @@ def get_available_tools(
|
||||
config = app_config or get_app_config()
|
||||
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
|
||||
|
||||
browser_context_config = getattr(config, "browser_context", None)
|
||||
if not getattr(browser_context_config, "enabled", False):
|
||||
tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
|
||||
elif not getattr(browser_context_config, "actions_enabled", True):
|
||||
# browser_context is on, but interactive page actions are disabled — keep
|
||||
# the read-only browser_fetch_page tool, drop only browser_act.
|
||||
tool_configs = [
|
||||
tool
|
||||
for tool in tool_configs
|
||||
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
|
||||
]
|
||||
|
||||
# Do not expose host bash by default when LocalSandboxProvider is active.
|
||||
if not is_host_bash_allowed(config):
|
||||
tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)]
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Tests for the interactive browser_act tool and its act-task queue support."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.browser_context import tasks as task_queue
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.tools.builtins import browser_act_tool as act_mod
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_task_home(tmp_path, monkeypatch):
|
||||
"""Point the file-backed task queue at a throwaway directory."""
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
|
||||
yield
|
||||
|
||||
|
||||
def _enable_actions(monkeypatch, *, enabled=True, actions_enabled=True):
|
||||
cfg = BrowserContextConfig(
|
||||
enabled=enabled,
|
||||
actions_enabled=actions_enabled,
|
||||
action_result_timeout_seconds=5,
|
||||
task_poll_interval_seconds=0.25,
|
||||
task_lease_seconds=30,
|
||||
)
|
||||
monkeypatch.setattr(act_mod, "_settings", lambda: cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
# --- task queue layer -------------------------------------------------------
|
||||
|
||||
|
||||
def test_act_task_round_trip_carries_session_and_observation():
|
||||
task = task_queue.create_browser_task(
|
||||
user_id="u1",
|
||||
url="https://example.com",
|
||||
kind="act",
|
||||
session_id="sess-1",
|
||||
action={"type": "navigate", "url": "https://example.com"},
|
||||
)
|
||||
assert task["kind"] == "act"
|
||||
assert task["session_id"] == "sess-1"
|
||||
|
||||
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1")
|
||||
assert claimed is not None
|
||||
assert claimed["kind"] == "act"
|
||||
assert claimed["session_id"] == "sess-1"
|
||||
assert claimed["action"] == {"type": "navigate", "url": "https://example.com"}
|
||||
|
||||
observation = {
|
||||
"action_status": "ok",
|
||||
"url": "https://example.com/home",
|
||||
"title": "Home",
|
||||
"elements": [{"index": 0, "tag": "button", "label": "Login"}],
|
||||
"text": "welcome",
|
||||
}
|
||||
done = task_queue.complete_browser_task(
|
||||
task_id=task["id"],
|
||||
user_id="u1",
|
||||
observation=observation,
|
||||
)
|
||||
assert done is not None
|
||||
assert done["status"] == "done"
|
||||
assert done["result"]["observation"] == observation
|
||||
|
||||
|
||||
def test_fetch_task_default_kind_is_fetch():
|
||||
task = task_queue.create_browser_task(user_id="u1", url="https://example.com")
|
||||
assert task["kind"] == "fetch"
|
||||
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1")
|
||||
assert claimed["kind"] == "fetch"
|
||||
assert claimed["session_id"] == ""
|
||||
assert claimed["action"] is None
|
||||
|
||||
|
||||
# --- tool validation --------------------------------------------------------
|
||||
|
||||
|
||||
async def _invoke(**kwargs) -> str:
|
||||
return await act_mod.browser_act_tool.ainvoke(kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_act_disabled_when_actions_off(monkeypatch):
|
||||
_enable_actions(monkeypatch, enabled=True, actions_enabled=False)
|
||||
out = await _invoke(action="navigate", url="https://example.com")
|
||||
assert "actions_enabled is false" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_act_unknown_action(monkeypatch):
|
||||
_enable_actions(monkeypatch)
|
||||
out = await _invoke(action="frobnicate")
|
||||
assert "unknown action" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_act_navigate_requires_url(monkeypatch):
|
||||
_enable_actions(monkeypatch)
|
||||
out = await _invoke(action="navigate")
|
||||
assert "requires an exact http(s) URL" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_act_non_navigate_needs_session(monkeypatch):
|
||||
_enable_actions(monkeypatch)
|
||||
out = await _invoke(action="click", index=0)
|
||||
assert "no session_id yet" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_act_click_requires_index(monkeypatch):
|
||||
_enable_actions(monkeypatch)
|
||||
out = await _invoke(action="click", session_id="sess-1")
|
||||
assert "requires a non-negative element index" in out
|
||||
|
||||
|
||||
# --- tool success path (simulated extension) --------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_act_returns_formatted_observation(monkeypatch):
|
||||
_enable_actions(monkeypatch)
|
||||
|
||||
async def fake_extension():
|
||||
# Wait for the tool to create the pending act task, then complete it.
|
||||
for _ in range(40):
|
||||
claimed = task_queue.claim_next_browser_task(user_id="default", worker_id="ext")
|
||||
if claimed is not None:
|
||||
task_queue.complete_browser_task(
|
||||
task_id=claimed["id"],
|
||||
user_id="default",
|
||||
observation={
|
||||
"action_status": "ok",
|
||||
"url": "https://example.com/home",
|
||||
"title": "Home Page",
|
||||
"elements": [
|
||||
{"index": 0, "tag": "input", "type": "search", "label": "Search", "value": ""},
|
||||
{"index": 1, "tag": "button", "label": "Go"},
|
||||
],
|
||||
"text": "Welcome to the home page.",
|
||||
},
|
||||
)
|
||||
return
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
ext = asyncio.create_task(fake_extension())
|
||||
out = await _invoke(action="navigate", url="https://example.com")
|
||||
await ext
|
||||
|
||||
assert "Browser session:" in out
|
||||
assert "Home Page" in out
|
||||
assert "[0] input[search]" in out
|
||||
assert "[1] button" in out
|
||||
assert "Welcome to the home page." in out
|
||||
@@ -1,76 +1,76 @@
|
||||
config_version: 8
|
||||
log_level: info
|
||||
models:
|
||||
- name: deepseek-chat
|
||||
display_name: DeepSeek Chat
|
||||
# - name: deepseek-chat
|
||||
# display_name: DeepSeek Chat
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-chat
|
||||
# api_key: sk-9808fb2375534112ac275541aec14b9b
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: false
|
||||
# supports_thinking: true
|
||||
# - name: zai-org/GLM-5-FP8
|
||||
# display_name: zai-org/GLM-5-FP8
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: zai-org/GLM-5-FP8
|
||||
# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
|
||||
# base_url: https://api.us-west-2.modal.direct/v1
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: deepseek-v4-pro
|
||||
# display_name: DeepSeek V4 Pro
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-v4-pro
|
||||
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: false
|
||||
# supports_thinking: true
|
||||
# - name: minimax-m2.7
|
||||
# display_name: MiniMax M2.7
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: MiniMax-M2.7
|
||||
# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
# base_url: https://api.minimaxi.com
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: minimax-m2.5
|
||||
# display_name: MiniMax M2.5
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: MiniMax-M2.5
|
||||
# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
# base_url: https://api.minimaxi.com
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: deepseek-ai/DeepSeek-OCR
|
||||
# display_name: deepseek-ai/DeepSeek-OCR
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-ai/DeepSeek-OCR
|
||||
# api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
|
||||
# base_url: https://api.siliconflow.cn/v1/
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
- name: deepseek-ai/DeepSeek-V4-Flash
|
||||
display_name: deepseek-ai/DeepSeek-V4-Flash
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-chat
|
||||
api_key: sk-9808fb2375534112ac275541aec14b9b
|
||||
base_url: https://api.deepseek.com
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: false
|
||||
supports_thinking: true
|
||||
- name: zai-org/GLM-5-FP8
|
||||
display_name: zai-org/GLM-5-FP8
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: zai-org/GLM-5-FP8
|
||||
api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
|
||||
base_url: https://api.us-west-2.modal.direct/v1
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: deepseek-v4-pro
|
||||
display_name: DeepSeek V4 Pro
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-v4-pro
|
||||
api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
base_url: https://api.deepseek.com
|
||||
request_timeout: 600.0
|
||||
max_retries: 2
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: false
|
||||
supports_thinking: true
|
||||
- name: minimax-m2.7
|
||||
display_name: MiniMax M2.7
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.7
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: minimax-m2.5
|
||||
display_name: MiniMax M2.5
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: MiniMax-M2.5
|
||||
api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
|
||||
base_url: https://api.minimaxi.com
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: deepseek-ai/DeepSeek-OCR
|
||||
display_name: deepseek-ai/DeepSeek-OCR
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-ai/DeepSeek-OCR
|
||||
api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: Qwen/Qwen3.6-27B
|
||||
display_name: Qwen/Qwen3.6-27B
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: Qwen/Qwen3.6-27B
|
||||
model: deepseek-ai/DeepSeek-V4-Flash
|
||||
api_key: sk-qwwwvjompnzxqhndctlwmmomvqdgaqqqgxvibfxchsikvgod
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
|
||||
Reference in new issue
Block a user