feat: 浏览器扩展支持交互式页面操作 (browser_act)
在原有只读抓取 (browser_fetch_page) 基础上,新增 observe→act 循环, 让 Agent 能在用户的 Chrome 中实际操作网页:点击/输入/滚动/等待/页内跳转。 后端: - 新工具 browser_act,按 session 复用标签页,返回带编号的可交互元素清单+正文 - browser_context 任务队列扩展 kind/session_id/action 与 observation 结果 - /api/browser-context 路由下发 act 任务、接收 observation - BrowserContextConfig 增加 actions_enabled 等开关,tools.py 两级门控 - 接线 app.py 路由挂载、app_config.py 配置字段、config.yaml 工具注册 前端: - ChatPage 处理「发送当前页」回跳的 browser_context query 参数并自动提交 扩展 (browser-extension/deer-web-clipper): - shared.js 自包含注入函数 deerObservePage / deerPerformAction - background.js 按 kind 分发任务,session→tab 映射存 chrome.storage.session 测试:tests/test_browser_act.py(队列往返 + 工具校验 + 模拟扩展成功路径,8 passed) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0001f86826
commit
d4a71ce384
24 files changed
+4499
-162
No files matched your search
@@ -0,0 +1,2 @@
|
||||
"""Browser-backed page context helpers."""
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Small file-backed task queue for browser page fetch requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LOCK = threading.RLock()
|
||||
_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$")
|
||||
_TERMINAL_STATUSES = {"done", "error", "expired"}
|
||||
|
||||
|
||||
def tasks_dir() -> Path:
|
||||
"""Return the browser-context task directory, creating it if needed."""
|
||||
|
||||
path = runtime_home() / "browser-context" / "tasks"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None = None) -> str:
|
||||
return (value or _now()).isoformat()
|
||||
|
||||
|
||||
def _parse_iso(value: Any) -> datetime | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _task_path(task_id: str) -> Path | None:
|
||||
if not _TASK_ID_RE.fullmatch(task_id or ""):
|
||||
return None
|
||||
return tasks_dir() / f"{task_id}.json"
|
||||
|
||||
|
||||
def _read_task(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except Exception:
|
||||
logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True)
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _write_task(path: Path, data: dict[str, Any]) -> None:
|
||||
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
|
||||
with tmp.open("w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, separators=(",", ":"))
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _public_task(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": data.get("id"),
|
||||
"kind": data.get("kind") or "fetch",
|
||||
"url": data.get("url") or "",
|
||||
"instruction": data.get("instruction") or "",
|
||||
"session_id": data.get("session_id") or "",
|
||||
"action": data.get("action") or None,
|
||||
"created_at": data.get("created_at") or "",
|
||||
"updated_at": data.get("updated_at") or "",
|
||||
"max_content_chars": int(data.get("max_content_chars") or 200_000),
|
||||
}
|
||||
|
||||
|
||||
def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]:
|
||||
if data.get("status") in _TERMINAL_STATUSES:
|
||||
return data
|
||||
expires_at = _parse_iso(data.get("expires_at"))
|
||||
if expires_at is not None and now >= expires_at:
|
||||
data = dict(data)
|
||||
data["status"] = "expired"
|
||||
data["error"] = "Browser task expired before the extension returned a result."
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return data
|
||||
|
||||
|
||||
def prune_browser_tasks(max_age_seconds: int = 86_400) -> int:
|
||||
"""Delete old terminal tasks and return the number removed."""
|
||||
|
||||
cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds)))
|
||||
removed = 0
|
||||
with _LOCK:
|
||||
for path in tasks_dir().glob("*.json"):
|
||||
data = _read_task(path)
|
||||
if not data or data.get("status") not in _TERMINAL_STATUSES:
|
||||
continue
|
||||
updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at"))
|
||||
if updated_at is None or updated_at < cutoff:
|
||||
try:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
logger.debug("Failed to delete old browser task: %s", path, exc_info=True)
|
||||
return removed
|
||||
|
||||
|
||||
def create_browser_task(
|
||||
*,
|
||||
user_id: str,
|
||||
url: str,
|
||||
instruction: str = "",
|
||||
timeout_seconds: int = 60,
|
||||
max_content_chars: int = 200_000,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
kind: str = "fetch",
|
||||
session_id: str = "",
|
||||
action: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a browser task for the current user's extension to pick up.
|
||||
|
||||
``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or
|
||||
``"act"`` for an interactive observe/act step driven by ``action`` within a
|
||||
long-lived ``session_id`` (a tab kept open by the extension).
|
||||
"""
|
||||
|
||||
now = _now()
|
||||
task_id = uuid4().hex
|
||||
task = {
|
||||
"id": task_id,
|
||||
"user_id": str(user_id or "default"),
|
||||
"status": "pending",
|
||||
"kind": kind if kind in {"fetch", "act"} else "fetch",
|
||||
"url": url,
|
||||
"instruction": instruction or "",
|
||||
"session_id": str(session_id or ""),
|
||||
"action": action or None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))),
|
||||
"max_content_chars": max(1_000, int(max_content_chars)),
|
||||
"worker_id": None,
|
||||
"claimed_at": None,
|
||||
"result": None,
|
||||
"error": None,
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
path = tasks_dir() / f"{task_id}.json"
|
||||
with _LOCK:
|
||||
_write_task(path, task)
|
||||
return task
|
||||
|
||||
|
||||
def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
"""Read a task if it belongs to the requested user."""
|
||||
|
||||
path = _task_path(task_id)
|
||||
if path is None:
|
||||
return None
|
||||
with _LOCK:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
|
||||
return None
|
||||
return _expire_if_needed(path, data, _now())
|
||||
|
||||
|
||||
def claim_next_browser_task(
|
||||
*,
|
||||
user_id: str,
|
||||
worker_id: str,
|
||||
lease_seconds: int = 120,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Claim the oldest pending task for a user."""
|
||||
|
||||
now = _now()
|
||||
user_id = str(user_id or "default")
|
||||
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
|
||||
lease_seconds = max(10, int(lease_seconds))
|
||||
|
||||
with _LOCK:
|
||||
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||
for path in paths:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != user_id:
|
||||
continue
|
||||
|
||||
data = _expire_if_needed(path, data, now)
|
||||
status = data.get("status")
|
||||
if status == "running":
|
||||
updated_at = _parse_iso(data.get("updated_at")) or now
|
||||
if (now - updated_at).total_seconds() <= lease_seconds:
|
||||
continue
|
||||
status = "pending"
|
||||
data = dict(data)
|
||||
data["status"] = status
|
||||
|
||||
if status != "pending":
|
||||
continue
|
||||
|
||||
data = dict(data)
|
||||
data["status"] = "running"
|
||||
data["worker_id"] = worker_id
|
||||
data["claimed_at"] = _iso(now)
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return _public_task(data)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def complete_browser_task(
|
||||
*,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
title: str = "",
|
||||
url: str = "",
|
||||
content: str = "",
|
||||
content_type: str = "markdown",
|
||||
error: str = "",
|
||||
metadata: dict[str, Any] | None = None,
|
||||
observation: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Mark a browser task complete or failed.
|
||||
|
||||
For ``act`` tasks the extension returns a structured ``observation`` (final
|
||||
URL/title, interactive element list, page text) instead of a captured
|
||||
markdown document; it is stored alongside the result verbatim.
|
||||
"""
|
||||
|
||||
path = _task_path(task_id)
|
||||
if path is None:
|
||||
return None
|
||||
|
||||
now = _now()
|
||||
with _LOCK:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
|
||||
return None
|
||||
|
||||
data = dict(data)
|
||||
if error:
|
||||
data["status"] = "error"
|
||||
data["error"] = str(error).strip()[:4000]
|
||||
data["result"] = None
|
||||
else:
|
||||
raw_content = str(content or "").strip()
|
||||
max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000))
|
||||
truncated = len(raw_content) > max_content_chars
|
||||
data["status"] = "done"
|
||||
data["error"] = None
|
||||
data["result"] = {
|
||||
"title": str(title or "").strip()[:500],
|
||||
"url": str(url or data.get("url") or "").strip()[:4096],
|
||||
"content": raw_content[:max_content_chars] if truncated else raw_content,
|
||||
"content_type": content_type if content_type in {"markdown", "text"} else "markdown",
|
||||
"truncated": truncated,
|
||||
"observation": observation if isinstance(observation, dict) else None,
|
||||
"captured_at": _iso(now),
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return data
|
||||
|
||||
|
||||
async def wait_for_browser_task_result(
|
||||
*,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
timeout_seconds: int,
|
||||
poll_interval_seconds: float = 1.0,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Wait for a browser task to reach a terminal state."""
|
||||
|
||||
deadline = time.monotonic() + max(1, int(timeout_seconds))
|
||||
interval = max(0.25, float(poll_interval_seconds))
|
||||
while True:
|
||||
task = get_browser_task(task_id=task_id, user_id=user_id)
|
||||
if task is None:
|
||||
return None
|
||||
if task.get("status") in _TERMINAL_STATUSES:
|
||||
return task
|
||||
if time.monotonic() >= deadline:
|
||||
return task
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
@@ -9,8 +9,9 @@ from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.gateway_config import GatewayConfig
|
||||
@@ -132,7 +133,8 @@ class AppConfig(BaseModel):
|
||||
gateway: GatewayConfig = Field(default_factory=GatewayConfig, description="Gateway FastAPI listening port (used by intra-process loopback)")
|
||||
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
|
||||
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
|
||||
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
|
||||
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
|
||||
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
|
||||
|
||||
@classmethod
|
||||
def resolve_config_path(cls, config_path: str | None = None) -> Path:
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Configuration for browser page context capture."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class BrowserContextConfig(BaseModel):
|
||||
"""Controls the optional Chrome-extension page capture integration."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable /api/browser-context endpoints for Chrome extension page capture.",
|
||||
)
|
||||
max_content_chars: int = Field(
|
||||
default=200_000,
|
||||
ge=1_000,
|
||||
description="Maximum captured page text stored per request. Longer content is truncated.",
|
||||
)
|
||||
max_title_chars: int = Field(
|
||||
default=300,
|
||||
ge=20,
|
||||
description="Maximum title length accepted from the browser extension.",
|
||||
)
|
||||
max_url_chars: int = Field(
|
||||
default=2_048,
|
||||
ge=100,
|
||||
description="Maximum source URL length accepted from the browser extension.",
|
||||
)
|
||||
filename_prefix: str = Field(
|
||||
default="webpage",
|
||||
description="Prefix used for Markdown files created from captured pages.",
|
||||
)
|
||||
route_base: str = Field(
|
||||
default="/page/workspace",
|
||||
description="Frontend route base used when returning chat paths to the extension.",
|
||||
)
|
||||
task_result_timeout_seconds: int = Field(
|
||||
default=60,
|
||||
ge=5,
|
||||
description="Maximum time the browser_fetch_page tool waits for the extension result.",
|
||||
)
|
||||
task_poll_interval_seconds: float = Field(
|
||||
default=1.0,
|
||||
ge=0.25,
|
||||
description="How often the backend tool checks for a returned browser task result.",
|
||||
)
|
||||
task_lease_seconds: int = Field(
|
||||
default=120,
|
||||
ge=10,
|
||||
description="How long a claimed browser task is reserved before another extension may retry it.",
|
||||
)
|
||||
tool_result_max_chars: int = Field(
|
||||
default=60_000,
|
||||
ge=1_000,
|
||||
description="Maximum page characters returned from browser_fetch_page into the model context.",
|
||||
)
|
||||
actions_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).",
|
||||
)
|
||||
action_result_timeout_seconds: int = Field(
|
||||
default=45,
|
||||
ge=5,
|
||||
description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.",
|
||||
)
|
||||
max_observed_elements: int = Field(
|
||||
default=150,
|
||||
ge=10,
|
||||
description="Maximum number of interactive elements returned in a browser_act observation.",
|
||||
)
|
||||
observation_text_max_chars: int = Field(
|
||||
default=12_000,
|
||||
ge=500,
|
||||
description="Maximum page text characters returned alongside a browser_act observation.",
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""Interactive browser action tool backed by the user's Chrome extension.
|
||||
|
||||
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
|
||||
*observe -> act* loop on a live page the extension keeps open:
|
||||
|
||||
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
|
||||
2. The extension performs the action on the session tab, then re-scans the page
|
||||
and returns an *observation*: the final URL/title, an indexed list of
|
||||
interactive elements, and the visible page text.
|
||||
3. The model reads the observation, picks the next element by its index, and
|
||||
calls ``browser_act`` again with the same ``session_id``.
|
||||
|
||||
The element ``index`` values are only valid for the most recent observation of a
|
||||
session; always act on indices from the latest result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
|
||||
_NEEDS_INDEX = {"click", "type"}
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
|
||||
if not elements:
|
||||
return "(no interactive elements detected)"
|
||||
lines: list[str] = []
|
||||
for el in elements:
|
||||
idx = el.get("index")
|
||||
tag = str(el.get("tag") or "?")
|
||||
el_type = str(el.get("type") or "").strip()
|
||||
label = str(el.get("label") or "").strip().replace("\n", " ")
|
||||
if len(label) > 120:
|
||||
label = label[:120] + "…"
|
||||
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
|
||||
extra = ""
|
||||
if el.get("value"):
|
||||
value = str(el["value"]).strip().replace("\n", " ")
|
||||
extra = f' value="{value[:60]}"'
|
||||
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
|
||||
observation = observation or {}
|
||||
status = str(observation.get("action_status") or "ok")
|
||||
message = str(observation.get("message") or "").strip()
|
||||
url = str(observation.get("url") or "").strip() or "(unknown)"
|
||||
title = str(observation.get("title") or "").strip() or "(untitled)"
|
||||
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
|
||||
text = str(observation.get("text") or "").strip()
|
||||
text_limit = max(500, int(settings.observation_text_max_chars))
|
||||
text_truncated = len(text) > text_limit
|
||||
if text_truncated:
|
||||
text = text[:text_limit]
|
||||
|
||||
parts = [
|
||||
f"Browser session: {session_id}",
|
||||
f"Last action: {action} -> {status}",
|
||||
]
|
||||
if message:
|
||||
parts.append(f"Note: {message}")
|
||||
parts.extend(
|
||||
[
|
||||
f"URL: {url}",
|
||||
f"Title: {title}",
|
||||
"",
|
||||
"Interactive elements (use the [index] with click/type):",
|
||||
_format_elements(elements),
|
||||
"",
|
||||
"Page text:",
|
||||
text or "(empty page text)",
|
||||
]
|
||||
)
|
||||
if text_truncated:
|
||||
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
|
||||
parts.append(
|
||||
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
|
||||
"When the task is finished, call browser_act with action=\"close\" to release the tab."
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@tool("browser_act", parse_docstring=True)
|
||||
async def browser_act_tool(
|
||||
action: str,
|
||||
session_id: str = "",
|
||||
url: str = "",
|
||||
index: int = -1,
|
||||
text: str = "",
|
||||
submit: bool = False,
|
||||
wait_for: str = "",
|
||||
wait_seconds: int = 0,
|
||||
) -> str:
|
||||
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
|
||||
|
||||
Use this to operate a real web page on the user's behalf — log in, fill forms,
|
||||
click through navigation, and read pages that need the user's session or local
|
||||
network. Start every task with ``action="navigate"`` and a URL to open a tab
|
||||
and get the first observation, then keep calling this tool with the same
|
||||
``session_id``, choosing elements by the ``index`` shown in the observation.
|
||||
Call ``action="close"`` when done to free the tab.
|
||||
|
||||
Args:
|
||||
action: One of navigate, click, type, scroll, wait, extract, observe, close.
|
||||
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
|
||||
url: Target http(s) URL. Required for navigate (and to start a new session).
|
||||
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
|
||||
text: Text to enter for the type action.
|
||||
submit: For type, press Enter / submit the form after typing.
|
||||
wait_for: For wait, a CSS selector to wait for before observing.
|
||||
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
|
||||
if not settings.actions_enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
|
||||
|
||||
action = (action or "").strip().lower()
|
||||
if action not in _ACTIONS:
|
||||
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
|
||||
|
||||
session_id = (session_id or "").strip()
|
||||
safe_url = _validate_url(url) if url else None
|
||||
|
||||
if action == "navigate" and safe_url is None:
|
||||
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
if not session_id:
|
||||
if action != "navigate":
|
||||
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
|
||||
session_id = uuid4().hex
|
||||
|
||||
if action in _NEEDS_INDEX and index < 0:
|
||||
return f"browser_act {action} requires a non-negative element index from the latest observation."
|
||||
if action == "type" and not text:
|
||||
return "browser_act type requires the text to enter."
|
||||
|
||||
action_payload: dict[str, Any] = {"type": action}
|
||||
if safe_url is not None:
|
||||
action_payload["url"] = safe_url
|
||||
if index >= 0:
|
||||
action_payload["index"] = int(index)
|
||||
if text:
|
||||
action_payload["text"] = text
|
||||
if submit:
|
||||
action_payload["submit"] = True
|
||||
if wait_for.strip():
|
||||
action_payload["wait_for"] = wait_for.strip()[:300]
|
||||
|
||||
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url or "",
|
||||
instruction="",
|
||||
kind="act",
|
||||
session_id=session_id,
|
||||
action=action_payload,
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_act_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
|
||||
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
if action == "close":
|
||||
return f"browser_act session {session_id}: tab closed."
|
||||
|
||||
result = completed.get("result") or {}
|
||||
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
|
||||
return _format_observation(action, observation, session_id, settings)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""Browser extension backed page fetch tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
@tool("browser_fetch_page", parse_docstring=True)
|
||||
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
|
||||
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
|
||||
|
||||
Use this when the user explicitly asks to use the browser extension or asks
|
||||
for data from a page that may require their browser session, login state, or
|
||||
local network access. The URL must be an exact http(s) URL provided by the
|
||||
user. If the user did not provide a URL, ask for the URL instead of guessing.
|
||||
|
||||
Args:
|
||||
url: Exact http(s) URL to open in the user's Chrome browser.
|
||||
question: Optional instruction describing what data the user wants from the page.
|
||||
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
|
||||
|
||||
safe_url = _validate_url(url)
|
||||
if safe_url is None:
|
||||
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url,
|
||||
instruction=(question or "").strip(),
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_fetch_page_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
|
||||
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
result = completed.get("result") or {}
|
||||
content = str(result.get("content") or "").strip()
|
||||
tool_limit = max(1_000, int(settings.tool_result_max_chars))
|
||||
tool_truncated = len(content) > tool_limit
|
||||
if tool_truncated:
|
||||
content = content[:tool_limit]
|
||||
|
||||
notes: list[str] = []
|
||||
if result.get("truncated"):
|
||||
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
|
||||
if tool_truncated:
|
||||
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
|
||||
|
||||
parts = [
|
||||
"Browser page fetched through the user's Chrome extension.",
|
||||
f"Task ID: {task['id']}",
|
||||
f"Title: {result.get('title') or '(untitled)'}",
|
||||
f"URL: {result.get('url') or safe_url}",
|
||||
]
|
||||
if question:
|
||||
parts.extend(["", "Requested extraction:", question.strip()])
|
||||
if notes:
|
||||
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
|
||||
parts.extend(["", "Page content:", content or "(empty page text)"])
|
||||
return "\n".join(parts)
|
||||
|
||||
@@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool"
|
||||
BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool"
|
||||
BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE}
|
||||
BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"}
|
||||
|
||||
BUILTIN_TOOLS = [
|
||||
present_file_tool,
|
||||
ask_clarification_tool,
|
||||
@@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _is_browser_context_tool(tool: object) -> bool:
|
||||
"""Return True if the tool config represents the browser extension bridge."""
|
||||
|
||||
use = getattr(tool, "use", None)
|
||||
name = getattr(tool, "name", None)
|
||||
return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
|
||||
|
||||
|
||||
def get_available_tools(
|
||||
groups: list[str] | None = None,
|
||||
include_mcp: bool = True,
|
||||
@@ -64,6 +77,18 @@ def get_available_tools(
|
||||
config = app_config or get_app_config()
|
||||
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
|
||||
|
||||
browser_context_config = getattr(config, "browser_context", None)
|
||||
if not getattr(browser_context_config, "enabled", False):
|
||||
tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
|
||||
elif not getattr(browser_context_config, "actions_enabled", True):
|
||||
# browser_context is on, but interactive page actions are disabled — keep
|
||||
# the read-only browser_fetch_page tool, drop only browser_act.
|
||||
tool_configs = [
|
||||
tool
|
||||
for tool in tool_configs
|
||||
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
|
||||
]
|
||||
|
||||
# Do not expose host bash by default when LocalSandboxProvider is active.
|
||||
if not is_host_bash_allowed(config):
|
||||
tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)]
|
||||
|
||||
Reference in new issue
Block a user