Merge remote-tracking branch 'origin/master' into xuhd

# Conflicts:
#	frontend-web/.env.production
#	frontend-web/.env.sit
#	frontend-web/package.json
#	offline-backend-20260512/backend/config.yaml
This commit is contained in:
xuhd committed 2026-06-11 14:19:01 +08:00
commit 1668c1da87
177 files changed
+21030 -2738

No files matched your search

@@ -0,0 +1,225 @@
"""Interactive browser action tool backed by the user's Chrome extension.
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
*observe -> act* loop on a live page the extension keeps open:
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
2. The extension performs the action on the session tab, then re-scans the page
and returns an *observation*: the final URL/title, an indexed list of
interactive elements, and the visible page text.
3. The model reads the observation, picks the next element by its index, and
calls ``browser_act`` again with the same ``session_id``.
The element ``index`` values are only valid for the most recent observation of a
session; always act on indices from the latest result.
"""
from __future__ import annotations
from typing import Any
from urllib.parse import urlparse
from uuid import uuid4
from langchain.tools import tool
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
from deerflow.config import get_app_config
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime.user_context import get_effective_user_id
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
_NEEDS_INDEX = {"click", "type"}
def _settings() -> BrowserContextConfig:
value = getattr(get_app_config(), "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _validate_url(raw_url: str) -> str | None:
url = (raw_url or "").strip()
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
return url
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
if not elements:
return "(no interactive elements detected)"
lines: list[str] = []
for el in elements:
idx = el.get("index")
tag = str(el.get("tag") or "?")
el_type = str(el.get("type") or "").strip()
label = str(el.get("label") or "").strip().replace("\n", " ")
if len(label) > 120:
label = label[:120] + "…"
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
extra = ""
if el.get("value"):
value = str(el["value"]).strip().replace("\n", " ")
extra = f' value="{value[:60]}"'
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
return "\n".join(lines)
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
observation = observation or {}
status = str(observation.get("action_status") or "ok")
message = str(observation.get("message") or "").strip()
url = str(observation.get("url") or "").strip() or "(unknown)"
title = str(observation.get("title") or "").strip() or "(untitled)"
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
text = str(observation.get("text") or "").strip()
text_limit = max(500, int(settings.observation_text_max_chars))
text_truncated = len(text) > text_limit
if text_truncated:
text = text[:text_limit]
parts = [
f"Browser session: {session_id}",
f"Last action: {action} -> {status}",
]
if message:
parts.append(f"Note: {message}")
parts.extend(
[
f"URL: {url}",
f"Title: {title}",
"",
"Interactive elements (use the [index] with click/type):",
_format_elements(elements),
"",
"Page text:",
text or "(empty page text)",
]
)
if text_truncated:
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
parts.append(
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
"When the task is finished, call browser_act with action=\"close\" to release the tab."
)
return "\n".join(parts)
@tool("browser_act", parse_docstring=True)
async def browser_act_tool(
action: str,
session_id: str = "",
url: str = "",
index: int = -1,
text: str = "",
submit: bool = False,
wait_for: str = "",
wait_seconds: int = 0,
) -> str:
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
Use this to operate a real web page on the user's behalf — log in, fill forms,
click through navigation, and read pages that need the user's session or local
network. Start every task with ``action="navigate"`` and a URL to open a tab
and get the first observation, then keep calling this tool with the same
``session_id``, choosing elements by the ``index`` shown in the observation.
Call ``action="close"`` when done to free the tab.
Args:
action: One of navigate, click, type, scroll, wait, extract, observe, close.
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
url: Target http(s) URL. Required for navigate (and to start a new session).
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
text: Text to enter for the type action.
submit: For type, press Enter / submit the form after typing.
wait_for: For wait, a CSS selector to wait for before observing.
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
"""
settings = _settings()
if not settings.enabled:
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
if not settings.actions_enabled:
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
action = (action or "").strip().lower()
if action not in _ACTIONS:
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
session_id = (session_id or "").strip()
safe_url = _validate_url(url) if url else None
if action == "navigate" and safe_url is None:
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
if not session_id:
if action != "navigate":
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
session_id = uuid4().hex
if action in _NEEDS_INDEX and index < 0:
return f"browser_act {action} requires a non-negative element index from the latest observation."
if action == "type" and not text:
return "browser_act type requires the text to enter."
action_payload: dict[str, Any] = {"type": action}
if safe_url is not None:
action_payload["url"] = safe_url
if index >= 0:
action_payload["index"] = int(index)
if text:
action_payload["text"] = text
if submit:
action_payload["submit"] = True
if wait_for.strip():
action_payload["wait_for"] = wait_for.strip()[:300]
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
requested_timeout = int(wait_seconds or configured_timeout)
timeout = min(max(5, requested_timeout), configured_timeout)
user_id = get_effective_user_id()
task = create_browser_task(
user_id=user_id,
url=safe_url or "",
instruction="",
kind="act",
session_id=session_id,
action=action_payload,
timeout_seconds=timeout + int(settings.task_lease_seconds),
max_content_chars=settings.max_content_chars,
metadata={"source": "browser_act_tool"},
)
completed = await wait_for_browser_task_result(
task_id=task["id"],
user_id=user_id,
timeout_seconds=timeout,
poll_interval_seconds=settings.task_poll_interval_seconds,
)
if completed is None:
return f"Browser task {task['id']} was not found after it was created."
status = completed.get("status")
if status == "error":
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
if status == "expired":
return (
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
)
if status != "done":
return (
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
)
if action == "close":
return f"browser_act session {session_id}: tab closed."
result = completed.get("result") or {}
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
return _format_observation(action, observation, session_id, settings)
@@ -0,0 +1,117 @@
"""Browser extension backed page fetch tool."""
from __future__ import annotations
from urllib.parse import urlparse
from langchain.tools import tool
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
from deerflow.config import get_app_config
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime.user_context import get_effective_user_id
def _settings() -> BrowserContextConfig:
value = getattr(get_app_config(), "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _validate_url(raw_url: str) -> str | None:
url = (raw_url or "").strip()
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
return url
@tool("browser_fetch_page", parse_docstring=True)
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
Use this when the user explicitly asks to use the browser extension or asks
for data from a page that may require their browser session, login state, or
local network access. The URL must be an exact http(s) URL provided by the
user. If the user did not provide a URL, ask for the URL instead of guessing.
Args:
url: Exact http(s) URL to open in the user's Chrome browser.
question: Optional instruction describing what data the user wants from the page.
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
"""
settings = _settings()
if not settings.enabled:
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
safe_url = _validate_url(url)
if safe_url is None:
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
requested_timeout = int(wait_seconds or configured_timeout)
timeout = min(max(5, requested_timeout), configured_timeout)
user_id = get_effective_user_id()
task = create_browser_task(
user_id=user_id,
url=safe_url,
instruction=(question or "").strip(),
timeout_seconds=timeout + int(settings.task_lease_seconds),
max_content_chars=settings.max_content_chars,
metadata={"source": "browser_fetch_page_tool"},
)
completed = await wait_for_browser_task_result(
task_id=task["id"],
user_id=user_id,
timeout_seconds=timeout,
poll_interval_seconds=settings.task_poll_interval_seconds,
)
if completed is None:
return f"Browser task {task['id']} was not found after it was created."
status = completed.get("status")
if status == "error":
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
if status == "expired":
return (
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
)
if status != "done":
return (
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
)
result = completed.get("result") or {}
content = str(result.get("content") or "").strip()
tool_limit = max(1_000, int(settings.tool_result_max_chars))
tool_truncated = len(content) > tool_limit
if tool_truncated:
content = content[:tool_limit]
notes: list[str] = []
if result.get("truncated"):
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
if tool_truncated:
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
parts = [
"Browser page fetched through the user's Chrome extension.",
f"Task ID: {task['id']}",
f"Title: {result.get('title') or '(untitled)'}",
f"URL: {result.get('url') or safe_url}",
]
if question:
parts.extend(["", "Requested extraction:", question.strip()])
if notes:
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
parts.extend(["", "Page content:", content or "(empty page text)"])
return "\n".join(parts)