Merge remote-tracking branch 'origin/master' into xuhd
# Conflicts: # frontend-web/.env.production # frontend-web/.env.sit # frontend-web/package.json # offline-backend-20260512/backend/config.yaml
This commit is contained in:
commit
1668c1da87
177 files changed
+21030
-2738
No files matched your search
+225
@@ -0,0 +1,225 @@
|
||||
"""Interactive browser action tool backed by the user's Chrome extension.
|
||||
|
||||
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
|
||||
*observe -> act* loop on a live page the extension keeps open:
|
||||
|
||||
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
|
||||
2. The extension performs the action on the session tab, then re-scans the page
|
||||
and returns an *observation*: the final URL/title, an indexed list of
|
||||
interactive elements, and the visible page text.
|
||||
3. The model reads the observation, picks the next element by its index, and
|
||||
calls ``browser_act`` again with the same ``session_id``.
|
||||
|
||||
The element ``index`` values are only valid for the most recent observation of a
|
||||
session; always act on indices from the latest result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
|
||||
_NEEDS_INDEX = {"click", "type"}
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
|
||||
if not elements:
|
||||
return "(no interactive elements detected)"
|
||||
lines: list[str] = []
|
||||
for el in elements:
|
||||
idx = el.get("index")
|
||||
tag = str(el.get("tag") or "?")
|
||||
el_type = str(el.get("type") or "").strip()
|
||||
label = str(el.get("label") or "").strip().replace("\n", " ")
|
||||
if len(label) > 120:
|
||||
label = label[:120] + "…"
|
||||
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
|
||||
extra = ""
|
||||
if el.get("value"):
|
||||
value = str(el["value"]).strip().replace("\n", " ")
|
||||
extra = f' value="{value[:60]}"'
|
||||
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
|
||||
observation = observation or {}
|
||||
status = str(observation.get("action_status") or "ok")
|
||||
message = str(observation.get("message") or "").strip()
|
||||
url = str(observation.get("url") or "").strip() or "(unknown)"
|
||||
title = str(observation.get("title") or "").strip() or "(untitled)"
|
||||
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
|
||||
text = str(observation.get("text") or "").strip()
|
||||
text_limit = max(500, int(settings.observation_text_max_chars))
|
||||
text_truncated = len(text) > text_limit
|
||||
if text_truncated:
|
||||
text = text[:text_limit]
|
||||
|
||||
parts = [
|
||||
f"Browser session: {session_id}",
|
||||
f"Last action: {action} -> {status}",
|
||||
]
|
||||
if message:
|
||||
parts.append(f"Note: {message}")
|
||||
parts.extend(
|
||||
[
|
||||
f"URL: {url}",
|
||||
f"Title: {title}",
|
||||
"",
|
||||
"Interactive elements (use the [index] with click/type):",
|
||||
_format_elements(elements),
|
||||
"",
|
||||
"Page text:",
|
||||
text or "(empty page text)",
|
||||
]
|
||||
)
|
||||
if text_truncated:
|
||||
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
|
||||
parts.append(
|
||||
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
|
||||
"When the task is finished, call browser_act with action=\"close\" to release the tab."
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@tool("browser_act", parse_docstring=True)
|
||||
async def browser_act_tool(
|
||||
action: str,
|
||||
session_id: str = "",
|
||||
url: str = "",
|
||||
index: int = -1,
|
||||
text: str = "",
|
||||
submit: bool = False,
|
||||
wait_for: str = "",
|
||||
wait_seconds: int = 0,
|
||||
) -> str:
|
||||
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
|
||||
|
||||
Use this to operate a real web page on the user's behalf — log in, fill forms,
|
||||
click through navigation, and read pages that need the user's session or local
|
||||
network. Start every task with ``action="navigate"`` and a URL to open a tab
|
||||
and get the first observation, then keep calling this tool with the same
|
||||
``session_id``, choosing elements by the ``index`` shown in the observation.
|
||||
Call ``action="close"`` when done to free the tab.
|
||||
|
||||
Args:
|
||||
action: One of navigate, click, type, scroll, wait, extract, observe, close.
|
||||
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
|
||||
url: Target http(s) URL. Required for navigate (and to start a new session).
|
||||
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
|
||||
text: Text to enter for the type action.
|
||||
submit: For type, press Enter / submit the form after typing.
|
||||
wait_for: For wait, a CSS selector to wait for before observing.
|
||||
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
|
||||
if not settings.actions_enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
|
||||
|
||||
action = (action or "").strip().lower()
|
||||
if action not in _ACTIONS:
|
||||
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
|
||||
|
||||
session_id = (session_id or "").strip()
|
||||
safe_url = _validate_url(url) if url else None
|
||||
|
||||
if action == "navigate" and safe_url is None:
|
||||
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
if not session_id:
|
||||
if action != "navigate":
|
||||
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
|
||||
session_id = uuid4().hex
|
||||
|
||||
if action in _NEEDS_INDEX and index < 0:
|
||||
return f"browser_act {action} requires a non-negative element index from the latest observation."
|
||||
if action == "type" and not text:
|
||||
return "browser_act type requires the text to enter."
|
||||
|
||||
action_payload: dict[str, Any] = {"type": action}
|
||||
if safe_url is not None:
|
||||
action_payload["url"] = safe_url
|
||||
if index >= 0:
|
||||
action_payload["index"] = int(index)
|
||||
if text:
|
||||
action_payload["text"] = text
|
||||
if submit:
|
||||
action_payload["submit"] = True
|
||||
if wait_for.strip():
|
||||
action_payload["wait_for"] = wait_for.strip()[:300]
|
||||
|
||||
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url or "",
|
||||
instruction="",
|
||||
kind="act",
|
||||
session_id=session_id,
|
||||
action=action_payload,
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_act_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
|
||||
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
if action == "close":
|
||||
return f"browser_act session {session_id}: tab closed."
|
||||
|
||||
result = completed.get("result") or {}
|
||||
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
|
||||
return _format_observation(action, observation, session_id, settings)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""Browser extension backed page fetch tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
@tool("browser_fetch_page", parse_docstring=True)
|
||||
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
|
||||
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
|
||||
|
||||
Use this when the user explicitly asks to use the browser extension or asks
|
||||
for data from a page that may require their browser session, login state, or
|
||||
local network access. The URL must be an exact http(s) URL provided by the
|
||||
user. If the user did not provide a URL, ask for the URL instead of guessing.
|
||||
|
||||
Args:
|
||||
url: Exact http(s) URL to open in the user's Chrome browser.
|
||||
question: Optional instruction describing what data the user wants from the page.
|
||||
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
|
||||
|
||||
safe_url = _validate_url(url)
|
||||
if safe_url is None:
|
||||
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url,
|
||||
instruction=(question or "").strip(),
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_fetch_page_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
|
||||
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
result = completed.get("result") or {}
|
||||
content = str(result.get("content") or "").strip()
|
||||
tool_limit = max(1_000, int(settings.tool_result_max_chars))
|
||||
tool_truncated = len(content) > tool_limit
|
||||
if tool_truncated:
|
||||
content = content[:tool_limit]
|
||||
|
||||
notes: list[str] = []
|
||||
if result.get("truncated"):
|
||||
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
|
||||
if tool_truncated:
|
||||
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
|
||||
|
||||
parts = [
|
||||
"Browser page fetched through the user's Chrome extension.",
|
||||
f"Task ID: {task['id']}",
|
||||
f"Title: {result.get('title') or '(untitled)'}",
|
||||
f"URL: {result.get('url') or safe_url}",
|
||||
]
|
||||
if question:
|
||||
parts.extend(["", "Requested extraction:", question.strip()])
|
||||
if notes:
|
||||
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
|
||||
parts.extend(["", "Page content:", content or "(empty page text)"])
|
||||
return "\n".join(parts)
|
||||
|
||||
Reference in new issue
Block a user