feat: 浏览器扩展支持交互式页面操作 (browser_act)

在原有只读抓取 (browser_fetch_page) 基础上,新增 observe→act 循环,
让 Agent 能在用户的 Chrome 中实际操作网页:点击/输入/滚动/等待/页内跳转。

后端:
- 新工具 browser_act,按 session 复用标签页,返回带编号的可交互元素清单+正文
- browser_context 任务队列扩展 kind/session_id/action 与 observation 结果
- /api/browser-context 路由下发 act 任务、接收 observation
- BrowserContextConfig 增加 actions_enabled 等开关,tools.py 两级门控
- 接线 app.py 路由挂载、app_config.py 配置字段、config.yaml 工具注册

前端:
- ChatPage 处理「发送当前页」回跳的 browser_context query 参数并自动提交

扩展 (browser-extension/deer-web-clipper):
- shared.js 自包含注入函数 deerObservePage / deerPerformAction
- background.js 按 kind 分发任务,session→tab 映射存 chrome.storage.session

测试:tests/test_browser_act.py(队列往返 + 工具校验 + 模拟扩展成功路径,8 passed)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-08 13:47:00 +08:00
1 parent 0001f86826
commit d4a71ce384
24 files changed
+4499 -162

No files matched your search

+109 -1
View File
@@ -1,6 +1,6 @@
"use client";
import { useRouter } from "next/navigation";
import { useRouter, useSearchParams } from "next/navigation";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
@@ -41,9 +41,47 @@ import { textOfMessage } from "@/core/threads/utils";
import { env } from "@/env";
import { cn } from "@/lib/utils";
const BROWSER_CONTEXT_PARAM_KEYS = [
"browser_context",
"bc_filename",
"bc_size",
"bc_path",
"bc_prompt",
] as const;
function clearBrowserContextParamsFromHash() {
if (typeof window === "undefined") return;
const hash = window.location.hash;
if (!hash.startsWith("#")) return;
const hashBody = hash.slice(1);
const queryStart = hashBody.indexOf("?");
if (queryStart < 0) return;
const path = hashBody.slice(0, queryStart);
const search = hashBody.slice(queryStart + 1);
const params = new URLSearchParams(search);
let changed = false;
for (const key of BROWSER_CONTEXT_PARAM_KEYS) {
if (params.has(key)) {
params.delete(key);
changed = true;
}
}
if (!changed) return;
const nextSearch = params.toString();
const nextHash = nextSearch ? `#${path}?${nextSearch}` : `#${path}`;
window.history.replaceState(
null,
"",
`${window.location.pathname}${window.location.search}${nextHash}`,
);
}
export default function ChatPage() {
const routeBase = useRouteBase();
const router = useRouter();
const searchParams = useSearchParams();
const { t } = useI18n();
const [showFollowups, setShowFollowups] = useState(false);
const [hasStartedConversation, setHasStartedConversation] = useState(false);
@@ -62,6 +100,7 @@ export default function ChatPage() {
const [selectedAgentId, setSelectedAgentId] = useState<string | null>(null);
const [agentHandoffPending, setAgentHandoffPending] = useState(false);
const agentHandoffPendingRef = useRef(false);
const browserContextAutoSubmitRef = useRef<string | null>(null);
const { agents: selectorAgents, isLoading: selectorLoading } =
useAgentSelector();
const { selectedArtifact } = useArtifacts();
@@ -242,6 +281,75 @@ export default function ChatPage() {
[sendMessage, threadId],
);
useEffect(() => {
const token = searchParams.get("browser_context")?.trim();
if (!token || browserContextAutoSubmitRef.current === token) {
return;
}
if (!threadId || isNewThread || thread.isLoading || agentHandoffPending) {
return;
}
const consumedKey = `deerflow.browser-context.consumed.${token}`;
if (window.sessionStorage.getItem(consumedKey)) {
browserContextAutoSubmitRef.current = token;
clearBrowserContextParamsFromHash();
return;
}
const filename = searchParams.get("bc_filename")?.trim();
const path = searchParams.get("bc_path")?.trim();
const size = Number(searchParams.get("bc_size") ?? "0");
const prompt =
searchParams.get("bc_prompt")?.trim() ||
"请基于刚刚发送的网页内容,先用中文总结要点,并等待我继续提问。";
if (!filename || !path || !Number.isFinite(size) || size < 0) {
browserContextAutoSubmitRef.current = token;
clearBrowserContextParamsFromHash();
toast.error("网页上下文参数不完整,无法自动发送到 Deer");
return;
}
browserContextAutoSubmitRef.current = token;
window.sessionStorage.setItem(consumedKey, "1");
clearBrowserContextParamsFromHash();
setHasStartedConversation(true);
void sendMessage(
threadId,
{ text: prompt, files: [] },
undefined,
{
additionalKwargs: {
browser_context: {
source: "chrome-extension",
token,
},
files: [
{
filename,
size,
path,
status: "uploaded",
},
],
},
},
).catch((err) => {
toast.error(
err instanceof Error ? err.message : "网页上下文发送失败,请重试",
);
});
}, [
searchParams,
threadId,
isNewThread,
thread.isLoading,
agentHandoffPending,
sendMessage,
]);
// Persist the picked agent the moment it changes so the next new chat
// defaults to it — even if the user never sends a message this session.
// Passing null (the ✕ clear) wipes the stored preference.