feat: 浏览器扩展支持交互式页面操作 (browser_act)
在原有只读抓取 (browser_fetch_page) 基础上,新增 observe→act 循环, 让 Agent 能在用户的 Chrome 中实际操作网页:点击/输入/滚动/等待/页内跳转。 后端: - 新工具 browser_act,按 session 复用标签页,返回带编号的可交互元素清单+正文 - browser_context 任务队列扩展 kind/session_id/action 与 observation 结果 - /api/browser-context 路由下发 act 任务、接收 observation - BrowserContextConfig 增加 actions_enabled 等开关,tools.py 两级门控 - 接线 app.py 路由挂载、app_config.py 配置字段、config.yaml 工具注册 前端: - ChatPage 处理「发送当前页」回跳的 browser_context query 参数并自动提交 扩展 (browser-extension/deer-web-clipper): - shared.js 自包含注入函数 deerObservePage / deerPerformAction - background.js 按 kind 分发任务,session→tab 映射存 chrome.storage.session 测试:tests/test_browser_act.py(队列往返 + 工具校验 + 模拟扩展成功路径,8 passed) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0001f86826
commit
d4a71ce384
24 files changed
+4499
-162
No files matched your search
@@ -1,6 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useRouter, useSearchParams } from "next/navigation";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
@@ -41,9 +41,47 @@ import { textOfMessage } from "@/core/threads/utils";
|
||||
import { env } from "@/env";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
const BROWSER_CONTEXT_PARAM_KEYS = [
|
||||
"browser_context",
|
||||
"bc_filename",
|
||||
"bc_size",
|
||||
"bc_path",
|
||||
"bc_prompt",
|
||||
] as const;
|
||||
|
||||
function clearBrowserContextParamsFromHash() {
|
||||
if (typeof window === "undefined") return;
|
||||
const hash = window.location.hash;
|
||||
if (!hash.startsWith("#")) return;
|
||||
const hashBody = hash.slice(1);
|
||||
const queryStart = hashBody.indexOf("?");
|
||||
if (queryStart < 0) return;
|
||||
|
||||
const path = hashBody.slice(0, queryStart);
|
||||
const search = hashBody.slice(queryStart + 1);
|
||||
const params = new URLSearchParams(search);
|
||||
let changed = false;
|
||||
for (const key of BROWSER_CONTEXT_PARAM_KEYS) {
|
||||
if (params.has(key)) {
|
||||
params.delete(key);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (!changed) return;
|
||||
|
||||
const nextSearch = params.toString();
|
||||
const nextHash = nextSearch ? `#${path}?${nextSearch}` : `#${path}`;
|
||||
window.history.replaceState(
|
||||
null,
|
||||
"",
|
||||
`${window.location.pathname}${window.location.search}${nextHash}`,
|
||||
);
|
||||
}
|
||||
|
||||
export default function ChatPage() {
|
||||
const routeBase = useRouteBase();
|
||||
const router = useRouter();
|
||||
const searchParams = useSearchParams();
|
||||
const { t } = useI18n();
|
||||
const [showFollowups, setShowFollowups] = useState(false);
|
||||
const [hasStartedConversation, setHasStartedConversation] = useState(false);
|
||||
@@ -62,6 +100,7 @@ export default function ChatPage() {
|
||||
const [selectedAgentId, setSelectedAgentId] = useState<string | null>(null);
|
||||
const [agentHandoffPending, setAgentHandoffPending] = useState(false);
|
||||
const agentHandoffPendingRef = useRef(false);
|
||||
const browserContextAutoSubmitRef = useRef<string | null>(null);
|
||||
const { agents: selectorAgents, isLoading: selectorLoading } =
|
||||
useAgentSelector();
|
||||
const { selectedArtifact } = useArtifacts();
|
||||
@@ -242,6 +281,75 @@ export default function ChatPage() {
|
||||
[sendMessage, threadId],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const token = searchParams.get("browser_context")?.trim();
|
||||
if (!token || browserContextAutoSubmitRef.current === token) {
|
||||
return;
|
||||
}
|
||||
if (!threadId || isNewThread || thread.isLoading || agentHandoffPending) {
|
||||
return;
|
||||
}
|
||||
|
||||
const consumedKey = `deerflow.browser-context.consumed.${token}`;
|
||||
if (window.sessionStorage.getItem(consumedKey)) {
|
||||
browserContextAutoSubmitRef.current = token;
|
||||
clearBrowserContextParamsFromHash();
|
||||
return;
|
||||
}
|
||||
|
||||
const filename = searchParams.get("bc_filename")?.trim();
|
||||
const path = searchParams.get("bc_path")?.trim();
|
||||
const size = Number(searchParams.get("bc_size") ?? "0");
|
||||
const prompt =
|
||||
searchParams.get("bc_prompt")?.trim() ||
|
||||
"请基于刚刚发送的网页内容,先用中文总结要点,并等待我继续提问。";
|
||||
|
||||
if (!filename || !path || !Number.isFinite(size) || size < 0) {
|
||||
browserContextAutoSubmitRef.current = token;
|
||||
clearBrowserContextParamsFromHash();
|
||||
toast.error("网页上下文参数不完整,无法自动发送到 Deer");
|
||||
return;
|
||||
}
|
||||
|
||||
browserContextAutoSubmitRef.current = token;
|
||||
window.sessionStorage.setItem(consumedKey, "1");
|
||||
clearBrowserContextParamsFromHash();
|
||||
setHasStartedConversation(true);
|
||||
|
||||
void sendMessage(
|
||||
threadId,
|
||||
{ text: prompt, files: [] },
|
||||
undefined,
|
||||
{
|
||||
additionalKwargs: {
|
||||
browser_context: {
|
||||
source: "chrome-extension",
|
||||
token,
|
||||
},
|
||||
files: [
|
||||
{
|
||||
filename,
|
||||
size,
|
||||
path,
|
||||
status: "uploaded",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
).catch((err) => {
|
||||
toast.error(
|
||||
err instanceof Error ? err.message : "网页上下文发送失败,请重试",
|
||||
);
|
||||
});
|
||||
}, [
|
||||
searchParams,
|
||||
threadId,
|
||||
isNewThread,
|
||||
thread.isLoading,
|
||||
agentHandoffPending,
|
||||
sendMessage,
|
||||
]);
|
||||
|
||||
// Persist the picked agent the moment it changes so the next new chat
|
||||
// defaults to it — even if the user never sends a message this session.
|
||||
// Passing null (the ✕ clear) wipes the stored preference.
|
||||
|
||||
Reference in new issue
Block a user