From d4a71ce384139c24f83ddd105b0996df78af7ab6 Mon Sep 17 00:00:00 2001 From: Administrator Date: Mon, 8 Jun 2026 13:46:35 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B5=8F=E8=A7=88=E5=99=A8=E6=89=A9?= =?UTF-8?q?=E5=B1=95=E6=94=AF=E6=8C=81=E4=BA=A4=E4=BA=92=E5=BC=8F=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=93=8D=E4=BD=9C=20(browser=5Fact)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在原有只读抓取 (browser_fetch_page) 基础上,新增 observe→act 循环, 让 Agent 能在用户的 Chrome 中实际操作网页:点击/输入/滚动/等待/页内跳转。 后端: - 新工具 browser_act,按 session 复用标签页,返回带编号的可交互元素清单+正文 - browser_context 任务队列扩展 kind/session_id/action 与 observation 结果 - /api/browser-context 路由下发 act 任务、接收 observation - BrowserContextConfig 增加 actions_enabled 等开关,tools.py 两级门控 - 接线 app.py 路由挂载、app_config.py 配置字段、config.yaml 工具注册 前端: - ChatPage 处理「发送当前页」回跳的 browser_context query 参数并自动提交 扩展 (browser-extension/deer-web-clipper): - shared.js 自包含注入函数 deerObservePage / deerPerformAction - background.js 按 kind 分发任务,session→tab 映射存 chrome.storage.session 测试:tests/test_browser_act.py(队列往返 + 工具校验 + 模拟扩展成功路径,8 passed) Co-Authored-By: Claude Opus 4.8 (1M context) --- browser-extension/deer-web-clipper/README.md | 136 +++ .../deer-web-clipper/background.js | 905 ++++++++++++++++++ browser-extension/deer-web-clipper/config.js | 9 + browser-extension/deer-web-clipper/content.js | 737 ++++++++++++++ .../deer-web-clipper/manifest.json | 24 + .../deer-web-clipper/options.html | 74 ++ browser-extension/deer-web-clipper/options.js | 148 +++ browser-extension/deer-web-clipper/popup.html | 31 + browser-extension/deer-web-clipper/popup.js | 151 +++ browser-extension/deer-web-clipper/shared.js | 389 ++++++++ browser-extension/deer-web-clipper/styles.css | 96 ++ frontend-web/src/pages/ChatPage.tsx | 110 ++- .../backend/app/gateway/app.py | 20 +- .../app/gateway/routers/browser_context.py | 590 ++++++++++++ offline-backend-20260512/backend/config.yaml | 191 ++-- .../deerflow/browser_context/__init__.py | 2 + .../harness/deerflow/browser_context/tasks.py | 303 ++++++ .../harness/deerflow/config/app_config.py | 8 +- .../deerflow/config/browser_context_config.py | 74 ++ .../tools/builtins/browser_act_tool.py | 225 +++++ .../tools/builtins/browser_context_tool.py | 117 +++ .../packages/harness/deerflow/tools/tools.py | 25 + .../backend/tests/test_browser_act.py | 158 +++ offline-backend-20260512/config.yaml | 138 +-- 24 files changed, 4499 insertions(+), 162 deletions(-) create mode 100644 browser-extension/deer-web-clipper/README.md create mode 100644 browser-extension/deer-web-clipper/background.js create mode 100644 browser-extension/deer-web-clipper/config.js create mode 100644 browser-extension/deer-web-clipper/content.js create mode 100644 browser-extension/deer-web-clipper/manifest.json create mode 100644 browser-extension/deer-web-clipper/options.html create mode 100644 browser-extension/deer-web-clipper/options.js create mode 100644 browser-extension/deer-web-clipper/popup.html create mode 100644 browser-extension/deer-web-clipper/popup.js create mode 100644 browser-extension/deer-web-clipper/shared.js create mode 100644 browser-extension/deer-web-clipper/styles.css create mode 100644 offline-backend-20260512/backend/app/gateway/routers/browser_context.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py create mode 100644 offline-backend-20260512/backend/tests/test_browser_act.py diff --git a/browser-extension/deer-web-clipper/README.md b/browser-extension/deer-web-clipper/README.md new file mode 100644 index 00000000..608c3303 --- /dev/null +++ b/browser-extension/deer-web-clipper/README.md @@ -0,0 +1,136 @@ +# zncm Web Clipper + +这是给 zncm 使用的 Chrome 插件,安装后默认连接 `config.js` 中配置好的服务地址。 + +默认地址配置在 `browser-extension/deer-web-clipper/config.js`: + +```js +globalThis.ZNCM_WEB_CLIPPER_CONFIG = Object.freeze({ + defaultWebBaseUrl: "http://47.94.209.59:2026", + defaultApiBaseUrl: "http://47.94.209.59:2026", +}); +``` + +如果部署地址变化,只需要修改 `config.js` 里的这两个字段,然后重新加载插件。 + +正常情况下不需要手动配置 URL,也不需要手动复制 token。 + +## 能做什么 + +1. 把当前浏览器标签页发送到 zncm,作为 Markdown 附件进入聊天。 +2. 接收 zncm 聊天中由 `browser_fetch_page` 工具创建的网页抓取任务。 +3. 插件在 Chrome 中打开目标网页,读取可见文本和表格,再把结果回传给 zncm。 +4. 在普通网页右下角显示“zncm 页面问答”浮窗,直接把当前页面内容交给 zncm 问答。 +5. 接收 zncm 的 `browser_act` 交互式操作任务:在 Chrome 中打开页面并按 zncm 的指令**操作页面**——点击、输入、滚动、等待、页内跳转,每一步把页面上的可交互元素清单和正文回传给 zncm,由 zncm 决定下一步(observe→act 循环)。 + +## 页面操作(browser_act) + +除了只读抓取,zncm 还能驱动插件**实际操作网页**,完成登录、填表、翻页找数据等多步任务: + +1. zncm 调用 `browser_act` 工具,先 `navigate` 打开页面。 +2. 插件打开标签页(同一会话会复用这个标签页),扫描页面上的可交互元素,给每个元素编号后连同正文回传给 zncm。 +3. zncm 根据元素编号决定下一步(点第 3 个按钮、在第 1 个输入框填字……),再次调用 `browser_act`。 +4. 插件执行动作、重新扫描页面、回传新状态,如此循环,直到任务完成。 +5. 任务结束时 zncm 用 `action="close"` 关闭该会话标签页。 + +支持的动作:`navigate`(打开/跳转 URL)、`click`(点击元素)、`type`(输入文字,可选回车提交)、`scroll`(滚动到元素或下翻一屏)、`wait`(等待某 CSS 选择器出现)、`extract`(重新读取正文)、`observe`(仅重新扫描)、`close`(关闭会话标签页)。 + +> 提示:页面操作能力较强(可点击任意按钮、提交表单),可在后端 `config.yaml` 用 `browser_context.actions_enabled: false` 关闭,只保留只读抓取。 + +## 后端启用 + +在 `offline-backend-20260512/backend/config.yaml` 中保持: + +```yaml +browser_context: + enabled: true # 只读抓取 + 页面问答 + actions_enabled: true # 交互式页面操作 browser_act(不需要可设为 false) +``` + +修改配置后需要重启后端。 + +## 安装插件 + +1. 打开 Chrome 的 `chrome://extensions`。 +2. 开启右上角的“开发者模式”。 +3. 点击“加载已解压的扩展程序”。 +4. 选择目录:`browser-extension/deer-web-clipper`。 +5. 安装后插件名会显示为 `zncm Web Clipper`。 + +## 安装即用 + +插件默认已经开启: + +- 自动识别已打开 zncm 页面的服务地址。 +- 自动同步已登录 zncm 的 token。 +- 自动轮询 zncm 聊天触发的浏览器抓取任务。 +- 页面问答框默认开启。 + +如果你打开的是 `config.js` 里配置的默认服务器,通常无需进入设置页。 + +如果部署地址变了,保持一个已登录的 zncm 页面打开,然后在插件设置里点击: + +- `重新识别服务地址` +- `同步 token` + +## 自动同步 Token + +插件会自动从两个位置获取登录 token: + +1. zncm API 域名下的 `access_token` Cookie,包括 HttpOnly Cookie。 +2. 已登录 zncm 前端页面里的浏览器本地登录态。 + +切换登录账号后,插件后台每次轮询任务前都会重新同步 token;点击插件里的“同步 token”也可以立即刷新。 + +如果当前浏览器已经登出 zncm,且打开的 zncm 页面里没有 token,插件会清空旧 token,避免继续用上一个账号请求。 + +## 页面问答框 + +安装插件后,普通 `http` / `https` 页面右下角会出现 `AI` 按钮。点击后可以在当前网页上直接提问,例如: + +```text +总结这页的核心结论 +提取页面里的价格和时间 +这张表里哪个指标最高? +``` + +插件会读取当前页面可见文本和表格,发送到 zncm,并把 zncm 的回答显示回浮窗里。多轮追问会复用同一个 zncm 会话;如果页面 URL 变化,会自动重新开始当前页会话。 + +是否显示问答框可以自己控制: + +- 插件弹窗里勾选或取消“启用页面问答框”。 +- 插件设置页里勾选或取消“启用页面问答框”。 + +关闭问答框只会隐藏页面浮窗,不会影响“发送当前页”和 zncm 聊天中的 `browser_fetch_page` 抓取任务。 + +## 手动发送当前网页 + +点击插件图标,选择“发送当前页”。 + +插件会读取当前页面的可见文本和表格,发送到 zncm,并打开一个新的 zncm 聊天页面。zncm 会把网页内容作为上传文件注入到首条问题里。 + +## 在 zncm 聊天中触发浏览器抓取 + +在 zncm 中输入带有明确 URL 的问题,例如: + +```text +使用 browser_fetch_page 打开 https://example.com/report,提取页面里的关键表格数据并总结。 +``` + +执行流程: + +1. zncm 调用 `browser_fetch_page` 工具创建浏览器任务。 +2. 插件后台轮询到任务。 +3. 插件在 Chrome 中打开目标网页。 +4. 插件提取页面文本和表格。 +5. 插件把结果回传给 zncm。 +6. zncm 基于回传内容回答你的问题。 + +这个方式适合读取需要浏览器登录态、内网权限或当前 Chrome 环境才能访问的页面。 + +## 注意事项 + +- 目标 URL 必须是明确的 `http` 或 `https` 地址。 +- 插件不能读取 `chrome://` 等浏览器内部页面。 +- 如果目标网站阻止扩展脚本注入,抓取可能失败。 +- 后台任务默认会打开新标签页;是否抓取完成后关闭标签页,可在插件设置里配置。 diff --git a/browser-extension/deer-web-clipper/background.js b/browser-extension/deer-web-clipper/background.js new file mode 100644 index 00000000..104380f7 --- /dev/null +++ b/browser-extension/deer-web-clipper/background.js @@ -0,0 +1,905 @@ +importScripts("config.js", "shared.js"); + +const ALARM_NAME = "deer-browser-task-poll"; +const LEGACY_LOCAL_API_BASE_URL = "http://localhost:8001"; +const LEGACY_LOCAL_WEB_BASE_URL = "http://localhost:5174"; +let pollTimer = null; +let isPolling = false; + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function getSettings() { + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + const updates = {}; + if (!settings.apiBaseUrl || settings.apiBaseUrl === LEGACY_LOCAL_API_BASE_URL) { + updates.apiBaseUrl = DEER_CLIPPER_DEFAULTS.apiBaseUrl; + } + if (!settings.webBaseUrl || settings.webBaseUrl === LEGACY_LOCAL_WEB_BASE_URL) { + updates.webBaseUrl = DEER_CLIPPER_DEFAULTS.webBaseUrl; + } + if (!settings.workerId) { + updates.workerId = deerMakeWorkerId(); + } + if (Object.keys(updates).length > 0) { + await chrome.storage.sync.set(updates); + Object.assign(settings, updates); + } + return settings; +} + +function pollIntervalMs(settings) { + const seconds = Number(settings.taskPollIntervalSeconds || DEER_CLIPPER_DEFAULTS.taskPollIntervalSeconds); + return Math.max(3, seconds) * 1000; +} + +function originMatchPattern(value) { + try { + const url = new URL(deerNormalizeBaseUrl(value)); + return `${url.protocol}//${url.host}/*`; + } catch { + return ""; + } +} + +function normalizeServiceBase(value) { + return deerNormalizeBaseUrl(value || ""); +} + +function serviceBaseFromTabUrl(value) { + try { + const url = new URL(value); + const path = url.pathname.replace(/\/+$/, ""); + if (path === "/deerflow" || path.startsWith("/deerflow/")) { + return `${url.origin}/deerflow`; + } + return url.origin; + } catch { + return ""; + } +} + +function apiCandidatesForWebBase(webBase, currentApiBase) { + const candidates = []; + const add = (value) => { + const normalized = normalizeServiceBase(value); + if (normalized && !candidates.includes(normalized)) { + candidates.push(normalized); + } + }; + + add(currentApiBase); + add(webBase); + add(DEER_CLIPPER_DEFAULTS.apiBaseUrl); + add(LEGACY_LOCAL_API_BASE_URL); + + try { + const url = new URL(normalizeServiceBase(webBase)); + if (["5173", "5174", "3000"].includes(url.port)) { + const apiUrl = new URL(url.href); + apiUrl.port = "8001"; + add(apiUrl.href); + } + } catch { + // Ignore invalid candidates. + } + + return candidates; +} + +async function browserContextStatus(apiBaseUrl) { + try { + const response = await fetch(`${apiBaseUrl}/api/browser-context/status`, { + credentials: "include", + }); + if (!response.ok) { + return null; + } + const data = await response.json().catch(() => null); + if (data && typeof data.enabled === "boolean") { + return data; + } + } catch { + return null; + } + return null; +} + +function tabLooksLikeZncm(tab) { + const value = `${tab.url || ""} ${tab.title || ""}`.toLowerCase(); + return ( + value.includes("deerflow") || + value.includes("deer") || + value.includes("zncm") || + value.includes("/page/workspace") || + value.includes("/chats/") || + value.includes("#/login") + ); +} + +async function findOpenZncmTabs(settings) { + const tabs = []; + const seen = new Set(); + const addTabs = (items) => { + for (const tab of items || []) { + if (tab?.id && !seen.has(tab.id)) { + seen.add(tab.id); + tabs.push(tab); + } + } + }; + + for (const pattern of [ + originMatchPattern(settings.webBaseUrl || DEER_CLIPPER_DEFAULTS.webBaseUrl), + originMatchPattern(DEER_CLIPPER_DEFAULTS.webBaseUrl), + ]) { + if (!pattern) { + continue; + } + try { + addTabs(await chrome.tabs.query({ url: pattern })); + } catch { + // Continue with broader discovery below. + } + } + + try { + const allTabs = await chrome.tabs.query({ url: ["http://*/*", "https://*/*"] }); + addTabs(allTabs.filter(tabLooksLikeZncm)); + } catch { + // Ignore tab discovery failures. + } + + return tabs; +} + +async function discoverServiceBases(settings, force = false) { + if (!settings.autoDiscoverServiceUrl && !force) { + return settings; + } + + let webBaseUrl = normalizeServiceBase(settings.webBaseUrl || DEER_CLIPPER_DEFAULTS.webBaseUrl); + let discoveredWebBaseUrl = webBaseUrl; + let source = "configured"; + const tabs = await findOpenZncmTabs(settings); + if (tabs.length > 0) { + const tabBase = serviceBaseFromTabUrl(tabs[0].url); + if (tabBase) { + discoveredWebBaseUrl = tabBase; + source = `tab:${tabs[0].url || tabBase}`; + } + } + + let apiBaseUrl = normalizeServiceBase(settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + let verified = false; + for (const candidate of apiCandidatesForWebBase(discoveredWebBaseUrl, apiBaseUrl)) { + const status = await browserContextStatus(candidate); + if (status) { + apiBaseUrl = candidate; + webBaseUrl = discoveredWebBaseUrl; + verified = true; + source = status.enabled ? `${source}:browser_context_enabled` : `${source}:browser_context_disabled`; + break; + } + } + if (!verified && force) { + source = `${source}:api_not_verified`; + } + + const next = { + ...settings, + apiBaseUrl, + webBaseUrl, + lastServiceSyncAt: new Date().toISOString(), + lastServiceSyncSource: source, + }; + + if ( + force || + settings.apiBaseUrl !== next.apiBaseUrl || + settings.webBaseUrl !== next.webBaseUrl || + settings.lastServiceSyncSource !== next.lastServiceSyncSource + ) { + await chrome.storage.sync.set({ + apiBaseUrl: next.apiBaseUrl, + webBaseUrl: next.webBaseUrl, + lastServiceSyncAt: next.lastServiceSyncAt, + lastServiceSyncSource: next.lastServiceSyncSource, + }); + } + return next; +} + +async function getCookieToken(settings) { + if (!chrome.cookies?.get) { + return null; + } + const apiBaseUrl = deerNormalizeBaseUrl(settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + try { + const cookie = await chrome.cookies.get({ url: apiBaseUrl, name: "access_token" }); + if (cookie?.value) { + return { token: cookie.value, source: `cookie:${new URL(apiBaseUrl).host}:access_token` }; + } + } catch (error) { + console.warn("Failed to read zncm access_token cookie", error); + } + return null; +} + +async function getTokenFromOpenZncmTab(settings) { + const tabs = await findOpenZncmTabs(settings); + let sawZncmTab = false; + + for (const tab of tabs) { + if (!tab.id) { + continue; + } + sawZncmTab = true; + try { + const [result] = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + func: deerExtractAuthTokenFromPage, + }); + const tokenInfo = result?.result; + if (tokenInfo?.token) { + const tabBase = serviceBaseFromTabUrl(tab.url); + if (tabBase && settings.autoDiscoverServiceUrl && settings.webBaseUrl !== tabBase) { + await chrome.storage.sync.set({ + webBaseUrl: tabBase, + lastServiceSyncAt: new Date().toISOString(), + lastServiceSyncSource: `token-tab:${tab.url || tabBase}`, + }); + } + return { + token: tokenInfo.token, + source: tokenInfo.source || `tab:${tab.url || "open-deer-tab"}`, + }; + } + } catch (error) { + console.warn("Failed to read zncm token from tab", tab.id, error); + } + } + + if (sawZncmTab) { + return { token: "", source: "open-deer-tab", clearStale: true }; + } + return null; +} + +async function syncAuthToken(settings = null, forceStatus = false) { + let current = settings || (await getSettings()); + current = await discoverServiceBases(current); + if (!current.tokenSyncEnabled && !forceStatus) { + return current; + } + + const tokenInfo = (await getTokenFromOpenZncmTab(current)) || (await getCookieToken(current)); + if (!tokenInfo?.token) { + if (tokenInfo?.clearStale && current.accessToken) { + const next = { + ...current, + accessToken: "", + lastTokenSyncSource: "cleared:open-deer-tab-no-token", + lastTokenSyncAt: new Date().toISOString(), + }; + await chrome.storage.sync.set({ + accessToken: "", + lastTokenSyncSource: next.lastTokenSyncSource, + lastTokenSyncAt: next.lastTokenSyncAt, + }); + return next; + } + return current; + } + + const next = { + ...current, + accessToken: tokenInfo.token, + lastTokenSyncSource: tokenInfo.source, + lastTokenSyncAt: new Date().toISOString(), + }; + if ( + forceStatus || + current.accessToken !== next.accessToken || + current.lastTokenSyncSource !== next.lastTokenSyncSource + ) { + await chrome.storage.sync.set({ + accessToken: next.accessToken, + lastTokenSyncSource: next.lastTokenSyncSource, + lastTokenSyncAt: next.lastTokenSyncAt, + }); + } + return next; +} + +async function refreshAlarm(settings) { + await chrome.alarms.clear(ALARM_NAME); + if (!settings.taskPollingEnabled) { + return; + } + const minutes = Math.max(0.5, pollIntervalMs(settings) / 60000); + await chrome.alarms.create(ALARM_NAME, { delayInMinutes: minutes, periodInMinutes: minutes }); +} + +async function scheduleNextPoll(delayMs = null) { + if (pollTimer) { + clearTimeout(pollTimer); + pollTimer = null; + } + const settings = await getSettings(); + if (!settings.taskPollingEnabled) { + return; + } + const ms = delayMs ?? pollIntervalMs(settings); + pollTimer = setTimeout(() => { + void pollOnce().catch((error) => console.warn("zncm task poll failed", error)); + }, ms); +} + +async function waitForTabLoad(tabId, timeoutSeconds) { + const timeoutMs = Math.max(5, Number(timeoutSeconds || 30)) * 1000; + try { + const tab = await chrome.tabs.get(tabId); + if (tab.status === "complete") { + return true; + } + } catch { + return false; + } + + return new Promise((resolve) => { + const timer = setTimeout(() => { + chrome.tabs.onUpdated.removeListener(listener); + resolve(false); + }, timeoutMs); + + function listener(updatedTabId, changeInfo) { + if (updatedTabId === tabId && changeInfo.status === "complete") { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(true); + } + } + + chrome.tabs.onUpdated.addListener(listener); + }); +} + +async function captureFromTab(tabId, maxContentChars) { + const [result] = await chrome.scripting.executeScript({ + target: { tabId }, + func: deerExtractCurrentPage, + }); + const page = result?.result; + if (!page?.content) { + throw new Error("当前页面没有可读取的文本。"); + } + const limit = Math.max(1000, Number(maxContentChars || 200000)); + if (page.content.length > limit) { + page.content = `${page.content.slice(0, limit)}\n\n[Truncated by zncm browser task limit]`; + } + return page; +} + +async function postTaskResult(taskId, settings, payload) { + const apiBaseUrl = deerNormalizeBaseUrl(settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + const response = await fetch(`${apiBaseUrl}/api/browser-context/tasks/${encodeURIComponent(taskId)}/result`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...deerAuthHeaders(settings), + }, + body: JSON.stringify(payload), + }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + const detail = typeof data.detail === "string" ? data.detail : JSON.stringify(data.detail || data); + throw new Error(detail || `HTTP ${response.status}`); + } + return data; +} + +function zncmApiBaseCandidates(settings) { + const candidates = []; + const add = (value) => { + const normalized = deerNormalizeBaseUrl(value || ""); + if (normalized && !candidates.includes(normalized)) { + candidates.push(normalized); + } + }; + + add(settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + try { + const url = new URL(deerNormalizeBaseUrl(settings.apiBaseUrl || "")); + if (url.pathname && url.pathname !== "/") { + add(url.origin); + } + } catch { + // Ignore invalid configured API URL. + } + add(DEER_CLIPPER_DEFAULTS.apiBaseUrl); + return candidates; +} + +async function zncmFetchJson(settings, path, options) { + let lastError = null; + for (const apiBaseUrl of zncmApiBaseCandidates(settings)) { + try { + const response = await fetch(`${apiBaseUrl}${path}`, options); + const contentType = response.headers.get("content-type") || ""; + const data = contentType.includes("application/json") + ? await response.json().catch(() => ({})) + : { detail: await response.text().catch(() => "") }; + if (response.ok) { + return { data, apiBaseUrl, status: response.status }; + } + const detail = typeof data.detail === "string" ? data.detail : JSON.stringify(data.detail || data); + lastError = new Error(detail || `HTTP ${response.status}`); + lastError.status = response.status; + lastError.detail = detail; + lastError.apiBaseUrl = apiBaseUrl; + if (response.status !== 404) { + throw lastError; + } + } catch (error) { + lastError = error; + if (error?.status && error.status !== 404) { + throw error; + } + } + } + throw lastError || new Error("zncm 服务请求失败。"); +} + +function zncmMakeThreadId() { + if (globalThis.crypto?.randomUUID) { + return globalThis.crypto.randomUUID(); + } + return `zncm-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; +} + +function zncmBuildPagePrompt(page, content, truncated, followUp) { + const title = page.title || "未知标题"; + const url = page.url || "未知 URL"; + const capturedAt = new Date().toISOString(); + if (followUp) { + return [ + "你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。", + "如果已有网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。", + `当前页面标题:${title}`, + `当前页面 URL:${url}`, + `本轮提问时间:${capturedAt}`, + ].join("\n\n"); + } + + return [ + "你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。", + "如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。", + `页面标题:${title}`, + `页面 URL:${url}`, + `读取时间:${capturedAt}`, + truncated ? "说明:页面内容较长,本次已按插件限制截断。" : "说明:页面内容如下。", + "当前网页内容:", + page.contentType === "text" ? `\`\`\`text\n${content}\n\`\`\`` : content, + ].join("\n\n"); +} + +function zncmExtractLatestAnswer(values) { + const messages = Array.isArray(values?.messages) ? values.messages : []; + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index] || {}; + const type = String(message.type || message.role || "").toLowerCase(); + if (type !== "ai" && type !== "assistant") { + continue; + } + const content = message.content; + if (typeof content === "string" && content.trim()) { + return content.trim(); + } + if (Array.isArray(content)) { + const text = content + .map((part) => { + if (typeof part === "string") return part; + if (part && typeof part.text === "string") return part.text; + return ""; + }) + .filter(Boolean) + .join("\n") + .trim(); + if (text) { + return text; + } + } + } + return ""; +} + +async function askZncmViaRunsWait(settings, payload, page, question) { + const contentLimit = 60000; + const rawContent = String(page.content || ""); + const truncated = rawContent.length > contentLimit; + const content = truncated ? `${rawContent.slice(0, contentLimit)}\n\n[Truncated locally by zncm Web Clipper]` : rawContent; + const threadId = payload.threadId || zncmMakeThreadId(); + const promptPrefix = zncmBuildPagePrompt(page, content, truncated, Boolean(payload.threadId)); + const { data } = await zncmFetchJson(settings, "/api/runs/wait", { + method: "POST", + headers: { + "Content-Type": "application/json", + ...deerAuthHeaders(settings), + }, + body: JSON.stringify({ + assistant_id: "lead_agent", + input: { + messages: [ + { + type: "human", + content: question, + additional_kwargs: { + prompt_prefix: promptPrefix, + }, + }, + ], + }, + config: { + configurable: { + thread_id: threadId, + }, + }, + metadata: { + source: "zncm_page_assistant", + browser_context_url: page.url || "", + browser_context_title: page.title || "", + browser_context_fallback: true, + }, + stream_mode: ["values"], + on_disconnect: "continue", + multitask_strategy: "reject", + }), + }); + return { + success: true, + thread_id: threadId, + run_id: "", + answer: zncmExtractLatestAnswer(data) || "zncm 没有返回可显示的答案。", + truncated, + message: "zncm 页面问答已完成。", + }; +} + +async function askZncmAboutPage(payload) { + let settings = await getSettings(); + settings = await discoverServiceBases(settings); + settings = await syncAuthToken(settings); + if (!settings.pageAssistantEnabled) { + throw new Error("页面问答框已关闭。"); + } + + const page = payload?.page || {}; + const question = String(payload?.question || "").trim(); + if (!question) { + throw new Error("请输入要询问的问题。"); + } + if (!page.content) { + throw new Error("当前页面没有可读取的正文。"); + } + + try { + const { data } = await zncmFetchJson(settings, "/api/browser-context/ask", { + method: "POST", + headers: { + "Content-Type": "application/json", + ...deerAuthHeaders(settings), + }, + body: JSON.stringify({ + thread_id: payload.threadId || null, + title: page.title || "", + url: page.url || "", + content: page.content || "", + content_type: page.contentType || "markdown", + question, + }), + }); + return data; + } catch (error) { + const detail = String(error?.detail || error?.message || ""); + if (error?.status === 404 && !/browser context integration is disabled/i.test(detail)) { + return askZncmViaRunsWait(settings, payload, page, question); + } + throw error; + } +} + +async function getActSessionMap() { + try { + const stored = await chrome.storage.session.get({ actSessions: {} }); + return stored.actSessions && typeof stored.actSessions === "object" ? stored.actSessions : {}; + } catch { + return {}; + } +} + +async function setActSessionTab(sessionId, tabId) { + if (!sessionId) return; + const map = await getActSessionMap(); + map[sessionId] = tabId; + await chrome.storage.session.set({ actSessions: map }).catch(() => {}); +} + +async function removeActSession(sessionId) { + if (!sessionId) return; + const map = await getActSessionMap(); + if (sessionId in map) { + delete map[sessionId]; + await chrome.storage.session.set({ actSessions: map }).catch(() => {}); + } +} + +async function resolveSessionTabId(sessionId) { + const map = await getActSessionMap(); + const tabId = map[sessionId]; + if (!tabId) return null; + try { + await chrome.tabs.get(tabId); + return tabId; + } catch { + await removeActSession(sessionId); + return null; + } +} + +async function performActionInTab(tabId, action) { + const [result] = await chrome.scripting.executeScript({ + target: { tabId }, + func: deerPerformAction, + args: [action], + }); + return result?.result || { status: "ok", message: "" }; +} + +async function observeTab(tabId, maxElements) { + const [result] = await chrome.scripting.executeScript({ + target: { tabId }, + func: deerObservePage, + args: [maxElements], + }); + const observation = result?.result; + if (!observation) { + throw new Error("当前页面无法读取交互元素。"); + } + return observation; +} + +async function runActTask(task, settings) { + const sessionId = task.session_id || ""; + const action = task.action || {}; + const actionType = String(action.type || "").toLowerCase(); + const maxElements = 150; + + try { + if (!sessionId) { + throw new Error("act 任务缺少 session_id。"); + } + + if (actionType === "close") { + const tabId = await resolveSessionTabId(sessionId); + if (tabId) { + await chrome.tabs.remove(tabId).catch(() => {}); + } + await removeActSession(sessionId); + await postTaskResult(task.id, settings, { + kind: "act", + observation: { action_status: "ok", message: "Tab closed.", url: "", title: "", elements: [], text: "" }, + metadata: { source: "zncm-web-clipper", worker_id: settings.workerId, session_id: sessionId }, + }); + return; + } + + let tabId = await resolveSessionTabId(sessionId); + + if (actionType === "navigate") { + const url = String(action.url || task.url || ""); + if (!url) { + throw new Error("navigate 动作缺少 url。"); + } + if (tabId) { + await chrome.tabs.update(tabId, { url }); + } else { + const tab = await chrome.tabs.create({ url, active: false }); + tabId = tab.id; + await setActSessionTab(sessionId, tabId); + } + await waitForTabLoad(tabId, settings.taskPageLoadTimeoutSeconds); + await delay(800); + } else { + if (!tabId) { + throw new Error("会话标签页不存在,请先用 navigate 打开页面。"); + } + const outcome = await performActionInTab(tabId, action); + // Give the page a moment to react / navigate, then settle. + await delay(400); + await waitForTabLoad(tabId, settings.taskPageLoadTimeoutSeconds); + await delay(500); + if (outcome.status === "error") { + const observation = await observeTab(tabId, maxElements).catch(() => ({ elements: [], text: "", url: "", title: "" })); + await postTaskResult(task.id, settings, { + kind: "act", + observation: { ...observation, action_status: outcome.status, message: outcome.message }, + metadata: { source: "zncm-web-clipper", worker_id: settings.workerId, session_id: sessionId }, + }); + return; + } + } + + const observation = await observeTab(tabId, maxElements); + await postTaskResult(task.id, settings, { + kind: "act", + title: observation.title || "", + url: observation.url || "", + observation: { ...observation, action_status: "ok", message: "" }, + metadata: { source: "zncm-web-clipper", worker_id: settings.workerId, session_id: sessionId }, + }); + } catch (error) { + await postTaskResult(task.id, settings, { + kind: "act", + error: error instanceof Error ? error.message : String(error), + metadata: { source: "zncm-web-clipper", worker_id: settings.workerId, session_id: sessionId }, + }).catch((submitError) => console.warn("Failed to submit zncm act task error", submitError)); + } +} + +async function runFetchTask(task, settings) { + let tabId = null; + try { + const tab = await chrome.tabs.create({ url: task.url, active: false }); + tabId = tab.id; + await waitForTabLoad(tabId, settings.taskPageLoadTimeoutSeconds); + await delay(750); + const page = await captureFromTab(tabId, task.max_content_chars); + await postTaskResult(task.id, settings, { + kind: "fetch", + title: page.title, + url: page.url, + content: page.content, + content_type: page.contentType || "markdown", + metadata: { + source: "zncm-web-clipper", + worker_id: settings.workerId, + instruction: task.instruction || "", + }, + }); + } catch (error) { + await postTaskResult(task.id, settings, { + kind: "fetch", + error: error instanceof Error ? error.message : String(error), + metadata: { + source: "zncm-web-clipper", + worker_id: settings.workerId, + }, + }).catch((submitError) => console.warn("Failed to submit zncm task error", submitError)); + } finally { + if (tabId && settings.closeTaskTabs) { + await chrome.tabs.remove(tabId).catch(() => {}); + } + } +} + +async function runTask(task, settings) { + if ((task.kind || "fetch") === "act") { + await runActTask(task, settings); + return; + } + await runFetchTask(task, settings); +} + +async function pollOnce() { + if (isPolling) { + return { ok: true, message: "任务轮询正在运行。" }; + } + + isPolling = true; + try { + let settings = await getSettings(); + settings = await discoverServiceBases(settings); + settings = await syncAuthToken(settings); + if (!settings.taskPollingEnabled) { + return { ok: false, message: "任务轮询已关闭。" }; + } + + const apiBaseUrl = deerNormalizeBaseUrl(settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + const url = `${apiBaseUrl}/api/browser-context/tasks/next?worker_id=${encodeURIComponent(settings.workerId)}`; + const response = await fetch(url, { headers: deerAuthHeaders(settings) }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + const detail = typeof data.detail === "string" ? data.detail : JSON.stringify(data.detail || data); + throw new Error(detail || `HTTP ${response.status}`); + } + if (!data.task) { + return { ok: true, message: "暂无待处理的 zncm 浏览器任务。" }; + } + + await runTask(data.task, settings); + return { ok: true, message: `已完成 zncm 浏览器任务 ${data.task.id}。` }; + } finally { + isPolling = false; + await scheduleNextPoll(); + } +} + +async function startup() { + let settings = await getSettings(); + settings = await discoverServiceBases(settings); + settings = await syncAuthToken(settings); + await refreshAlarm(settings); + await scheduleNextPoll(1000); +} + +chrome.runtime.onInstalled.addListener(() => void startup()); +chrome.runtime.onStartup.addListener(() => void startup()); +chrome.storage.onChanged.addListener((changes, areaName) => { + if (areaName === "sync") { + void startup(); + } +}); +chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === ALARM_NAME) { + void pollOnce().catch((error) => console.warn("zncm alarm poll failed", error)); + } +}); +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if (message?.type === "zncm-discover-service") { + getSettings() + .then((settings) => discoverServiceBases(settings, true)) + .then((settings) => + sendResponse({ + ok: Boolean(settings.apiBaseUrl && settings.webBaseUrl), + message: `服务地址:${settings.webBaseUrl};API:${settings.apiBaseUrl}`, + apiBaseUrl: settings.apiBaseUrl, + webBaseUrl: settings.webBaseUrl, + source: settings.lastServiceSyncSource || "", + }), + ) + .catch((error) => sendResponse({ ok: false, message: error instanceof Error ? error.message : String(error) })); + return true; + } + + if (message?.type === "zncm-sync-token") { + syncAuthToken(null, true) + .then((settings) => { + const hasToken = Boolean(settings.accessToken); + sendResponse({ + ok: hasToken, + message: hasToken + ? `Token 已从 ${settings.lastTokenSyncSource || "zncm"} 同步。` + : "没有找到 zncm token。请打开已登录的 zncm 页面后重试。", + source: settings.lastTokenSyncSource || "", + }); + }) + .catch((error) => sendResponse({ ok: false, message: error instanceof Error ? error.message : String(error) })); + return true; + } + + if (message?.type === "zncm-page-agent-ask") { + askZncmAboutPage(message) + .then((result) => sendResponse({ ok: true, ...result })) + .catch((error) => sendResponse({ ok: false, message: error instanceof Error ? error.message : String(error) })); + return true; + } + + if (message?.type === "zncm-open-options") { + chrome.runtime.openOptionsPage() + .then(() => sendResponse({ ok: true })) + .catch((error) => sendResponse({ ok: false, message: error instanceof Error ? error.message : String(error) })); + return true; + } + + if (message?.type !== "zncm-poll-now") { + return false; + } + pollOnce() + .then((result) => sendResponse(result)) + .catch((error) => sendResponse({ ok: false, message: error instanceof Error ? error.message : String(error) })); + return true; +}); + +void startup(); diff --git a/browser-extension/deer-web-clipper/config.js b/browser-extension/deer-web-clipper/config.js new file mode 100644 index 00000000..5ab04b8d --- /dev/null +++ b/browser-extension/deer-web-clipper/config.js @@ -0,0 +1,9 @@ +// Default deployment settings for zncm Web Clipper. +// +// Change these two values before loading or packaging the extension when the +// zncm service address changes. Users can still override them in the extension +// settings page after installation. +globalThis.ZNCM_WEB_CLIPPER_CONFIG = Object.freeze({ + defaultWebBaseUrl: "http://47.94.209.59:2026", + defaultApiBaseUrl: "http://47.94.209.59:2026", +}); diff --git a/browser-extension/deer-web-clipper/content.js b/browser-extension/deer-web-clipper/content.js new file mode 100644 index 00000000..1c387461 --- /dev/null +++ b/browser-extension/deer-web-clipper/content.js @@ -0,0 +1,737 @@ +(() => { + if (window.top !== window || window.__zncmPageAssistantLoaded) { + return; + } + window.__zncmPageAssistantLoaded = true; + + const HOST_ID = "zncm-page-assistant-host"; + const state = { + enabled: true, + open: false, + busy: false, + threadId: "", + pageUrl: location.href, + messages: [], + }; + + let host = null; + let shadow = null; + let ui = {}; + + function sendRuntimeMessage(message) { + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage(message, (response) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message)); + return; + } + resolve(response || {}); + }); + }); + } + + function makeElement(tag, className = "", text = "") { + const element = document.createElement(tag); + if (className) { + element.className = className; + } + if (text) { + element.textContent = text; + } + return element; + } + + function setStatus(message) { + if (ui.status) { + ui.status.textContent = message || ""; + } + } + + function setBusy(busy) { + state.busy = busy; + if (ui.send) ui.send.disabled = busy; + if (ui.input) ui.input.disabled = busy; + if (ui.panel) ui.panel.classList.toggle("is-busy", busy); + } + + function resetThreadIfPageChanged() { + if (state.pageUrl === location.href) { + return; + } + state.pageUrl = location.href; + state.threadId = ""; + state.messages = []; + renderMessages(); + setStatus("页面已变化,下一次提问会重新读取当前页面。"); + } + + function destroyWidget() { + if (host) { + host.remove(); + } + host = null; + shadow = null; + ui = {}; + } + + function renderMessages() { + if (!ui.messages) { + return; + } + ui.messages.replaceChildren(); + if (state.messages.length === 0) { + const empty = makeElement("div", "empty"); + const emptyTitle = makeElement("strong", "", "问问当前页面"); + const emptyText = makeElement("span", "", "可以提问页面里的数据、表格、段落和结论。"); + empty.append(emptyTitle, emptyText); + ui.messages.appendChild(empty); + return; + } + + for (const message of state.messages) { + const item = makeElement("div", `message ${message.role}`); + const avatar = makeElement("div", "avatar", message.role === "user" ? "你" : "zn"); + const bubble = makeElement("div", "bubble"); + const label = makeElement("div", "message-label", message.role === "user" ? "你" : "zncm"); + const body = makeElement("div", "message-body", message.text); + if (message.pending) { + body.classList.add("pending"); + } + bubble.append(label, body); + item.append(avatar, bubble); + ui.messages.appendChild(item); + } + ui.messages.scrollTop = ui.messages.scrollHeight; + } + + function setOpen(open) { + state.open = open; + if (!ui.panel || !ui.launcher) { + return; + } + ui.panel.classList.toggle("open", open); + ui.launcher.classList.toggle("open", open); + ui.launcher.setAttribute("aria-expanded", String(open)); + if (open) { + resetThreadIfPageChanged(); + setTimeout(() => ui.input?.focus(), 50); + } + } + + async function ask(question) { + const normalized = String(question || "").trim(); + if (!normalized || state.busy) { + return; + } + + resetThreadIfPageChanged(); + setBusy(true); + setStatus("正在读取页面并发送给 zncm..."); + state.messages.push({ role: "user", text: normalized }); + const pending = { role: "assistant", text: "zncm 正在理解当前页面...", pending: true }; + state.messages.push(pending); + renderMessages(); + ui.input.value = ""; + + try { + if (typeof deerExtractCurrentPage !== "function") { + throw new Error("页面读取脚本未加载,请刷新当前页面后重试。"); + } + const page = deerExtractCurrentPage(); + const response = await sendRuntimeMessage({ + type: "zncm-page-agent-ask", + page, + question: normalized, + threadId: state.threadId || "", + }); + if (!response.ok) { + throw new Error(response.message || "zncm 没有返回答案。"); + } + state.threadId = response.thread_id || state.threadId; + pending.text = response.answer || "zncm 没有返回答案。"; + pending.pending = false; + setStatus(response.truncated ? "已回答。页面内容较长,本次按配置做了截断。" : "已回答。"); + } catch (error) { + pending.text = error instanceof Error ? error.message : "页面问答失败。"; + pending.pending = false; + setStatus("本次问答失败。"); + } finally { + setBusy(false); + renderMessages(); + ui.input.focus(); + } + } + + function buildWidget() { + host = document.createElement("div"); + host.id = HOST_ID; + document.documentElement.appendChild(host); + shadow = host.attachShadow({ mode: "open" }); + + const style = document.createElement("style"); + style.textContent = ` + :host { + all: initial; + color-scheme: light; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + } + + .launcher, + .panel { + position: fixed; + z-index: 2147483647; + font: 14px/1.45 Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + } + + .launcher { + right: 20px; + bottom: 20px; + display: grid; + grid-template-columns: auto 1fr; + align-items: center; + gap: 8px; + height: 44px; + min-width: 112px; + border: 1px solid rgba(15, 118, 110, 0.28); + border-radius: 8px; + padding: 0 12px 0 10px; + background: #0f766e; + color: #fff; + box-shadow: 0 12px 34px rgba(15, 118, 110, 0.28), 0 2px 8px rgba(17, 24, 39, 0.12); + cursor: pointer; + font-weight: 750; + transition: transform 160ms ease, box-shadow 160ms ease, background 160ms ease; + } + + .launcher:hover { + transform: translateY(-2px); + box-shadow: 0 16px 40px rgba(15, 118, 110, 0.32), 0 4px 12px rgba(17, 24, 39, 0.14); + } + + .launcher-mark { + display: grid; + place-items: center; + width: 24px; + height: 24px; + border-radius: 7px; + background: rgba(255, 255, 255, 0.18); + font-size: 12px; + letter-spacing: 0; + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.2); + } + + .launcher.open { + background: #115e59; + } + + .panel { + right: 20px; + bottom: 72px; + width: min(430px, calc(100vw - 24px)); + max-height: min(640px, calc(100vh - 96px)); + display: none; + grid-template-rows: auto auto minmax(180px, 1fr) auto; + overflow: hidden; + border: 1px solid #d6d8dc; + border-radius: 8px; + background: #ffffff; + color: #171717; + box-shadow: 0 24px 70px rgba(17, 24, 39, 0.2), 0 1px 2px rgba(17, 24, 39, 0.08); + } + + .panel.open { + display: grid; + animation: zncm-panel-in 160ms ease-out; + } + + @keyframes zncm-panel-in { + from { + opacity: 0; + transform: translateY(8px) scale(0.985); + } + to { + opacity: 1; + transform: translateY(0) scale(1); + } + } + + .header { + display: grid; + gap: 10px; + padding: 14px; + background: linear-gradient(180deg, #f4fffc 0%, #ffffff 84%); + border-bottom: 1px solid #e5e7eb; + } + + .header-top { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + } + + .brand { + display: flex; + align-items: center; + min-width: 0; + gap: 10px; + } + + .brand-mark { + display: grid; + place-items: center; + width: 34px; + height: 34px; + flex: 0 0 auto; + border-radius: 8px; + background: #0f766e; + color: #fff; + font-weight: 800; + box-shadow: 0 8px 20px rgba(15, 118, 110, 0.18); + } + + .brand-copy { + display: grid; + min-width: 0; + gap: 1px; + } + + .brand-copy strong { + color: #111827; + font-size: 15px; + line-height: 1.25; + } + + .brand-copy span { + overflow: hidden; + color: #6b7280; + font-size: 12px; + text-overflow: ellipsis; + white-space: nowrap; + } + + .header-actions { + display: flex; + flex: 0 0 auto; + gap: 6px; + } + + .icon-button { + display: grid; + place-items: center; + width: auto; + min-width: 34px; + height: 30px; + padding: 0 8px; + border: 1px solid #dfe3e8; + border-radius: 7px; + background: #fff; + color: #374151; + cursor: pointer; + font: inherit; + font-size: 12px; + white-space: nowrap; + transition: background 140ms ease, border-color 140ms ease, color 140ms ease; + } + + .icon-button:hover { + border-color: #0f766e; + color: #0f766e; + background: #f0fdfa; + } + + .page-chip { + display: grid; + grid-template-columns: auto 1fr; + align-items: center; + gap: 8px; + min-width: 0; + border: 1px solid #dbeafe; + border-radius: 8px; + padding: 8px 10px; + background: #f8fbff; + color: #334155; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.8); + } + + .page-chip-dot { + width: 8px; + height: 8px; + border-radius: 999px; + background: #0f766e; + } + + .page-chip-text { + overflow: hidden; + font-size: 12px; + text-overflow: ellipsis; + white-space: nowrap; + } + + .status { + min-height: 20px; + padding: 9px 14px; + border-bottom: 1px solid #f0f2f5; + color: #475569; + background: #fbfcfd; + font-size: 12px; + } + + .messages { + display: grid; + align-content: start; + gap: 12px; + overflow: auto; + padding: 14px; + background: linear-gradient(180deg, #f7f8fa 0%, #f3f6f8 100%); + } + + .empty { + display: grid; + gap: 5px; + padding: 16px; + border: 1px solid #dfe7ed; + border-radius: 8px; + background: linear-gradient(180deg, #ffffff 0%, #fbfefd 100%); + color: #4b5563; + box-shadow: 0 8px 20px rgba(15, 23, 42, 0.04); + } + + .empty strong { + color: #111827; + font-size: 14px; + } + + .empty span { + font-size: 13px; + } + + .message { + display: grid; + grid-template-columns: 28px minmax(0, 1fr); + gap: 8px; + align-items: start; + } + + .message.user { + grid-template-columns: minmax(0, 1fr) 28px; + } + + .message.user .avatar { + grid-column: 2; + } + + .message.user .bubble { + grid-column: 1; + justify-self: end; + } + + .avatar { + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 8px; + background: #e6fffb; + color: #0f766e; + font-size: 11px; + font-weight: 800; + } + + .message.user .avatar { + background: #111827; + color: #fff; + } + + .bubble { + display: grid; + gap: 4px; + max-width: 100%; + } + + .message-label { + color: #6b7280; + font-size: 11px; + } + + .message.user .message-label { + text-align: right; + } + + .message-body { + max-width: 330px; + padding: 10px 12px; + border: 1px solid #e5e7eb; + border-radius: 8px; + background: #fff; + color: #171717; + white-space: pre-wrap; + overflow-wrap: anywhere; + box-shadow: 0 1px 2px rgba(15, 23, 42, 0.04); + } + + .message.user .message-body { + border-color: #0f172a; + background: #0f172a; + color: #fff; + } + + .message-body.pending { + color: #6b7280; + } + + .message-body.pending::after { + content: ""; + display: inline-block; + width: 18px; + height: 4px; + margin-left: 6px; + border-radius: 999px; + background: linear-gradient(90deg, #0f766e 0 33%, transparent 33% 100%); + background-size: 9px 4px; + animation: zncm-thinking 900ms linear infinite; + vertical-align: middle; + } + + @keyframes zncm-thinking { + from { + background-position: 0 0; + } + to { + background-position: 18px 0; + } + } + + .composer-wrap { + display: grid; + gap: 10px; + padding: 12px 14px 14px; + background: #fff; + border-top: 1px solid #e5e7eb; + } + + .quick { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 7px; + } + + .quick button { + border: 1px solid #d6d8dc; + border-radius: 8px; + padding: 8px 10px; + background: #fbfcfd; + color: #374151; + cursor: pointer; + font: inherit; + font-size: 12px; + text-align: left; + transition: background 140ms ease, border-color 140ms ease, color 140ms ease, transform 140ms ease; + } + + .quick button:hover { + border-color: #0f766e; + color: #0f766e; + background: #f0fdfa; + transform: translateY(-1px); + } + + .composer { + display: grid; + grid-template-columns: 1fr auto; + gap: 8px; + align-items: end; + } + + .composer textarea { + min-height: 46px; + max-height: 130px; + resize: vertical; + border: 1px solid #cfd6dd; + border-radius: 8px; + padding: 10px 11px; + color: #171717; + background: #fbfcfd; + font: inherit; + outline: none; + } + + .composer textarea:focus { + border-color: #0f766e; + box-shadow: 0 0 0 3px rgba(15, 118, 110, 0.12); + } + + .composer button { + height: 42px; + border: 0; + border-radius: 8px; + padding: 0 14px; + background: #0f766e; + color: #fff; + cursor: pointer; + font: inherit; + font-weight: 750; + box-shadow: 0 8px 18px rgba(15, 118, 110, 0.2); + transition: transform 140ms ease, background 140ms ease, box-shadow 140ms ease; + } + + .composer button:hover { + background: #115e59; + transform: translateY(-1px); + box-shadow: 0 10px 22px rgba(15, 118, 110, 0.24); + } + + .panel.is-busy .brand-mark, + .panel.is-busy .launcher-mark { + opacity: 0.82; + } + + .composer button:disabled, + .quick button:disabled, + .icon-button:disabled { + cursor: not-allowed; + opacity: 0.6; + } + + @media (max-width: 480px) { + .launcher { + right: 12px; + bottom: 12px; + min-width: 96px; + } + + .panel { + right: 12px; + bottom: 64px; + width: calc(100vw - 24px); + max-height: calc(100vh - 80px); + } + + .message-body { + max-width: calc(100vw - 104px); + } + + .quick { + grid-template-columns: 1fr; + } + } + `; + + const launcher = makeElement("button", "launcher"); + launcher.type = "button"; + launcher.title = "打开 zncm 页面问答"; + launcher.setAttribute("aria-expanded", "false"); + launcher.append(makeElement("span", "launcher-mark", "zn"), makeElement("span", "", "页面问答")); + + const panel = makeElement("section", "panel"); + const header = makeElement("header", "header"); + const headerTop = makeElement("div", "header-top"); + const brand = makeElement("div", "brand"); + const brandCopy = makeElement("div", "brand-copy"); + const title = makeElement("strong", "", "zncm 页面问答"); + const subtitle = makeElement("span", "", "读取当前页面并回答"); + brandCopy.append(title, subtitle); + brand.append(makeElement("div", "brand-mark", "zn"), brandCopy); + + const actions = makeElement("div", "header-actions"); + const settings = makeElement("button", "icon-button", "设置"); + settings.type = "button"; + settings.title = "打开插件设置"; + const minimize = makeElement("button", "icon-button", "-"); + minimize.type = "button"; + minimize.title = "收起"; + actions.append(settings, minimize); + headerTop.append(brand, actions); + + const pageChip = makeElement("div", "page-chip"); + const pageTitle = makeElement("span", "page-chip-text", document.title || location.hostname); + pageChip.append(makeElement("span", "page-chip-dot"), pageTitle); + header.append(headerTop, pageChip); + + const status = makeElement("div", "status", "已连接当前页面。"); + const messages = makeElement("div", "messages"); + + const composerWrap = makeElement("div", "composer-wrap"); + const quick = makeElement("div", "quick"); + for (const text of ["总结此页", "提取关键数据", "列出待办", "解释表格"]) { + const chip = makeElement("button", "", text); + chip.type = "button"; + chip.addEventListener("click", () => ask(text)); + quick.appendChild(chip); + } + + const composer = makeElement("form", "composer"); + const input = makeElement("textarea"); + input.placeholder = "问当前页面..."; + input.rows = 2; + const send = makeElement("button", "", "发送"); + send.type = "submit"; + composer.append(input, send); + composerWrap.append(quick, composer); + + panel.append(header, status, messages, composerWrap); + shadow.append(style, panel, launcher); + + ui = { launcher, panel, subtitle, pageTitle, status, messages, input, send }; + renderMessages(); + + launcher.addEventListener("click", () => setOpen(!state.open)); + minimize.addEventListener("click", () => setOpen(false)); + settings.addEventListener("click", () => { + sendRuntimeMessage({ type: "zncm-open-options" }).catch((error) => { + setStatus(error instanceof Error ? error.message : "无法打开插件设置。"); + }); + }); + composer.addEventListener("submit", (event) => { + event.preventDefault(); + ask(input.value); + }); + input.addEventListener("keydown", (event) => { + if (event.key === "Enter" && !event.shiftKey) { + event.preventDefault(); + ask(input.value); + } + }); + } + + function ensureWidget() { + if (!state.enabled) { + destroyWidget(); + return; + } + if (document.getElementById(HOST_ID)) { + return; + } + buildWidget(); + } + + async function loadSettings() { + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + state.enabled = Boolean(settings.pageAssistantEnabled); + ensureWidget(); + } + + chrome.storage.onChanged.addListener((changes, areaName) => { + if (areaName !== "sync" || !changes.pageAssistantEnabled) { + return; + } + state.enabled = Boolean(changes.pageAssistantEnabled.newValue); + ensureWidget(); + if (state.enabled) { + setStatus("页面问答框已开启。"); + } + }); + + setInterval(() => { + if (!state.enabled || !ui.pageTitle) { + return; + } + if (state.pageUrl !== location.href) { + resetThreadIfPageChanged(); + } + ui.pageTitle.textContent = document.title || location.hostname; + }, 1200); + + void loadSettings(); +})(); diff --git a/browser-extension/deer-web-clipper/manifest.json b/browser-extension/deer-web-clipper/manifest.json new file mode 100644 index 00000000..637adddb --- /dev/null +++ b/browser-extension/deer-web-clipper/manifest.json @@ -0,0 +1,24 @@ +{ + "manifest_version": 3, + "name": "zncm Web Clipper", + "version": "0.2.0", + "description": "Send pages to zncm and execute browser-backed fetch tasks.", + "permissions": ["activeTab", "alarms", "cookies", "scripting", "storage", "tabs"], + "host_permissions": [""], + "background": { + "service_worker": "background.js" + }, + "action": { + "default_title": "zncm Web Clipper", + "default_popup": "popup.html" + }, + "options_page": "options.html", + "content_scripts": [ + { + "matches": ["http://*/*", "https://*/*"], + "js": ["config.js", "shared.js", "content.js"], + "run_at": "document_idle", + "all_frames": false + } + ] +} diff --git a/browser-extension/deer-web-clipper/options.html b/browser-extension/deer-web-clipper/options.html new file mode 100644 index 00000000..107964e1 --- /dev/null +++ b/browser-extension/deer-web-clipper/options.html @@ -0,0 +1,74 @@ + + + + + zncm Web Clipper Settings + + + +
+

zncm Web Clipper Settings

+ + + +
+ +
+

+ + +
+ +
+

+ + + + + + + + +

+
+ + + + + diff --git a/browser-extension/deer-web-clipper/options.js b/browser-extension/deer-web-clipper/options.js new file mode 100644 index 00000000..855a6589 --- /dev/null +++ b/browser-extension/deer-web-clipper/options.js @@ -0,0 +1,148 @@ +const textFields = [ + "apiBaseUrl", + "webBaseUrl", + "accessToken", + "defaultPrompt", + "workerId", + "taskPollIntervalSeconds", + "taskPageLoadTimeoutSeconds", +]; +const checkboxFields = [ + "autoDiscoverServiceUrl", + "tokenSyncEnabled", + "pageAssistantEnabled", + "taskPollingEnabled", + "closeTaskTabs", +]; + +function formatServiceStatus(settings) { + if (!settings.lastServiceSyncAt) { + return `当前服务地址:${settings.webBaseUrl || DEER_CLIPPER_DEFAULTS.webBaseUrl};API:${settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl}`; + } + return `当前服务地址:${settings.webBaseUrl || ""};API:${settings.apiBaseUrl || ""}\n上次识别:${settings.lastServiceSyncAt}(${settings.lastServiceSyncSource || "unknown source"})`; +} + +function formatTokenStatus(settings) { + if (!settings.lastTokenSyncAt) { + return "Token 尚未自动同步。"; + } + return `上次 token 同步:${settings.lastTokenSyncAt}(${settings.lastTokenSyncSource || "unknown source"})`; +} + +async function loadSettings() { + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + if (!settings.workerId) { + settings.workerId = deerMakeWorkerId(); + await chrome.storage.sync.set({ workerId: settings.workerId }); + } + document.getElementById("apiBaseUrl").placeholder = + DEER_CLIPPER_DEFAULTS.apiBaseUrl || "在 config.js 中配置默认 API 地址"; + document.getElementById("webBaseUrl").placeholder = + DEER_CLIPPER_DEFAULTS.webBaseUrl || "在 config.js 中配置默认前端地址"; + for (const field of textFields) { + document.getElementById(field).value = settings[field] ?? ""; + } + for (const field of checkboxFields) { + document.getElementById(field).checked = Boolean(settings[field]); + } + document.getElementById("serviceStatus").textContent = formatServiceStatus(settings); + document.getElementById("tokenStatus").textContent = formatTokenStatus(settings); +} + +async function saveSettings() { + const next = {}; + for (const field of textFields) { + next[field] = document.getElementById(field).value.trim(); + } + for (const field of checkboxFields) { + next[field] = document.getElementById(field).checked; + } + next.apiBaseUrl = deerNormalizeBaseUrl(next.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + next.webBaseUrl = deerNormalizeBaseUrl(next.webBaseUrl || DEER_CLIPPER_DEFAULTS.webBaseUrl); + next.defaultPrompt = next.defaultPrompt || DEER_CLIPPER_DEFAULT_PROMPT; + next.workerId = next.workerId || deerMakeWorkerId(); + next.taskPollIntervalSeconds = Math.max(3, Number(next.taskPollIntervalSeconds || DEER_CLIPPER_DEFAULTS.taskPollIntervalSeconds)); + next.taskPageLoadTimeoutSeconds = Math.max(5, Number(next.taskPageLoadTimeoutSeconds || DEER_CLIPPER_DEFAULTS.taskPageLoadTimeoutSeconds)); + await chrome.storage.sync.set(next); + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + document.getElementById("status").textContent = "已保存。"; + document.getElementById("serviceStatus").textContent = formatServiceStatus(settings); + document.getElementById("tokenStatus").textContent = formatTokenStatus(settings); +} + +function sendDiscoverServiceMessage() { + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage({ type: "zncm-discover-service" }, (response) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message)); + return; + } + resolve(response || { ok: false, message: "服务地址识别没有返回结果。" }); + }); + }); +} + +function sendSyncTokenMessage() { + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage({ type: "zncm-sync-token" }, (response) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message)); + return; + } + resolve(response || { ok: false, message: "Token 同步没有返回结果。" }); + }); + }); +} + +async function discoverServiceNow() { + const button = document.getElementById("discoverService"); + button.disabled = true; + document.getElementById("serviceStatus").textContent = "正在识别服务地址..."; + try { + await chrome.storage.sync.set({ + autoDiscoverServiceUrl: document.getElementById("autoDiscoverServiceUrl").checked, + apiBaseUrl: deerNormalizeBaseUrl(document.getElementById("apiBaseUrl").value || DEER_CLIPPER_DEFAULTS.apiBaseUrl), + webBaseUrl: deerNormalizeBaseUrl(document.getElementById("webBaseUrl").value || DEER_CLIPPER_DEFAULTS.webBaseUrl), + }); + const response = await sendDiscoverServiceMessage(); + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + document.getElementById("apiBaseUrl").value = settings.apiBaseUrl || ""; + document.getElementById("webBaseUrl").value = settings.webBaseUrl || ""; + document.getElementById("serviceStatus").textContent = response.message || formatServiceStatus(settings); + } catch (error) { + document.getElementById("serviceStatus").textContent = error instanceof Error ? error.message : "服务地址识别失败。"; + } finally { + button.disabled = false; + } +} + +async function syncTokenNow() { + const button = document.getElementById("syncToken"); + button.disabled = true; + document.getElementById("tokenStatus").textContent = "正在同步 zncm token..."; + try { + await chrome.storage.sync.set({ + tokenSyncEnabled: document.getElementById("tokenSyncEnabled").checked, + apiBaseUrl: deerNormalizeBaseUrl(document.getElementById("apiBaseUrl").value || DEER_CLIPPER_DEFAULTS.apiBaseUrl), + webBaseUrl: deerNormalizeBaseUrl(document.getElementById("webBaseUrl").value || DEER_CLIPPER_DEFAULTS.webBaseUrl), + }); + const response = await sendSyncTokenMessage(); + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + document.getElementById("accessToken").value = settings.accessToken || ""; + document.getElementById("apiBaseUrl").value = settings.apiBaseUrl || ""; + document.getElementById("webBaseUrl").value = settings.webBaseUrl || ""; + document.getElementById("serviceStatus").textContent = formatServiceStatus(settings); + document.getElementById("tokenStatus").textContent = response.message || formatTokenStatus(settings); + } catch (error) { + document.getElementById("tokenStatus").textContent = error instanceof Error ? error.message : "Token 同步失败。"; + } finally { + button.disabled = false; + } +} + +document.getElementById("save").addEventListener("click", saveSettings); +document.getElementById("discoverService").addEventListener("click", discoverServiceNow); +document.getElementById("syncToken").addEventListener("click", syncTokenNow); +void loadSettings(); diff --git a/browser-extension/deer-web-clipper/popup.html b/browser-extension/deer-web-clipper/popup.html new file mode 100644 index 00000000..14f81811 --- /dev/null +++ b/browser-extension/deer-web-clipper/popup.html @@ -0,0 +1,31 @@ + + + + + zncm Web Clipper + + + +
+

zncm Web Clipper

+ + +
+ + + + +
+

+
+ + + + + diff --git a/browser-extension/deer-web-clipper/popup.js b/browser-extension/deer-web-clipper/popup.js new file mode 100644 index 00000000..2bf7d403 --- /dev/null +++ b/browser-extension/deer-web-clipper/popup.js @@ -0,0 +1,151 @@ +function setStatus(message) { + document.getElementById("status").textContent = message; +} + +async function getActiveTab() { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (!tab?.id) { + throw new Error("No readable active tab found."); + } + return tab; +} + +async function captureFromActiveTab(tabId) { + const [result] = await chrome.scripting.executeScript({ + target: { tabId }, + func: deerExtractCurrentPage, + }); + if (!result?.result?.content) { + throw new Error("The current page has no readable text to send."); + } + return result.result; +} + +async function submitToZncm(page, settings, prompt) { + const apiBaseUrl = deerNormalizeBaseUrl(settings.apiBaseUrl || DEER_CLIPPER_DEFAULTS.apiBaseUrl); + const response = await fetch(`${apiBaseUrl}/api/browser-context/capture`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...deerAuthHeaders(settings), + }, + body: JSON.stringify({ + title: page.title, + url: page.url, + content: page.content, + content_type: page.contentType, + prompt, + }), + }); + + const data = await response.json().catch(() => ({})); + if (!response.ok) { + const detail = typeof data.detail === "string" ? data.detail : JSON.stringify(data.detail || data); + throw new Error(detail || `HTTP ${response.status}`); + } + return data; +} + +async function sendCurrentPage() { + const sendButton = document.getElementById("send"); + sendButton.disabled = true; + setStatus("正在读取当前页面..."); + try { + await syncToken({ silent: true }); + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + const prompt = document.getElementById("prompt").value.trim() || settings.defaultPrompt || DEER_CLIPPER_DEFAULT_PROMPT; + const tab = await getActiveTab(); + const page = await captureFromActiveTab(tab.id); + setStatus("正在发送到 zncm..."); + const result = await submitToZncm(page, settings, prompt); + const webBaseUrl = deerNormalizeBaseUrl(settings.webBaseUrl || DEER_CLIPPER_DEFAULTS.webBaseUrl); + await chrome.tabs.create({ url: `${webBaseUrl}/${result.chat_url}`.replace(/\/#/g, "/#") }); + setStatus("已发送,zncm 聊天页面已打开。"); + } catch (error) { + setStatus(error instanceof Error ? error.message : "发送失败。"); + } finally { + sendButton.disabled = false; + } +} + +function sendSyncTokenMessage() { + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage({ type: "zncm-sync-token" }, (response) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message)); + return; + } + resolve(response || { ok: false, message: "Token 同步没有返回结果。" }); + }); + }); +} + +function sendPollNowMessage() { + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage({ type: "zncm-poll-now" }, (response) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message)); + return; + } + resolve(response || { ok: true, message: "Poll requested." }); + }); + }); +} + +async function syncToken(options = {}) { + const button = document.getElementById("syncToken"); + button.disabled = true; + if (!options.silent) { + setStatus("正在同步 zncm token..."); + } + try { + const response = await sendSyncTokenMessage(); + if (!options.silent) { + setStatus(response.message || "Token sync finished."); + } + return response; + } catch (error) { + if (!options.silent) { + setStatus(error instanceof Error ? error.message : "Token 同步失败。"); + } + return { ok: false, message: error instanceof Error ? error.message : "Token 同步失败。" }; + } finally { + button.disabled = false; + } +} + +async function pollNow() { + const button = document.getElementById("pollNow"); + button.disabled = true; + setStatus("正在检查 zncm 浏览器任务..."); + try { + await syncToken({ silent: true }); + const response = await sendPollNowMessage(); + setStatus(response.message || "任务检查完成。"); + } catch (error) { + setStatus(error instanceof Error ? error.message : "任务检查失败。"); + } finally { + button.disabled = false; + } +} + +async function togglePageAssistant() { + const enabled = document.getElementById("pageAssistantEnabled").checked; + await chrome.storage.sync.set({ pageAssistantEnabled: enabled }); + setStatus(enabled ? "页面问答框已开启。" : "页面问答框已关闭。"); +} + +async function init() { + const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS); + document.getElementById("prompt").value = settings.defaultPrompt || DEER_CLIPPER_DEFAULT_PROMPT; + document.getElementById("pageAssistantEnabled").checked = Boolean(settings.pageAssistantEnabled); + document.getElementById("send").addEventListener("click", sendCurrentPage); + document.getElementById("syncToken").addEventListener("click", () => syncToken()); + document.getElementById("pollNow").addEventListener("click", pollNow); + document.getElementById("pageAssistantEnabled").addEventListener("change", togglePageAssistant); + document.getElementById("options").addEventListener("click", () => chrome.runtime.openOptionsPage()); +} + +void init(); diff --git a/browser-extension/deer-web-clipper/shared.js b/browser-extension/deer-web-clipper/shared.js new file mode 100644 index 00000000..35cf3f08 --- /dev/null +++ b/browser-extension/deer-web-clipper/shared.js @@ -0,0 +1,389 @@ +const DEER_CLIPPER_DEFAULT_PROMPT = + "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002"; + +const ZNCM_EXTENSION_CONFIG = globalThis.ZNCM_WEB_CLIPPER_CONFIG || {}; +const ZNCM_DEFAULT_WEB_BASE_URL = String(ZNCM_EXTENSION_CONFIG.defaultWebBaseUrl || "").trim().replace(/\/+$/, ""); +const ZNCM_DEFAULT_API_BASE_URL = String(ZNCM_EXTENSION_CONFIG.defaultApiBaseUrl || "").trim().replace(/\/+$/, ""); + +const DEER_CLIPPER_DEFAULTS = { + apiBaseUrl: ZNCM_DEFAULT_API_BASE_URL, + webBaseUrl: ZNCM_DEFAULT_WEB_BASE_URL, + autoDiscoverServiceUrl: true, + lastServiceSyncAt: "", + lastServiceSyncSource: "", + accessToken: "", + tokenSyncEnabled: true, + lastTokenSyncAt: "", + lastTokenSyncSource: "", + defaultPrompt: DEER_CLIPPER_DEFAULT_PROMPT, + pageAssistantEnabled: true, + taskPollingEnabled: true, + workerId: "", + taskPollIntervalSeconds: 5, + taskPageLoadTimeoutSeconds: 30, + closeTaskTabs: false, +}; + +function deerNormalizeBaseUrl(value) { + return String(value || "").trim().replace(/\/+$/, ""); +} + +function deerAuthHeaders(settings) { + const token = String(settings.accessToken || "").trim(); + if (!token) { + return {}; + } + return { + Authorization: /^bearer\s+/i.test(token) ? token : `Bearer ${token}`, + }; +} + +function deerMakeWorkerId() { + return `zncm-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; +} + +function deerExtractAuthTokenFromPage() { + const storageKeys = ["deerflow.auth"]; + const tokenFieldNames = new Set(["access_token", "accessToken", "token", "id_token", "idToken"]); + + function looksLikeToken(value) { + const text = String(value || "").trim(); + return text.length > 20 && !/\s/.test(text); + } + + function findToken(value, depth = 0) { + if (value == null || depth > 4) { + return ""; + } + if (typeof value === "string") { + const text = value.trim(); + if (!text) { + return ""; + } + if ((text.startsWith("{") && text.endsWith("}")) || (text.startsWith("[") && text.endsWith("]"))) { + try { + return findToken(JSON.parse(text), depth + 1); + } catch { + return looksLikeToken(text) ? text : ""; + } + } + return looksLikeToken(text) ? text : ""; + } + if (typeof value !== "object") { + return ""; + } + for (const [key, nested] of Object.entries(value)) { + if (tokenFieldNames.has(key) && looksLikeToken(nested)) { + return String(nested).trim(); + } + } + for (const nested of Object.values(value)) { + const token = findToken(nested, depth + 1); + if (token) { + return token; + } + } + return ""; + } + + function scanStorage(storage, storageName) { + for (const key of storageKeys) { + const token = findToken(storage.getItem(key)); + if (token) { + return { token, source: `${storageName}:${key}` }; + } + } + + for (let index = 0; index < storage.length; index += 1) { + const key = storage.key(index); + if (!key || !/(auth|token|session)/i.test(key)) { + continue; + } + const token = findToken(storage.getItem(key)); + if (token) { + return { token, source: `${storageName}:${key}` }; + } + } + return null; + } + + try { + return scanStorage(window.localStorage, "localStorage") || scanStorage(window.sessionStorage, "sessionStorage"); + } catch (error) { + return { token: "", source: "", error: error instanceof Error ? error.message : String(error) }; + } +} + +function deerExtractCurrentPage() { + const maxLocalChars = 220000; + + function cleanText(value) { + return String(value || "") + .replace(/\s+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); + } + + function escapeCell(value) { + return cleanText(value).replace(/\|/g, "\\|").replace(/\n/g, " "); + } + + function tableToMarkdown(table, index) { + const rows = Array.from(table.rows || []).map((row) => + Array.from(row.cells || []).map((cell) => escapeCell(cell.innerText)), + ); + if (rows.length === 0 || rows.every((row) => row.length === 0)) { + return ""; + } + const width = Math.max(...rows.map((row) => row.length)); + const normalized = rows.map((row) => { + const next = row.slice(); + while (next.length < width) next.push(""); + return next; + }); + const header = normalized[0]; + const divider = Array.from({ length: width }, () => "---"); + const body = normalized.slice(1); + return [ + `### Table ${index + 1}`, + "", + `| ${header.join(" | ")} |`, + `| ${divider.join(" | ")} |`, + ...body.map((row) => `| ${row.join(" | ")} |`), + ].join("\n"); + } + + const clone = document.body ? document.body.cloneNode(true) : null; + if (clone) { + clone.querySelectorAll("script, style, noscript, svg, canvas, iframe").forEach((node) => node.remove()); + } + + const title = document.title || ""; + const url = location.href; + const description = document.querySelector('meta[name="description"]')?.getAttribute("content") || ""; + const main = clone?.querySelector("article, main, [role='main']") || clone; + const text = cleanText(main?.innerText || document.body?.innerText || ""); + const tables = Array.from(document.querySelectorAll("table")) + .map((table, index) => tableToMarkdown(table, index)) + .filter(Boolean); + + const parts = []; + if (description.trim()) { + parts.push(`## Meta Description\n\n${cleanText(description)}`); + } + if (text) { + parts.push(`## Page Text\n\n${text}`); + } + if (tables.length > 0) { + parts.push(`## Tables\n\n${tables.join("\n\n")}`); + } + + let content = parts.join("\n\n"); + if (content.length > maxLocalChars) { + content = `${content.slice(0, maxLocalChars)}\n\n[Truncated locally by zncm Web Clipper]`; + } + + return { title, url, content, contentType: "markdown" }; +} + +// Injected into the page to build an indexed list of interactive elements plus +// the visible page text. Each interactive element is tagged with +// `data-deer-index` so a follow-up deerPerformAction call can find it by index. +function deerObservePage(maxElements) { + const limit = Math.max(10, Number(maxElements || 150)); + const DEER_ATTR = "data-deer-index"; + + document.querySelectorAll(`[${DEER_ATTR}]`).forEach((node) => node.removeAttribute(DEER_ATTR)); + + function isVisible(el) { + if (!(el instanceof Element)) return false; + const rect = el.getBoundingClientRect(); + if (rect.width <= 1 && rect.height <= 1) return false; + const style = window.getComputedStyle(el); + if (style.visibility === "hidden" || style.display === "none" || style.opacity === "0") return false; + if (el.disabled) return false; + return true; + } + + function labelFor(el) { + const aria = el.getAttribute("aria-label"); + if (aria && aria.trim()) return aria.trim(); + const placeholder = el.getAttribute("placeholder"); + const title = el.getAttribute("title"); + const name = el.getAttribute("name"); + const text = (el.innerText || el.textContent || "").trim(); + if (text) return text.replace(/\s+/g, " "); + if (placeholder && placeholder.trim()) return placeholder.trim(); + if (title && title.trim()) return title.trim(); + if (el.value && String(el.value).trim()) return String(el.value).trim(); + if (name && name.trim()) return name.trim(); + const alt = el.querySelector?.("img[alt]")?.getAttribute("alt"); + if (alt && alt.trim()) return alt.trim(); + return ""; + } + + const selector = [ + "a[href]", + "button", + "input:not([type=hidden])", + "select", + "textarea", + "[role=button]", + "[role=link]", + "[role=tab]", + "[role=menuitem]", + "[role=checkbox]", + "[role=radio]", + "[role=switch]", + "[contenteditable=true]", + "[onclick]", + ].join(","); + + const seen = new Set(); + const elements = []; + let index = 0; + for (const el of document.querySelectorAll(selector)) { + if (index >= limit) break; + if (seen.has(el) || !isVisible(el)) continue; + seen.add(el); + el.setAttribute(DEER_ATTR, String(index)); + const tag = el.tagName.toLowerCase(); + let label = labelFor(el); + if (label.length > 200) label = label.slice(0, 200); + const entry = { index, tag, label }; + const type = el.getAttribute("type") || el.getAttribute("role"); + if (type) entry.type = type; + if (tag === "input" || tag === "textarea" || tag === "select") { + const value = String(el.value || ""); + if (value) entry.value = value.slice(0, 80); + } + if (tag === "a") { + const href = el.getAttribute("href") || ""; + if (href) entry.href = href.slice(0, 200); + } + elements.push(entry); + index += 1; + } + + // Inline text extraction so this function stays self-contained when injected + // via chrome.scripting.executeScript (no dependency on other content-script fns). + let text = ""; + try { + const clone = document.body ? document.body.cloneNode(true) : null; + if (clone) { + clone.querySelectorAll(`script, style, noscript, svg, canvas, iframe, [${DEER_ATTR}-skip]`).forEach((node) => node.remove()); + } + const main = clone?.querySelector("article, main, [role='main']") || clone; + text = String(main?.innerText || document.body?.innerText || "") + .replace(/\s+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); + } catch { + text = ""; + } + + return { + url: location.href, + title: document.title || "", + elements, + text, + }; +} + +// Injected into the page to perform a single action on the element previously +// tagged by deerObservePage (by index). Returns {status, message}. +async function deerPerformAction(action) { + const DEER_ATTR = "data-deer-index"; + const type = String(action?.type || "").toLowerCase(); + + function findByIndex(idx) { + if (idx === undefined || idx === null || idx < 0) return null; + return document.querySelector(`[${DEER_ATTR}="${idx}"]`); + } + + function fireInput(el, value) { + const proto = el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + const setter = Object.getOwnPropertyDescriptor(proto, "value")?.set; + if (setter) { + setter.call(el, value); + } else { + el.value = value; + } + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + } + + try { + if (type === "observe" || type === "extract") { + return { status: "ok", message: type === "extract" ? "Page text re-read." : "Observed page." }; + } + + if (type === "scroll") { + const el = findByIndex(action.index); + if (el) { + el.scrollIntoView({ block: "center", behavior: "instant" }); + return { status: "ok", message: "Scrolled element into view." }; + } + window.scrollBy(0, Math.round(window.innerHeight * 0.85)); + return { status: "ok", message: "Scrolled down one viewport." }; + } + + if (type === "wait") { + const selector = String(action.wait_for || "").trim(); + const timeoutMs = Math.max(500, Number(action.timeout_ms || 8000)); + if (!selector) { + await new Promise((resolve) => setTimeout(resolve, Math.min(timeoutMs, 3000))); + return { status: "ok", message: "Waited." }; + } + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + if (document.querySelector(selector)) { + return { status: "ok", message: `Element ${selector} appeared.` }; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + return { status: "timeout", message: `Element ${selector} did not appear within ${timeoutMs}ms.` }; + } + + if (type === "click") { + const el = findByIndex(action.index); + if (!el) return { status: "error", message: `No element with index ${action.index}. Re-observe the page.` }; + el.scrollIntoView({ block: "center", behavior: "instant" }); + if (typeof el.focus === "function") el.focus(); + el.click(); + return { status: "ok", message: `Clicked element ${action.index}.` }; + } + + if (type === "type") { + const el = findByIndex(action.index); + if (!el) return { status: "error", message: `No element with index ${action.index}. Re-observe the page.` }; + el.scrollIntoView({ block: "center", behavior: "instant" }); + if (typeof el.focus === "function") el.focus(); + const value = String(action.text ?? ""); + if (el.isContentEditable) { + el.textContent = value; + el.dispatchEvent(new Event("input", { bubbles: true })); + } else { + fireInput(el, value); + } + if (action.submit) { + const enterInit = { bubbles: true, cancelable: true, key: "Enter", code: "Enter", keyCode: 13, which: 13 }; + el.dispatchEvent(new KeyboardEvent("keydown", enterInit)); + el.dispatchEvent(new KeyboardEvent("keyup", enterInit)); + const form = el.form || el.closest("form"); + if (form && typeof form.requestSubmit === "function") { + try { + form.requestSubmit(); + } catch { + form.submit?.(); + } + } + } + return { status: "ok", message: `Typed into element ${action.index}${action.submit ? " and submitted" : ""}.` }; + } + + return { status: "error", message: `Unsupported action type: ${type}` }; + } catch (error) { + return { status: "error", message: error instanceof Error ? error.message : String(error) }; + } +} diff --git a/browser-extension/deer-web-clipper/styles.css b/browser-extension/deer-web-clipper/styles.css new file mode 100644 index 00000000..9b971b15 --- /dev/null +++ b/browser-extension/deer-web-clipper/styles.css @@ -0,0 +1,96 @@ +* { + box-sizing: border-box; +} + +body { + margin: 0; + color: #171717; + background: #fafafa; + font: 14px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +h1 { + margin: 0 0 14px; + font-size: 16px; +} + +.popup, +.options { + display: grid; + gap: 12px; + padding: 14px; +} + +.popup { + width: 340px; +} + +.options { + max-width: 720px; +} + +label { + display: grid; + gap: 6px; + font-weight: 600; +} + +label.checkbox { + grid-template-columns: auto 1fr; + align-items: center; + font-weight: 500; +} + +label.checkbox input { + width: auto; +} + +.actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +input, +textarea { + width: 100%; + border: 1px solid #d4d4d4; + border-radius: 6px; + padding: 8px; + background: #fff; + color: inherit; + font: inherit; + font-weight: 400; +} + +textarea { + resize: vertical; +} + +button { + border: 0; + border-radius: 6px; + padding: 9px 12px; + background: #111827; + color: #fff; + cursor: pointer; + font: inherit; + font-weight: 600; +} + +button.secondary { + background: #e5e7eb; + color: #111827; +} + +button:disabled { + cursor: not-allowed; + opacity: 0.6; +} + +.status { + min-height: 20px; + margin: 0; + color: #525252; + white-space: pre-wrap; +} diff --git a/frontend-web/src/pages/ChatPage.tsx b/frontend-web/src/pages/ChatPage.tsx index f96db7f6..c0c1290c 100644 --- a/frontend-web/src/pages/ChatPage.tsx +++ b/frontend-web/src/pages/ChatPage.tsx @@ -1,6 +1,6 @@ "use client"; -import { useRouter } from "next/navigation"; +import { useRouter, useSearchParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; @@ -41,9 +41,47 @@ import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; import { cn } from "@/lib/utils"; +const BROWSER_CONTEXT_PARAM_KEYS = [ + "browser_context", + "bc_filename", + "bc_size", + "bc_path", + "bc_prompt", +] as const; + +function clearBrowserContextParamsFromHash() { + if (typeof window === "undefined") return; + const hash = window.location.hash; + if (!hash.startsWith("#")) return; + const hashBody = hash.slice(1); + const queryStart = hashBody.indexOf("?"); + if (queryStart < 0) return; + + const path = hashBody.slice(0, queryStart); + const search = hashBody.slice(queryStart + 1); + const params = new URLSearchParams(search); + let changed = false; + for (const key of BROWSER_CONTEXT_PARAM_KEYS) { + if (params.has(key)) { + params.delete(key); + changed = true; + } + } + if (!changed) return; + + const nextSearch = params.toString(); + const nextHash = nextSearch ? `#${path}?${nextSearch}` : `#${path}`; + window.history.replaceState( + null, + "", + `${window.location.pathname}${window.location.search}${nextHash}`, + ); +} + export default function ChatPage() { const routeBase = useRouteBase(); const router = useRouter(); + const searchParams = useSearchParams(); const { t } = useI18n(); const [showFollowups, setShowFollowups] = useState(false); const [hasStartedConversation, setHasStartedConversation] = useState(false); @@ -62,6 +100,7 @@ export default function ChatPage() { const [selectedAgentId, setSelectedAgentId] = useState(null); const [agentHandoffPending, setAgentHandoffPending] = useState(false); const agentHandoffPendingRef = useRef(false); + const browserContextAutoSubmitRef = useRef(null); const { agents: selectorAgents, isLoading: selectorLoading } = useAgentSelector(); const { selectedArtifact } = useArtifacts(); @@ -242,6 +281,75 @@ export default function ChatPage() { [sendMessage, threadId], ); + useEffect(() => { + const token = searchParams.get("browser_context")?.trim(); + if (!token || browserContextAutoSubmitRef.current === token) { + return; + } + if (!threadId || isNewThread || thread.isLoading || agentHandoffPending) { + return; + } + + const consumedKey = `deerflow.browser-context.consumed.${token}`; + if (window.sessionStorage.getItem(consumedKey)) { + browserContextAutoSubmitRef.current = token; + clearBrowserContextParamsFromHash(); + return; + } + + const filename = searchParams.get("bc_filename")?.trim(); + const path = searchParams.get("bc_path")?.trim(); + const size = Number(searchParams.get("bc_size") ?? "0"); + const prompt = + searchParams.get("bc_prompt")?.trim() || + "请基于刚刚发送的网页内容,先用中文总结要点,并等待我继续提问。"; + + if (!filename || !path || !Number.isFinite(size) || size < 0) { + browserContextAutoSubmitRef.current = token; + clearBrowserContextParamsFromHash(); + toast.error("网页上下文参数不完整,无法自动发送到 Deer"); + return; + } + + browserContextAutoSubmitRef.current = token; + window.sessionStorage.setItem(consumedKey, "1"); + clearBrowserContextParamsFromHash(); + setHasStartedConversation(true); + + void sendMessage( + threadId, + { text: prompt, files: [] }, + undefined, + { + additionalKwargs: { + browser_context: { + source: "chrome-extension", + token, + }, + files: [ + { + filename, + size, + path, + status: "uploaded", + }, + ], + }, + }, + ).catch((err) => { + toast.error( + err instanceof Error ? err.message : "网页上下文发送失败,请重试", + ); + }); + }, [ + searchParams, + threadId, + isNewThread, + thread.isLoading, + agentHandoffPending, + sendMessage, + ]); + // Persist the picked agent the moment it changes so the next new chat // defaults to it — even if the user never sends a message this session. // Passing null (the ✕ clear) wipes the stored preference. diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py index 8da8c6c3..b3d2fa64 100644 --- a/offline-backend-20260512/backend/app/gateway/app.py +++ b/offline-backend-20260512/backend/app/gateway/app.py @@ -29,9 +29,10 @@ from app.gateway.routers import ( ai_writing, admin_users, agents, - artifacts, - assistants_compat, - auth, + artifacts, + assistants_compat, + auth, + browser_context, channels, checkpoint_migration, feedback, @@ -1197,11 +1198,14 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an # Artifacts API is mounted at /api/threads/{thread_id}/artifacts app.include_router(artifacts.router) - # Uploads API is mounted at /api/threads/{thread_id}/uploads - app.include_router(uploads.router) - - # Thread cleanup API is mounted at /api/threads/{thread_id} - app.include_router(threads.router) + # Uploads API is mounted at /api/threads/{thread_id}/uploads + app.include_router(uploads.router) + + # Optional Chrome-extension page capture API is mounted at /api/browser-context + app.include_router(browser_context.router) + + # Thread cleanup API is mounted at /api/threads/{thread_id} + app.include_router(threads.router) # Agents API is mounted at /api/agents app.include_router(agents.router) diff --git a/offline-backend-20260512/backend/app/gateway/routers/browser_context.py b/offline-backend-20260512/backend/app/gateway/routers/browser_context.py new file mode 100644 index 00000000..3a2237c0 --- /dev/null +++ b/offline-backend-20260512/backend/app/gateway/routers/browser_context.py @@ -0,0 +1,590 @@ +"""Browser page context capture endpoints. + +This router is intentionally isolated behind ``config.yaml``: + + browser_context: + enabled: true + +When disabled, mutating endpoints behave as if the integration is unavailable. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from datetime import UTC, datetime +from typing import Any, Literal +from urllib.parse import urlencode +from uuid import uuid4 + +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel, Field + +from app.gateway.authz import require_permission +from app.gateway.deps import get_checkpointer, get_config, get_thread_store +from app.gateway.routers.thread_runs import RunCreateRequest +from app.gateway.routers.threads import ThreadCreateRequest, create_thread +from app.gateway.routers.uploads import _make_file_sandbox_writable +from app.gateway.services import start_run +from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks +from deerflow.config.app_config import AppConfig +from deerflow.config.browser_context_config import BrowserContextConfig +from deerflow.runtime import serialize_channel_values +from deerflow.runtime.user_context import get_effective_user_id +from deerflow.sandbox.sandbox_provider import get_sandbox_provider +from deerflow.uploads.manager import ( + UnsafeUploadPathError, + ensure_uploads_dir, + normalize_filename, + open_upload_file_no_symlink, + upload_artifact_url, + upload_virtual_path, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/browser-context", tags=["browser-context"]) +DEFAULT_BROWSER_CONTEXT_PROMPT = "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002" + + +class BrowserContextStatusResponse(BaseModel): + enabled: bool + max_content_chars: int + max_title_chars: int + max_url_chars: int + task_result_timeout_seconds: int + task_poll_interval_seconds: float + task_lease_seconds: int + tool_result_max_chars: int + + +class BrowserContextCaptureRequest(BaseModel): + thread_id: str | None = Field(default=None, description="Existing or caller-generated zncm thread id") + title: str | None = Field(default=None, description="Current browser page title") + url: str | None = Field(default=None, description="Current browser page URL") + content: str = Field(description="Captured page content, preferably Markdown") + content_type: Literal["markdown", "text"] = Field(default="markdown") + prompt: str | None = Field(default=None, description="Optional first question to ask in zncm after capture") + + +class BrowserContextFileInfo(BaseModel): + filename: str + size: int + path: str + virtual_path: str + artifact_url: str + + +class BrowserContextCaptureResponse(BaseModel): + success: bool + thread_id: str + file: BrowserContextFileInfo + chat_path: str + chat_url: str + browser_context_token: str + prompt: str + truncated: bool = False + message: str + + +class BrowserContextAskRequest(BaseModel): + thread_id: str | None = Field(default=None, description="Existing zncm thread id for follow-up questions") + title: str | None = Field(default=None, description="Current browser page title") + url: str | None = Field(default=None, description="Current browser page URL") + content: str = Field(description="Captured page content, preferably Markdown") + content_type: Literal["markdown", "text"] = Field(default="markdown") + question: str = Field(description="Question asked from the in-page assistant") + + +class BrowserContextAskResponse(BaseModel): + success: bool + thread_id: str + run_id: str + answer: str + truncated: bool = False + message: str + + +class BrowserContextTaskPayload(BaseModel): + id: str + kind: str = "fetch" + url: str + instruction: str = "" + session_id: str = "" + action: dict[str, Any] | None = None + created_at: str + updated_at: str + max_content_chars: int + + +class BrowserContextTaskNextResponse(BaseModel): + task: BrowserContextTaskPayload | None = None + + +class BrowserContextTaskResultRequest(BaseModel): + kind: str = Field(default="fetch", description="Task kind the extension processed: fetch or act") + title: str | None = Field(default=None, description="Captured page title") + url: str | None = Field(default=None, description="Final page URL after redirects") + content: str | None = Field(default=None, description="Captured page content") + content_type: Literal["markdown", "text"] = Field(default="markdown") + observation: dict[str, Any] | None = Field(default=None, description="Structured observation for an act step (elements, text, status)") + error: str | None = Field(default=None, description="Extension-side task error") + metadata: dict[str, Any] = Field(default_factory=dict) + + +class BrowserContextTaskResultResponse(BaseModel): + success: bool + task_id: str + status: str + truncated: bool = False + + +def _settings(config: AppConfig) -> BrowserContextConfig: + value = getattr(config, "browser_context", None) + if isinstance(value, BrowserContextConfig): + return value + if isinstance(value, dict): + return BrowserContextConfig.model_validate(value) + return BrowserContextConfig() + + +def _require_enabled(config: AppConfig) -> BrowserContextConfig: + settings = _settings(config) + if not settings.enabled: + raise HTTPException(status_code=404, detail="Browser context integration is disabled") + return settings + + +def _trim(value: str | None, limit: int) -> str: + if not value: + return "" + return value.strip()[:limit] + + +def _limit_utf8_bytes(value: str, max_bytes: int) -> str: + text = value + while len(text.encode("utf-8")) > max_bytes: + text = text[:-1] + return text + + +def _filename_part(value: str, fallback: str, *, max_bytes: int) -> str: + text = value.strip() or fallback + # Keep unicode word chars so Chinese titles remain readable, while removing + # path separators and shell-hostile punctuation. + text = re.sub(r"[^\w.-]+", "-", text, flags=re.UNICODE).strip("-._") + text = _limit_utf8_bytes(text, max_bytes) + return text or fallback + + +def _build_markdown( + *, + title: str, + url: str, + content: str, + content_type: str, + captured_at: str, + truncated: bool, +) -> str: + lines = [ + f"# {title or 'Captured Web Page'}", + "", + f"- Source URL: {url or '(unknown)'}", + f"- Captured at: {captured_at}", + ] + if truncated: + lines.append("- Note: Content was truncated by the browser_context.max_content_chars limit.") + lines.extend(["", "---", ""]) + + if content_type == "markdown": + lines.append(content.strip()) + else: + lines.append("```text") + lines.append(content.strip()) + lines.append("```") + lines.append("") + return "\n".join(lines) + + +def _build_ask_prompt( + *, + title: str, + url: str, + content: str, + content_type: str, + captured_at: str, + truncated: bool, +) -> str: + note = "页面内容已按配置截断,请只基于可见内容回答。" if truncated else "页面内容如下。" + body = content.strip() + if content_type == "text": + body = f"```text\n{body}\n```" + return "\n\n".join( + [ + "你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。", + "如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。", + f"页面标题:{title or '未知标题'}", + f"页面 URL:{url or '未知 URL'}", + f"读取时间:{captured_at}", + f"说明:{note}", + "当前网页内容:", + body, + ] + ) + + +def _message_type(message: Any) -> str: + if isinstance(message, dict): + return str(message.get("type") or message.get("role") or "").lower() + return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower() + + +def _content_to_text(content: Any) -> str: + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + if isinstance(item.get("text"), str): + parts.append(item["text"]) + elif item.get("type") == "text" and isinstance(item.get("content"), str): + parts.append(item["content"]) + return "\n".join(part.strip() for part in parts if part and part.strip()).strip() + return "" + + +def _latest_ai_answer(values: dict[str, Any]) -> str: + messages = values.get("messages") + if not isinstance(messages, list): + return "" + for message in reversed(messages): + if _message_type(message) not in {"ai", "assistant"}: + continue + answer = _content_to_text(message.get("content") if isinstance(message, dict) else getattr(message, "content", None)) + if answer: + return answer + return "" + + +async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_path, data: bytes) -> None: + sandbox_provider = get_sandbox_provider() + if bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)): + return + sandbox_id = sandbox_provider.acquire(thread_id) + sandbox = sandbox_provider.get(sandbox_id) + if sandbox is None: + raise HTTPException(status_code=500, detail="Failed to acquire sandbox") + _make_file_sandbox_writable(file_path) + sandbox.update_file(virtual_path, data) + + +@router.get("/status", response_model=BrowserContextStatusResponse) +async def get_browser_context_status( + config: AppConfig = Depends(get_config), +) -> BrowserContextStatusResponse: + settings = _settings(config) + return BrowserContextStatusResponse( + enabled=settings.enabled, + max_content_chars=settings.max_content_chars, + max_title_chars=settings.max_title_chars, + max_url_chars=settings.max_url_chars, + task_result_timeout_seconds=settings.task_result_timeout_seconds, + task_poll_interval_seconds=settings.task_poll_interval_seconds, + task_lease_seconds=settings.task_lease_seconds, + tool_result_max_chars=settings.tool_result_max_chars, + ) + + +@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse) +@require_permission("runs", "create") +async def claim_browser_context_task( + request: Request, + worker_id: str | None = None, + config: AppConfig = Depends(get_config), +) -> BrowserContextTaskNextResponse: + settings = _require_enabled(config) + prune_browser_tasks() + task = claim_next_browser_task( + user_id=get_effective_user_id(), + worker_id=worker_id or "chrome-extension", + lease_seconds=settings.task_lease_seconds, + ) + if task is None: + return BrowserContextTaskNextResponse(task=None) + return BrowserContextTaskNextResponse(task=BrowserContextTaskPayload.model_validate(task)) + + +@router.post("/tasks/{task_id}/result", response_model=BrowserContextTaskResultResponse) +@require_permission("runs", "create") +async def submit_browser_context_task_result( + task_id: str, + body: BrowserContextTaskResultRequest, + request: Request, + config: AppConfig = Depends(get_config), +) -> BrowserContextTaskResultResponse: + settings = _require_enabled(config) + error = _trim(body.error, 4000) + content = body.content or "" + is_act = (body.kind or "fetch") == "act" + if not error and not is_act and not content.strip(): + raise HTTPException(status_code=400, detail="Captured page content is empty") + if not error and is_act and not isinstance(body.observation, dict): + raise HTTPException(status_code=400, detail="Act task result requires an observation") + + task = complete_browser_task( + task_id=task_id, + user_id=get_effective_user_id(), + title=_trim(body.title, settings.max_title_chars), + url=_trim(body.url, settings.max_url_chars), + content=content, + content_type=body.content_type, + error=error, + metadata=body.metadata, + observation=body.observation, + ) + if task is None: + raise HTTPException(status_code=404, detail=f"Browser task {task_id} not found") + + result = task.get("result") or {} + return BrowserContextTaskResultResponse( + success=True, + task_id=task_id, + status=str(task.get("status") or "unknown"), + truncated=bool(result.get("truncated")), + ) + + +@router.post("/capture", response_model=BrowserContextCaptureResponse) +@require_permission("threads", "write") +async def capture_browser_context( + body: BrowserContextCaptureRequest, + request: Request, + config: AppConfig = Depends(get_config), +) -> BrowserContextCaptureResponse: + settings = _require_enabled(config) + + raw_content = (body.content or "").strip() + if not raw_content: + raise HTTPException(status_code=400, detail="Captured page content is empty") + + title = _trim(body.title, settings.max_title_chars) + url = _trim(body.url, settings.max_url_chars) + truncated = len(raw_content) > settings.max_content_chars + content = raw_content[: settings.max_content_chars] if truncated else raw_content + + requested_thread_id = (body.thread_id or "").strip() or None + if requested_thread_id: + thread_store = get_thread_store(request) + allowed = await thread_store.check_access( + requested_thread_id, + get_effective_user_id(), + require_existing=False, + ) + if not allowed: + raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found") + + thread_response = await create_thread( + ThreadCreateRequest( + thread_id=requested_thread_id, + metadata={ + "source": "browser_context", + "browser_context_url": url, + "browser_context_title": title, + }, + ), + request, + ) + thread_id = thread_response.thread_id + + captured_at = datetime.now(UTC).isoformat() + markdown = _build_markdown( + title=title, + url=url, + content=content, + content_type=body.content_type, + captured_at=captured_at, + truncated=truncated, + ) + data = markdown.encode("utf-8") + + timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + prefix = _filename_part(settings.filename_prefix, "webpage", max_bytes=40) + title_part = _filename_part(title, "page", max_bytes=120) + filename = normalize_filename(f"{prefix}-{timestamp}-{title_part}.md") + + try: + uploads_dir = ensure_uploads_dir(thread_id) + file_path, fh = open_upload_file_no_symlink(uploads_dir, filename) + with fh: + fh.write(data) + except UnsafeUploadPathError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + logger.exception("Failed to save browser context capture for thread %s", thread_id) + raise HTTPException(status_code=500, detail="Failed to save captured page") from exc + + virtual_path = upload_virtual_path(filename) + await _sync_to_sandbox_if_needed(thread_id, virtual_path, file_path, data) + + prompt = (body.prompt or "").strip() or DEFAULT_BROWSER_CONTEXT_PROMPT + token = uuid4().hex + route_base = settings.route_base.rstrip("/") or "/page/workspace" + chat_path = f"{route_base}/chats/{thread_id}" + query = urlencode( + { + "browser_context": token, + "bc_filename": filename, + "bc_size": str(len(data)), + "bc_path": virtual_path, + "bc_prompt": prompt, + } + ) + chat_url = f"#{chat_path}?{query}" + + return BrowserContextCaptureResponse( + success=True, + thread_id=thread_id, + file=BrowserContextFileInfo( + filename=filename, + size=len(data), + path=virtual_path, + virtual_path=virtual_path, + artifact_url=upload_artifact_url(thread_id, filename), + ), + chat_path=chat_path, + chat_url=chat_url, + browser_context_token=token, + prompt=prompt, + truncated=truncated, + message="Captured page saved as a zncm upload", + ) + + +@router.post("/ask", response_model=BrowserContextAskResponse) +@require_permission("runs", "create") +async def ask_browser_context( + body: BrowserContextAskRequest, + request: Request, + config: AppConfig = Depends(get_config), +) -> BrowserContextAskResponse: + settings = _require_enabled(config) + + raw_content = (body.content or "").strip() + if not raw_content: + raise HTTPException(status_code=400, detail="Captured page content is empty") + + question = (body.question or "").strip() + if not question: + raise HTTPException(status_code=400, detail="Question is empty") + + title = _trim(body.title, settings.max_title_chars) + url = _trim(body.url, settings.max_url_chars) + content_limit = max(1_000, min(settings.max_content_chars, settings.tool_result_max_chars)) + truncated = len(raw_content) > content_limit + content = raw_content[:content_limit] if truncated else raw_content + + requested_thread_id = (body.thread_id or "").strip() or None + if requested_thread_id: + thread_store = get_thread_store(request) + allowed = await thread_store.check_access( + requested_thread_id, + get_effective_user_id(), + require_existing=False, + ) + if not allowed: + raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found") + + thread_response = await create_thread( + ThreadCreateRequest( + thread_id=requested_thread_id, + assistant_id="lead_agent", + metadata={ + "source": "browser_context_page_assistant", + "browser_context_url": url, + "browser_context_title": title, + }, + ), + request, + ) + thread_id = thread_response.thread_id + captured_at = datetime.now(UTC).isoformat() + is_follow_up = requested_thread_id is not None + if is_follow_up: + prompt_prefix = "\n\n".join( + [ + "你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。", + "如果本会话里的网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。", + f"当前页面标题:{title or '未知标题'}", + f"当前页面 URL:{url or '未知 URL'}", + f"本轮提问时间:{captured_at}", + ] + ) + else: + prompt_prefix = _build_ask_prompt( + title=title, + url=url, + content=content, + content_type=body.content_type, + captured_at=captured_at, + truncated=truncated, + ) + + run_body = RunCreateRequest( + assistant_id="lead_agent", + input={ + "messages": [ + { + "type": "human", + "content": question, + "additional_kwargs": { + "prompt_prefix": prompt_prefix, + }, + } + ] + }, + metadata={ + "source": "browser_context_page_assistant", + "browser_context_url": url, + "browser_context_title": title, + "browser_context_truncated": truncated, + }, + stream_mode=["values"], + on_disconnect="continue", + multitask_strategy="reject", + ) + + record = await start_run(run_body, thread_id, request) + if record.task is not None: + try: + await record.task + except asyncio.CancelledError: + pass + + checkpointer = get_checkpointer(request) + values: dict[str, Any] = {} + try: + checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}}) + if checkpoint_tuple is not None: + checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {} + values = serialize_channel_values(checkpoint.get("channel_values", {}) or {}) + except Exception: + logger.exception("Failed to fetch browser context answer for run %s", record.run_id) + + answer = _latest_ai_answer(values) + if not answer: + answer = record.error or "zncm did not return an answer." + + return BrowserContextAskResponse( + success=bool(answer), + thread_id=thread_id, + run_id=record.run_id, + answer=answer, + truncated=truncated, + message="zncm page assistant answer completed", + ) diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index 8f727d9c..95c205dc 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -10,90 +10,90 @@ gateway: port: 8001 models: - - name: deepseek-chat - display_name: DeepSeek Chat - use: langchain_openai:ChatOpenAI - model: deepseek-chat - api_key: sk-1520028668cb4cc388ed8315ff1e856e - base_url: https://api.deepseek.com - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: false - supports_thinking: true + # - name: deepseek-chat + # display_name: DeepSeek Chat + # use: langchain_openai:ChatOpenAI + # model: deepseek-chat + # api_key: sk-1520028668cb4cc388ed8315ff1e856e + # base_url: https://api.deepseek.com + # request_timeout: 600.0 + # max_retries: 2 + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: false + # supports_thinking: true - - name: deepseek-v4-flash - display_name: DeepSeek V4 Flash - use: langchain_openai:ChatOpenAI - model: deepseek-v4-flash - api_key: sk-e1c22b7931dc446f8b67e57878edd105 - base_url: https://api.deepseek.com - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: false - supports_thinking: true + # - name: deepseek-v4-flash + # display_name: DeepSeek V4 Flash + # use: langchain_openai:ChatOpenAI + # model: deepseek-v4-flash + # api_key: sk-e1c22b7931dc446f8b67e57878edd105 + # base_url: https://api.deepseek.com + # request_timeout: 600.0 + # max_retries: 2 + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: false + # supports_thinking: true - - name: deepseek-v4-pro - display_name: DeepSeek V4 Pro + # - name: deepseek-v4-pro + # display_name: DeepSeek V4 Pro + # use: langchain_openai:ChatOpenAI + # model: deepseek-v4-pro + # api_key: sk-e1c22b7931dc446f8b67e57878edd105 + # base_url: https://api.deepseek.com + # request_timeout: 600.0 + # max_retries: 2 + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: false + # supports_thinking: true + # - name: MiniMax-M2.7 + # display_name: MiniMax-M2.7 + # use: langchain_openai:ChatOpenAI + # model: MiniMax-M2.7 + # api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk + # base_url: https://api.minimaxi.com/v1 + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: true + # supports_thinking: true + # - name: glm-5.1 + # display_name: glm-5.1 + # use: langchain_openai:ChatOpenAI + # model: glm-5.1 + # api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR + # base_url: https://open.bigmodel.cn/api/paas/v4/ + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: true + # supports_thinking: true + # - name: deepseek-ai/DeepSeek-OCR + # display_name: deepseek-ai/DeepSeek-OCR + # use: langchain_openai:ChatOpenAI + # model: deepseek-ai/DeepSeek-OCR + # api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb + # base_url: https://api.siliconflow.cn/v1/ + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: true + # supports_thinking: true + # - name: zai-org/GLM-5-FP8 + # display_name: zai-org/GLM-5-FP8 + # use: langchain_openai:ChatOpenAI + # model: zai-org/GLM-5-FP8 + # api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA + # base_url: https://api.us-west-2.modal.direct/v1 + # request_timeout: 600.0 + # max_retries: 2 + # max_tokens: 4096 + # temperature: 1.0 + # supports_vision: true + # supports_thinking: true + - name: deepseek-ai/DeepSeek-V4-Flash + display_name: deepseek-ai/DeepSeek-V4-Flash use: langchain_openai:ChatOpenAI - model: deepseek-v4-pro - api_key: sk-e1c22b7931dc446f8b67e57878edd105 - base_url: https://api.deepseek.com - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: false - supports_thinking: true - - name: MiniMax-M2.7 - display_name: MiniMax-M2.7 - use: langchain_openai:ChatOpenAI - model: MiniMax-M2.7 - api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk - base_url: https://api.minimaxi.com/v1 - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true - - name: glm-5.1 - display_name: glm-5.1 - use: langchain_openai:ChatOpenAI - model: glm-5.1 - api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR - base_url: https://open.bigmodel.cn/api/paas/v4/ - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true - - name: deepseek-ai/DeepSeek-OCR - display_name: deepseek-ai/DeepSeek-OCR - use: langchain_openai:ChatOpenAI - model: deepseek-ai/DeepSeek-OCR - api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb - base_url: https://api.siliconflow.cn/v1/ - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true - - name: zai-org/GLM-5-FP8 - display_name: zai-org/GLM-5-FP8 - use: langchain_openai:ChatOpenAI - model: zai-org/GLM-5-FP8 - api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA - base_url: https://api.us-west-2.modal.direct/v1 - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true - - name: Qwen/Qwen3.6-27B - display_name: Qwen/Qwen3.6-27B - use: langchain_openai:ChatOpenAI - model: Qwen/Qwen3.6-27B + model: deepseek-ai/DeepSeek-V4-Flash api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt base_url: https://api.siliconflow.cn/v1/ max_tokens: 4096 @@ -122,6 +122,26 @@ uploads: auto_convert_documents: true pdf_converter: auto +# Chrome 插件「当前网页发送到 Deer」入口。默认关闭,打开后才启用 +# /api/browser-context/capture。它只保存当前页 Markdown 到线程 uploads, +# 不会改变普通聊天、上传或工具调用流程。 +browser_context: + enabled: true + max_content_chars: 200000 + max_title_chars: 300 + max_url_chars: 2048 + filename_prefix: webpage + route_base: /page/workspace + task_result_timeout_seconds: 60 + task_poll_interval_seconds: 1 + task_lease_seconds: 120 + tool_result_max_chars: 60000 + # 交互式页面操作(browser_act:点击/输入/滚动/等待/跳转)。关闭后只保留只读抓取。 + actions_enabled: true + action_result_timeout_seconds: 45 + max_observed_elements: 150 + observation_text_max_chars: 12000 + title: enabled: true max_words: 6 @@ -139,6 +159,7 @@ title: tool_groups: - name: web + - name: browser - name: file:read - name: file:write - name: bash @@ -154,6 +175,14 @@ tools: use: deerflow.community.jina_ai.tools:web_fetch_tool timeout: 15 + - name: browser_fetch_page + group: browser + use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool + + - name: browser_act + group: browser + use: deerflow.tools.builtins.browser_act_tool:browser_act_tool + - name: ls group: file:read use: deerflow.sandbox.tools:ls_tool diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py new file mode 100644 index 00000000..4fcd9227 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py @@ -0,0 +1,2 @@ +"""Browser-backed page context helpers.""" + diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py new file mode 100644 index 00000000..9a8d2f51 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py @@ -0,0 +1,303 @@ +"""Small file-backed task queue for browser page fetch requests.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +import threading +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from deerflow.config.runtime_paths import runtime_home + +logger = logging.getLogger(__name__) + +_LOCK = threading.RLock() +_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$") +_TERMINAL_STATUSES = {"done", "error", "expired"} + + +def tasks_dir() -> Path: + """Return the browser-context task directory, creating it if needed.""" + + path = runtime_home() / "browser-context" / "tasks" + path.mkdir(parents=True, exist_ok=True) + return path + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _iso(value: datetime | None = None) -> str: + return (value or _now()).isoformat() + + +def _parse_iso(value: Any) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _task_path(task_id: str) -> Path | None: + if not _TASK_ID_RE.fullmatch(task_id or ""): + return None + return tasks_dir() / f"{task_id}.json" + + +def _read_task(path: Path) -> dict[str, Any] | None: + try: + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + except FileNotFoundError: + return None + except Exception: + logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True) + return None + return data if isinstance(data, dict) else None + + +def _write_task(path: Path, data: dict[str, Any]) -> None: + tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp") + with tmp.open("w", encoding="utf-8") as fh: + json.dump(data, fh, ensure_ascii=False, separators=(",", ":")) + tmp.replace(path) + + +def _public_task(data: dict[str, Any]) -> dict[str, Any]: + return { + "id": data.get("id"), + "kind": data.get("kind") or "fetch", + "url": data.get("url") or "", + "instruction": data.get("instruction") or "", + "session_id": data.get("session_id") or "", + "action": data.get("action") or None, + "created_at": data.get("created_at") or "", + "updated_at": data.get("updated_at") or "", + "max_content_chars": int(data.get("max_content_chars") or 200_000), + } + + +def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]: + if data.get("status") in _TERMINAL_STATUSES: + return data + expires_at = _parse_iso(data.get("expires_at")) + if expires_at is not None and now >= expires_at: + data = dict(data) + data["status"] = "expired" + data["error"] = "Browser task expired before the extension returned a result." + data["updated_at"] = _iso(now) + _write_task(path, data) + return data + + +def prune_browser_tasks(max_age_seconds: int = 86_400) -> int: + """Delete old terminal tasks and return the number removed.""" + + cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds))) + removed = 0 + with _LOCK: + for path in tasks_dir().glob("*.json"): + data = _read_task(path) + if not data or data.get("status") not in _TERMINAL_STATUSES: + continue + updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at")) + if updated_at is None or updated_at < cutoff: + try: + path.unlink() + removed += 1 + except OSError: + logger.debug("Failed to delete old browser task: %s", path, exc_info=True) + return removed + + +def create_browser_task( + *, + user_id: str, + url: str, + instruction: str = "", + timeout_seconds: int = 60, + max_content_chars: int = 200_000, + metadata: dict[str, Any] | None = None, + kind: str = "fetch", + session_id: str = "", + action: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Create a browser task for the current user's extension to pick up. + + ``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or + ``"act"`` for an interactive observe/act step driven by ``action`` within a + long-lived ``session_id`` (a tab kept open by the extension). + """ + + now = _now() + task_id = uuid4().hex + task = { + "id": task_id, + "user_id": str(user_id or "default"), + "status": "pending", + "kind": kind if kind in {"fetch", "act"} else "fetch", + "url": url, + "instruction": instruction or "", + "session_id": str(session_id or ""), + "action": action or None, + "created_at": _iso(now), + "updated_at": _iso(now), + "expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))), + "max_content_chars": max(1_000, int(max_content_chars)), + "worker_id": None, + "claimed_at": None, + "result": None, + "error": None, + "metadata": metadata or {}, + } + path = tasks_dir() / f"{task_id}.json" + with _LOCK: + _write_task(path, task) + return task + + +def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None: + """Read a task if it belongs to the requested user.""" + + path = _task_path(task_id) + if path is None: + return None + with _LOCK: + data = _read_task(path) + if not data or str(data.get("user_id") or "") != str(user_id or "default"): + return None + return _expire_if_needed(path, data, _now()) + + +def claim_next_browser_task( + *, + user_id: str, + worker_id: str, + lease_seconds: int = 120, +) -> dict[str, Any] | None: + """Claim the oldest pending task for a user.""" + + now = _now() + user_id = str(user_id or "default") + worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension" + lease_seconds = max(10, int(lease_seconds)) + + with _LOCK: + paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime) + for path in paths: + data = _read_task(path) + if not data or str(data.get("user_id") or "") != user_id: + continue + + data = _expire_if_needed(path, data, now) + status = data.get("status") + if status == "running": + updated_at = _parse_iso(data.get("updated_at")) or now + if (now - updated_at).total_seconds() <= lease_seconds: + continue + status = "pending" + data = dict(data) + data["status"] = status + + if status != "pending": + continue + + data = dict(data) + data["status"] = "running" + data["worker_id"] = worker_id + data["claimed_at"] = _iso(now) + data["updated_at"] = _iso(now) + _write_task(path, data) + return _public_task(data) + + return None + + +def complete_browser_task( + *, + task_id: str, + user_id: str, + title: str = "", + url: str = "", + content: str = "", + content_type: str = "markdown", + error: str = "", + metadata: dict[str, Any] | None = None, + observation: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """Mark a browser task complete or failed. + + For ``act`` tasks the extension returns a structured ``observation`` (final + URL/title, interactive element list, page text) instead of a captured + markdown document; it is stored alongside the result verbatim. + """ + + path = _task_path(task_id) + if path is None: + return None + + now = _now() + with _LOCK: + data = _read_task(path) + if not data or str(data.get("user_id") or "") != str(user_id or "default"): + return None + + data = dict(data) + if error: + data["status"] = "error" + data["error"] = str(error).strip()[:4000] + data["result"] = None + else: + raw_content = str(content or "").strip() + max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000)) + truncated = len(raw_content) > max_content_chars + data["status"] = "done" + data["error"] = None + data["result"] = { + "title": str(title or "").strip()[:500], + "url": str(url or data.get("url") or "").strip()[:4096], + "content": raw_content[:max_content_chars] if truncated else raw_content, + "content_type": content_type if content_type in {"markdown", "text"} else "markdown", + "truncated": truncated, + "observation": observation if isinstance(observation, dict) else None, + "captured_at": _iso(now), + "metadata": metadata or {}, + } + data["updated_at"] = _iso(now) + _write_task(path, data) + return data + + +async def wait_for_browser_task_result( + *, + task_id: str, + user_id: str, + timeout_seconds: int, + poll_interval_seconds: float = 1.0, +) -> dict[str, Any] | None: + """Wait for a browser task to reach a terminal state.""" + + deadline = time.monotonic() + max(1, int(timeout_seconds)) + interval = max(0.25, float(poll_interval_seconds)) + while True: + task = get_browser_task(task_id=task_id, user_id=user_id) + if task is None: + return None + if task.get("status") in _TERMINAL_STATUSES: + return task + if time.monotonic() >= deadline: + return task + await asyncio.sleep(interval) + diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py index f1494bb6..eb35ce1d 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py @@ -9,8 +9,9 @@ from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, Field from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict -from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict -from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict +from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict +from deerflow.config.browser_context_config import BrowserContextConfig +from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict from deerflow.config.database_config import DatabaseConfig from deerflow.config.extensions_config import ExtensionsConfig from deerflow.config.gateway_config import GatewayConfig @@ -132,7 +133,8 @@ class AppConfig(BaseModel): gateway: GatewayConfig = Field(default_factory=GatewayConfig, description="Gateway FastAPI listening port (used by intra-process loopback)") run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration") checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration") - stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration") + stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration") + browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration") @classmethod def resolve_config_path(cls, config_path: str | None = None) -> Path: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py new file mode 100644 index 00000000..fdbfed8b --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py @@ -0,0 +1,74 @@ +"""Configuration for browser page context capture.""" + +from pydantic import BaseModel, Field + + +class BrowserContextConfig(BaseModel): + """Controls the optional Chrome-extension page capture integration.""" + + enabled: bool = Field( + default=False, + description="Enable /api/browser-context endpoints for Chrome extension page capture.", + ) + max_content_chars: int = Field( + default=200_000, + ge=1_000, + description="Maximum captured page text stored per request. Longer content is truncated.", + ) + max_title_chars: int = Field( + default=300, + ge=20, + description="Maximum title length accepted from the browser extension.", + ) + max_url_chars: int = Field( + default=2_048, + ge=100, + description="Maximum source URL length accepted from the browser extension.", + ) + filename_prefix: str = Field( + default="webpage", + description="Prefix used for Markdown files created from captured pages.", + ) + route_base: str = Field( + default="/page/workspace", + description="Frontend route base used when returning chat paths to the extension.", + ) + task_result_timeout_seconds: int = Field( + default=60, + ge=5, + description="Maximum time the browser_fetch_page tool waits for the extension result.", + ) + task_poll_interval_seconds: float = Field( + default=1.0, + ge=0.25, + description="How often the backend tool checks for a returned browser task result.", + ) + task_lease_seconds: int = Field( + default=120, + ge=10, + description="How long a claimed browser task is reserved before another extension may retry it.", + ) + tool_result_max_chars: int = Field( + default=60_000, + ge=1_000, + description="Maximum page characters returned from browser_fetch_page into the model context.", + ) + actions_enabled: bool = Field( + default=True, + description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).", + ) + action_result_timeout_seconds: int = Field( + default=45, + ge=5, + description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.", + ) + max_observed_elements: int = Field( + default=150, + ge=10, + description="Maximum number of interactive elements returned in a browser_act observation.", + ) + observation_text_max_chars: int = Field( + default=12_000, + ge=500, + description="Maximum page text characters returned alongside a browser_act observation.", + ) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py new file mode 100644 index 00000000..ed6797ae --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py @@ -0,0 +1,225 @@ +"""Interactive browser action tool backed by the user's Chrome extension. + +Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an +*observe -> act* loop on a live page the extension keeps open: + +1. The model calls ``browser_act`` with an action (navigate/click/type/...). +2. The extension performs the action on the session tab, then re-scans the page + and returns an *observation*: the final URL/title, an indexed list of + interactive elements, and the visible page text. +3. The model reads the observation, picks the next element by its index, and + calls ``browser_act`` again with the same ``session_id``. + +The element ``index`` values are only valid for the most recent observation of a +session; always act on indices from the latest result. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse +from uuid import uuid4 + +from langchain.tools import tool + +from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result +from deerflow.config import get_app_config +from deerflow.config.browser_context_config import BrowserContextConfig +from deerflow.runtime.user_context import get_effective_user_id + +_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"} +_NEEDS_INDEX = {"click", "type"} + + +def _settings() -> BrowserContextConfig: + value = getattr(get_app_config(), "browser_context", None) + if isinstance(value, BrowserContextConfig): + return value + if isinstance(value, dict): + return BrowserContextConfig.model_validate(value) + return BrowserContextConfig() + + +def _validate_url(raw_url: str) -> str | None: + url = (raw_url or "").strip() + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + return None + return url + + +def _format_elements(elements: list[dict[str, Any]] | None) -> str: + if not elements: + return "(no interactive elements detected)" + lines: list[str] = [] + for el in elements: + idx = el.get("index") + tag = str(el.get("tag") or "?") + el_type = str(el.get("type") or "").strip() + label = str(el.get("label") or "").strip().replace("\n", " ") + if len(label) > 120: + label = label[:120] + "…" + descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "") + extra = "" + if el.get("value"): + value = str(el["value"]).strip().replace("\n", " ") + extra = f' value="{value[:60]}"' + lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip()) + return "\n".join(lines) + + +def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str: + observation = observation or {} + status = str(observation.get("action_status") or "ok") + message = str(observation.get("message") or "").strip() + url = str(observation.get("url") or "").strip() or "(unknown)" + title = str(observation.get("title") or "").strip() or "(untitled)" + elements = observation.get("elements") if isinstance(observation.get("elements"), list) else [] + text = str(observation.get("text") or "").strip() + text_limit = max(500, int(settings.observation_text_max_chars)) + text_truncated = len(text) > text_limit + if text_truncated: + text = text[:text_limit] + + parts = [ + f"Browser session: {session_id}", + f"Last action: {action} -> {status}", + ] + if message: + parts.append(f"Note: {message}") + parts.extend( + [ + f"URL: {url}", + f"Title: {title}", + "", + "Interactive elements (use the [index] with click/type):", + _format_elements(elements), + "", + "Page text:", + text or "(empty page text)", + ] + ) + if text_truncated: + parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]") + parts.append( + "\nNext step: call browser_act again with this session_id and the index of the element to act on. " + "When the task is finished, call browser_act with action=\"close\" to release the tab." + ) + return "\n".join(parts) + + +@tool("browser_act", parse_docstring=True) +async def browser_act_tool( + action: str, + session_id: str = "", + url: str = "", + index: int = -1, + text: str = "", + submit: bool = False, + wait_for: str = "", + wait_seconds: int = 0, +) -> str: + """Drive an interactive observe/act loop on a page in the user's Chrome browser. + + Use this to operate a real web page on the user's behalf — log in, fill forms, + click through navigation, and read pages that need the user's session or local + network. Start every task with ``action="navigate"`` and a URL to open a tab + and get the first observation, then keep calling this tool with the same + ``session_id``, choosing elements by the ``index`` shown in the observation. + Call ``action="close"`` when done to free the tab. + + Args: + action: One of navigate, click, type, scroll, wait, extract, observe, close. + session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url). + url: Target http(s) URL. Required for navigate (and to start a new session). + index: Index of the interactive element to act on, taken from the latest observation. Required for click and type. + text: Text to enter for the type action. + submit: For type, press Enter / submit the form after typing. + wait_for: For wait, a CSS selector to wait for before observing. + wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default. + """ + + settings = _settings() + if not settings.enabled: + return "browser_act is disabled by config.yaml: browser_context.enabled is false." + if not settings.actions_enabled: + return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false." + + action = (action or "").strip().lower() + if action not in _ACTIONS: + return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}." + + session_id = (session_id or "").strip() + safe_url = _validate_url(url) if url else None + + if action == "navigate" and safe_url is None: + return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page." + + if not session_id: + if action != "navigate": + return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first." + session_id = uuid4().hex + + if action in _NEEDS_INDEX and index < 0: + return f"browser_act {action} requires a non-negative element index from the latest observation." + if action == "type" and not text: + return "browser_act type requires the text to enter." + + action_payload: dict[str, Any] = {"type": action} + if safe_url is not None: + action_payload["url"] = safe_url + if index >= 0: + action_payload["index"] = int(index) + if text: + action_payload["text"] = text + if submit: + action_payload["submit"] = True + if wait_for.strip(): + action_payload["wait_for"] = wait_for.strip()[:300] + + configured_timeout = max(5, int(settings.action_result_timeout_seconds)) + requested_timeout = int(wait_seconds or configured_timeout) + timeout = min(max(5, requested_timeout), configured_timeout) + user_id = get_effective_user_id() + + task = create_browser_task( + user_id=user_id, + url=safe_url or "", + instruction="", + kind="act", + session_id=session_id, + action=action_payload, + timeout_seconds=timeout + int(settings.task_lease_seconds), + max_content_chars=settings.max_content_chars, + metadata={"source": "browser_act_tool"}, + ) + + completed = await wait_for_browser_task_result( + task_id=task["id"], + user_id=user_id, + timeout_seconds=timeout, + poll_interval_seconds=settings.task_poll_interval_seconds, + ) + if completed is None: + return f"Browser task {task['id']} was not found after it was created." + + status = completed.get("status") + if status == "error": + return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}" + if status == "expired": + return ( + f"browser_act session {session_id}: the Chrome extension did not respond in time. " + "Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled." + ) + if status != "done": + return ( + f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. " + "Keep the Deer Web Clipper extension running with task polling enabled, then try again." + ) + + if action == "close": + return f"browser_act session {session_id}: tab closed." + + result = completed.get("result") or {} + observation = result.get("observation") if isinstance(result.get("observation"), dict) else None + return _format_observation(action, observation, session_id, settings) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py new file mode 100644 index 00000000..6b065834 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py @@ -0,0 +1,117 @@ +"""Browser extension backed page fetch tool.""" + +from __future__ import annotations + +from urllib.parse import urlparse + +from langchain.tools import tool + +from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result +from deerflow.config import get_app_config +from deerflow.config.browser_context_config import BrowserContextConfig +from deerflow.runtime.user_context import get_effective_user_id + + +def _settings() -> BrowserContextConfig: + value = getattr(get_app_config(), "browser_context", None) + if isinstance(value, BrowserContextConfig): + return value + if isinstance(value, dict): + return BrowserContextConfig.model_validate(value) + return BrowserContextConfig() + + +def _validate_url(raw_url: str) -> str | None: + url = (raw_url or "").strip() + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + return None + return url + + +@tool("browser_fetch_page", parse_docstring=True) +async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str: + """Ask the user's paired Chrome extension to fetch a page and return visible page text. + + Use this when the user explicitly asks to use the browser extension or asks + for data from a page that may require their browser session, login state, or + local network access. The URL must be an exact http(s) URL provided by the + user. If the user did not provide a URL, ask for the URL instead of guessing. + + Args: + url: Exact http(s) URL to open in the user's Chrome browser. + question: Optional instruction describing what data the user wants from the page. + wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default. + """ + + settings = _settings() + if not settings.enabled: + return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false." + + safe_url = _validate_url(url) + if safe_url is None: + return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page." + + configured_timeout = max(5, int(settings.task_result_timeout_seconds)) + requested_timeout = int(wait_seconds or configured_timeout) + timeout = min(max(5, requested_timeout), configured_timeout) + user_id = get_effective_user_id() + + task = create_browser_task( + user_id=user_id, + url=safe_url, + instruction=(question or "").strip(), + timeout_seconds=timeout + int(settings.task_lease_seconds), + max_content_chars=settings.max_content_chars, + metadata={"source": "browser_fetch_page_tool"}, + ) + + completed = await wait_for_browser_task_result( + task_id=task["id"], + user_id=user_id, + timeout_seconds=timeout, + poll_interval_seconds=settings.task_poll_interval_seconds, + ) + if completed is None: + return f"Browser task {task['id']} was not found after it was created." + + status = completed.get("status") + if status == "error": + return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}" + if status == "expired": + return ( + f"Browser task {task['id']} expired before the Chrome extension returned a result. " + "Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled." + ) + if status != "done": + return ( + f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. " + "Keep the Deer Web Clipper extension running with task polling enabled, then try again." + ) + + result = completed.get("result") or {} + content = str(result.get("content") or "").strip() + tool_limit = max(1_000, int(settings.tool_result_max_chars)) + tool_truncated = len(content) > tool_limit + if tool_truncated: + content = content[:tool_limit] + + notes: list[str] = [] + if result.get("truncated"): + notes.append("The browser capture was truncated by browser_context.max_content_chars.") + if tool_truncated: + notes.append("The tool response was truncated by browser_context.tool_result_max_chars.") + + parts = [ + "Browser page fetched through the user's Chrome extension.", + f"Task ID: {task['id']}", + f"Title: {result.get('title') or '(untitled)'}", + f"URL: {result.get('url') or safe_url}", + ] + if question: + parts.extend(["", "Requested extraction:", question.strip()]) + if notes: + parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)]) + parts.extend(["", "Page content:", content or "(empty page text)"]) + return "\n".join(parts) + diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py index 8fa87aac..3c81c44d 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py @@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry logger = logging.getLogger(__name__) +BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool" +BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool" +BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE} +BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"} + BUILTIN_TOOLS = [ present_file_tool, ask_clarification_tool, @@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool: return False +def _is_browser_context_tool(tool: object) -> bool: + """Return True if the tool config represents the browser extension bridge.""" + + use = getattr(tool, "use", None) + name = getattr(tool, "name", None) + return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES + + def get_available_tools( groups: list[str] | None = None, include_mcp: bool = True, @@ -64,6 +77,18 @@ def get_available_tools( config = app_config or get_app_config() tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups] + browser_context_config = getattr(config, "browser_context", None) + if not getattr(browser_context_config, "enabled", False): + tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)] + elif not getattr(browser_context_config, "actions_enabled", True): + # browser_context is on, but interactive page actions are disabled — keep + # the read-only browser_fetch_page tool, drop only browser_act. + tool_configs = [ + tool + for tool in tool_configs + if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act" + ] + # Do not expose host bash by default when LocalSandboxProvider is active. if not is_host_bash_allowed(config): tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)] diff --git a/offline-backend-20260512/backend/tests/test_browser_act.py b/offline-backend-20260512/backend/tests/test_browser_act.py new file mode 100644 index 00000000..3dee216a --- /dev/null +++ b/offline-backend-20260512/backend/tests/test_browser_act.py @@ -0,0 +1,158 @@ +"""Tests for the interactive browser_act tool and its act-task queue support.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from deerflow.browser_context import tasks as task_queue +from deerflow.config.browser_context_config import BrowserContextConfig +from deerflow.tools.builtins import browser_act_tool as act_mod + + +@pytest.fixture(autouse=True) +def _isolated_task_home(tmp_path, monkeypatch): + """Point the file-backed task queue at a throwaway directory.""" + monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path)) + yield + + +def _enable_actions(monkeypatch, *, enabled=True, actions_enabled=True): + cfg = BrowserContextConfig( + enabled=enabled, + actions_enabled=actions_enabled, + action_result_timeout_seconds=5, + task_poll_interval_seconds=0.25, + task_lease_seconds=30, + ) + monkeypatch.setattr(act_mod, "_settings", lambda: cfg) + return cfg + + +# --- task queue layer ------------------------------------------------------- + + +def test_act_task_round_trip_carries_session_and_observation(): + task = task_queue.create_browser_task( + user_id="u1", + url="https://example.com", + kind="act", + session_id="sess-1", + action={"type": "navigate", "url": "https://example.com"}, + ) + assert task["kind"] == "act" + assert task["session_id"] == "sess-1" + + claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1") + assert claimed is not None + assert claimed["kind"] == "act" + assert claimed["session_id"] == "sess-1" + assert claimed["action"] == {"type": "navigate", "url": "https://example.com"} + + observation = { + "action_status": "ok", + "url": "https://example.com/home", + "title": "Home", + "elements": [{"index": 0, "tag": "button", "label": "Login"}], + "text": "welcome", + } + done = task_queue.complete_browser_task( + task_id=task["id"], + user_id="u1", + observation=observation, + ) + assert done is not None + assert done["status"] == "done" + assert done["result"]["observation"] == observation + + +def test_fetch_task_default_kind_is_fetch(): + task = task_queue.create_browser_task(user_id="u1", url="https://example.com") + assert task["kind"] == "fetch" + claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1") + assert claimed["kind"] == "fetch" + assert claimed["session_id"] == "" + assert claimed["action"] is None + + +# --- tool validation -------------------------------------------------------- + + +async def _invoke(**kwargs) -> str: + return await act_mod.browser_act_tool.ainvoke(kwargs) + + +@pytest.mark.asyncio +async def test_browser_act_disabled_when_actions_off(monkeypatch): + _enable_actions(monkeypatch, enabled=True, actions_enabled=False) + out = await _invoke(action="navigate", url="https://example.com") + assert "actions_enabled is false" in out + + +@pytest.mark.asyncio +async def test_browser_act_unknown_action(monkeypatch): + _enable_actions(monkeypatch) + out = await _invoke(action="frobnicate") + assert "unknown action" in out + + +@pytest.mark.asyncio +async def test_browser_act_navigate_requires_url(monkeypatch): + _enable_actions(monkeypatch) + out = await _invoke(action="navigate") + assert "requires an exact http(s) URL" in out + + +@pytest.mark.asyncio +async def test_browser_act_non_navigate_needs_session(monkeypatch): + _enable_actions(monkeypatch) + out = await _invoke(action="click", index=0) + assert "no session_id yet" in out + + +@pytest.mark.asyncio +async def test_browser_act_click_requires_index(monkeypatch): + _enable_actions(monkeypatch) + out = await _invoke(action="click", session_id="sess-1") + assert "requires a non-negative element index" in out + + +# --- tool success path (simulated extension) -------------------------------- + + +@pytest.mark.asyncio +async def test_browser_act_returns_formatted_observation(monkeypatch): + _enable_actions(monkeypatch) + + async def fake_extension(): + # Wait for the tool to create the pending act task, then complete it. + for _ in range(40): + claimed = task_queue.claim_next_browser_task(user_id="default", worker_id="ext") + if claimed is not None: + task_queue.complete_browser_task( + task_id=claimed["id"], + user_id="default", + observation={ + "action_status": "ok", + "url": "https://example.com/home", + "title": "Home Page", + "elements": [ + {"index": 0, "tag": "input", "type": "search", "label": "Search", "value": ""}, + {"index": 1, "tag": "button", "label": "Go"}, + ], + "text": "Welcome to the home page.", + }, + ) + return + await asyncio.sleep(0.1) + + ext = asyncio.create_task(fake_extension()) + out = await _invoke(action="navigate", url="https://example.com") + await ext + + assert "Browser session:" in out + assert "Home Page" in out + assert "[0] input[search]" in out + assert "[1] button" in out + assert "Welcome to the home page." in out diff --git a/offline-backend-20260512/config.yaml b/offline-backend-20260512/config.yaml index 1956e4df..751c0156 100644 --- a/offline-backend-20260512/config.yaml +++ b/offline-backend-20260512/config.yaml @@ -1,76 +1,76 @@ config_version: 8 log_level: info models: -- name: deepseek-chat - display_name: DeepSeek Chat +# - name: deepseek-chat +# display_name: DeepSeek Chat +# use: langchain_openai:ChatOpenAI +# model: deepseek-chat +# api_key: sk-9808fb2375534112ac275541aec14b9b +# base_url: https://api.deepseek.com +# request_timeout: 600.0 +# max_retries: 2 +# max_tokens: 4096 +# temperature: 1.0 +# supports_vision: false +# supports_thinking: true +# - name: zai-org/GLM-5-FP8 +# display_name: zai-org/GLM-5-FP8 +# use: langchain_openai:ChatOpenAI +# model: zai-org/GLM-5-FP8 +# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA +# base_url: https://api.us-west-2.modal.direct/v1 +# request_timeout: 600.0 +# max_retries: 2 +# max_tokens: 4096 +# temperature: 1.0 +# supports_vision: true +# supports_thinking: true +# - name: deepseek-v4-pro +# display_name: DeepSeek V4 Pro +# use: langchain_openai:ChatOpenAI +# model: deepseek-v4-pro +# api_key: sk-e1c22b7931dc446f8b67e57878edd105 +# base_url: https://api.deepseek.com +# request_timeout: 600.0 +# max_retries: 2 +# max_tokens: 4096 +# temperature: 1.0 +# supports_vision: false +# supports_thinking: true +# - name: minimax-m2.7 +# display_name: MiniMax M2.7 +# use: langchain_openai:ChatOpenAI +# model: MiniMax-M2.7 +# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk +# base_url: https://api.minimaxi.com +# max_tokens: 4096 +# temperature: 1.0 +# supports_vision: true +# supports_thinking: true +# - name: minimax-m2.5 +# display_name: MiniMax M2.5 +# use: langchain_openai:ChatOpenAI +# model: MiniMax-M2.5 +# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk +# base_url: https://api.minimaxi.com +# max_tokens: 4096 +# temperature: 1.0 +# supports_vision: true +# supports_thinking: true +# - name: deepseek-ai/DeepSeek-OCR +# display_name: deepseek-ai/DeepSeek-OCR +# use: langchain_openai:ChatOpenAI +# model: deepseek-ai/DeepSeek-OCR +# api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb +# base_url: https://api.siliconflow.cn/v1/ +# max_tokens: 4096 +# temperature: 1.0 +# supports_vision: true +# supports_thinking: true +- name: deepseek-ai/DeepSeek-V4-Flash + display_name: deepseek-ai/DeepSeek-V4-Flash use: langchain_openai:ChatOpenAI - model: deepseek-chat - api_key: sk-9808fb2375534112ac275541aec14b9b - base_url: https://api.deepseek.com - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: false - supports_thinking: true -- name: zai-org/GLM-5-FP8 - display_name: zai-org/GLM-5-FP8 - use: langchain_openai:ChatOpenAI - model: zai-org/GLM-5-FP8 - api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA - base_url: https://api.us-west-2.modal.direct/v1 - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true -- name: deepseek-v4-pro - display_name: DeepSeek V4 Pro - use: langchain_openai:ChatOpenAI - model: deepseek-v4-pro - api_key: sk-e1c22b7931dc446f8b67e57878edd105 - base_url: https://api.deepseek.com - request_timeout: 600.0 - max_retries: 2 - max_tokens: 4096 - temperature: 1.0 - supports_vision: false - supports_thinking: true -- name: minimax-m2.7 - display_name: MiniMax M2.7 - use: langchain_openai:ChatOpenAI - model: MiniMax-M2.7 - api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk - base_url: https://api.minimaxi.com - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true -- name: minimax-m2.5 - display_name: MiniMax M2.5 - use: langchain_openai:ChatOpenAI - model: MiniMax-M2.5 - api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk - base_url: https://api.minimaxi.com - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true -- name: deepseek-ai/DeepSeek-OCR - display_name: deepseek-ai/DeepSeek-OCR - use: langchain_openai:ChatOpenAI - model: deepseek-ai/DeepSeek-OCR - api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb - base_url: https://api.siliconflow.cn/v1/ - max_tokens: 4096 - temperature: 1.0 - supports_vision: true - supports_thinking: true -- name: Qwen/Qwen3.6-27B - display_name: Qwen/Qwen3.6-27B - use: langchain_openai:ChatOpenAI - model: Qwen/Qwen3.6-27B + model: deepseek-ai/DeepSeek-V4-Flash api_key: sk-qwwwvjompnzxqhndctlwmmomvqdgaqqqgxvibfxchsikvgod base_url: https://api.siliconflow.cn/v1/ max_tokens: 4096