Compare commits

..
4 Commits
37 changed files with 2949 additions and 248 deletions

No files matched your search

+2
View File
@@ -7,6 +7,8 @@
// 4. 这里的默认值来自 .env.network;如某个环境不用某项,可以留空字符串。
// 5. 布尔开关也用字符串形式填写:"true" / "false"。
window.__CM_RUNTIME_CONFIG__ = {
// CMZS 扩展 ID;部署后可直接修改,无需重新构建前端。
VITE_BROWSER_EXTENSION_ID: "fkphoilppiodafnombhnjfocpphcjlam",
// 品牌图片相对路径会以 runtime-config.js 所在目录为基准解析。
// Nginx 以子路径部署时不需要修改前端图片路径。
// 打包后可直接替换 dist/brand-assets/ 内的同名 PNG。
@@ -1,17 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="152" height="42.359" viewBox="0 0 152 42.359" fill="none">
<defs>
<linearGradient id="bg" x1="0" y1="0" x2="152" y2="42.359" gradientUnits="userSpaceOnUse">
<stop stop-color="#A90F1A"/>
<stop offset="0.5" stop-color="#D82932"/>
<stop offset="1" stop-color="#941019"/>
</linearGradient>
<linearGradient id="shine" x1="76" y1="0" x2="76" y2="42.359" gradientUnits="userSpaceOnUse">
<stop stop-color="#FFF" stop-opacity=".34"/>
<stop offset="1" stop-color="#FFF" stop-opacity="0"/>
</linearGradient>
</defs>
<path d="M5 0h142a5 5 0 0 1 5 5v32.359a5 5 0 0 1-5 5H5a5 5 0 0 1-5-5V5a5 5 0 0 1 5-5Z" fill="url(#bg)"/>
<path d="M5 .75h142a4.25 4.25 0 0 1 4.25 4.25v32.359a4.25 4.25 0 0 1-4.25 4.25H5A4.25 4.25 0 0 1 .75 37.359V5A4.25 4.25 0 0 1 5 .75Z" stroke="#FFF" stroke-opacity=".34" stroke-width="1.5"/>
<path d="M1 1h150v18.5c-29.8 10.6-119.1 10.6-150 0V1Z" fill="url(#shine)" opacity=".62"/>
<path d="M22 40.859h108" stroke="#FFF" stroke-opacity=".8" stroke-width="2" stroke-linecap="round"/>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="152" height="42.359" viewBox="0 0 152 42.359"><defs><style>.a{fill:url(#a);}</style><pattern id="a" preserveAspectRatio="none" width="100%" height="100%" viewBox="0 0 536 150"><image width="536" height="150" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAhgAAACWCAYAAACcofGRAABDtklEQVR42u2dW5bkOJJkVUC6R2RlzRLmu9fSK56t9Hz3EqarMjLcCJ0PgCQA4kkCfJjD8sTJCHcz2hu4FBUVBfVLvzS6/J//+N94g6eBb3Kf73Dhb3KfVS//+X//m/tHp1/6QtYvHSie+z3o37XvByPcgaNfOmD0S7+8P1DgDb5D/ft6/sbN3x0+OnD0S1+w+qVDRZvPNfp3r0PJScflDhv90gGjXzpQvNfnF/071S83BgbuwNEvHTD6pYPFcz636N+ffjlxg7/LMTpo9EsHjH7pUFHpM4pv/v1A91Pc+jnwu71+HTb6pQNGv9wNKNCViP7dfDNQuVK14A4c/dIXsX75rmCBb6ZGoH+3b7Uh8jdTPbiDRr90wOiXDhTXKxpX32f/Xt4DGp4U3MUdOPqlA0a/dLC4l7LxLt0r/fJ+3RxnKBUdNPqlA0a/PAYsWgLF3UOt+vfp+wLJXcO/HgMcHTQ6YPRLB4p3gAm86/eFH/6dxjO6Su5ovHw76OjA0QGjX74fWLSAhDvBBDogdEA5ET7uAB18Z9jooNEBo1/eFyqeAhR3LbVUhwZ80+8tV97ccG9PBXfg6LDRAaNf3hUungAWt4OKWjCB/t28BD5OhI4rN/FHg0aHjA4Y/dKh4qoZH7gjPHRgeB8gwfPKGjWO1WGjXzpgdKi4dNM+s30TdwGIp8LDWX4PPDQOfC+E4LryxtntuFfAUIeNDhj9cmOwqLl544kwUbKxops+u6mzEoTgmlLHmXHlZ0JQB40OGP3ypnDxOLC4O1TcASRqPe/aZsyngMeFsHEn0OiQ0S8dML4JVNRSIs4ACpy5UeNhps/u42gDMGeBSO5jxnllDr7RfXCHjQ4Y/fJeYNF6w28OFDmbN25o+mwNCk8tq7Te7G/mqch+TLgXDLQ8fgeNDhj98lCoaF2SQKvP2RUgsWeTxgnPs1/aA8EeEKkNISeBB1847fUMhaXDRgeMDhYXQsXtgOJMmCjd0Pfcb4eG94ORUgCpBR8nQMdVwHEZbHTQ6IDR4eJecHEJWFwBFaX3+Y4wkXoN7mDovBo6rjB3pu7zhqDRIaMDRr90qLgeKM6EiZL7eqdOD+oGz1MA5ESPRfS+bgYcHTY6YPRLh4o6x2upTtSEiLMB4omdLE9sJT0bSnKfV2v4aKRydNjolw4YDweLI9BweemjlTpRCyZabrg4qTumX9oAS0sYyXl8V0IH6k6CbVFKuQQ2Omh0wPjuYFEbKqqpFHs33BqqR4uNutXz6ZdnwEgLAKkBHrWfT2V1ozZsdNDogNHh4iQFoWYJ5HKl4ugx7zAavcNE7yi5CjZagFNFZePMkkyHjA4Y3x4szlQsmkJF7Y04o3vhsoTMGwIE3tfT+T26S47ed23oqKhsnAEbpysaHTQ6YDwVLGqqFYeh4g5AUWNDf0i3SP8+PhhManeXHLm/mwJHLdi4jarRQaMvaHcBi7PUiiZQgRN9GUc2+Bt2jDS5H/zjz2/13eV//4ufDCM1O0tqqxy1gONmsHGqqtFBowPGHcGillqBFqUPnFA+aQ0TDUECbwMILNs8Fgh+I0DhO3eW1FQ5apSCKpVS+GmqRgeNDhgdLi4Gi6ObfutySu3vymkw0QoUTl+VzgGTg9BxOnAcLauEjn0j0OiQ0QGjg0UjsHgMVJylVJxp9DzyfTgMEO8CBm8GKgcA5NTOktrD2WoAx41ho4NGB4zHwkUpWOAsb0ULqKgJFCfDBE6Bhw4Obw8ihRDCd4WOUuA4ETZqeDVKYYU7ZHTAeIJicalacQQqSlSK2upEJZhoM9TsH392gPjOAPLvf7VqU+W7lVd8xzuqbjSAjZaqRlc0OmBcBhctyyC3gYpaPorGQIHqIPHHH+jfXeqZGLmXv/5q0aLKdwGOEv/GA2DjsvLJd4MMdLh4X7hoUQK5EVjUhYp7AkVXSJ4KH5nAcQZstAaN2iWUDhkdML4rXJwJFk0Vi1ZqRaPOEbyJOnGr7xszn/J4AHAHkiYqB5/VUXKGqlFZ0bg1aHwXyEAHiyY+i6ZqRWuouBgo8FBVAu8CBLdftM4BFn6o2sF3Ao4LYKO2qsFdzeiAcRfF4lSwuAIqzgaK5HNsDxHooNDB5DIQScBHBgBcAhwtYeNmoNEVje8EGDdWLN4KKrjuwLU8VaItTKBDQ79UhBG+UvHgyhtiSUfJG8FGVzQ6YHS4KL3NBWrFXcECHSj65QTg4HcAjTNUjQ4Z7wUZ6HBxus+iCBJqKRZ3USsuKnmgQ0Tww4PGKwx3+DgRPPaXVC5VNWooGjtB45b+jHeBDHS4uIdicVStuLgEght4KN4fIri3rT4NWi6Hj2MeDr5DCaWRqnF7ReMdIANdtciGgepg0aIEwge6MGpAxUlAgbcAiA4M3x5ICgGEn6BwHCmlpJSNBrDRGjS+tZqBrlpUybJAa7XCvF0tqGiuUtQvd+ARUHEDcAA9q7zDdIOsjAtB5HTQyICOq9WNEGzsTQrdoWrs7dDhrmY8GDBOhIs9qkVVuGilWPCxltr44z4OFs9RKC6AiafBw6Mh5GL141TwiKgbR5WN2pCxd+hbA8g4Tc14ImTgm8FFrkJxa7B4Q6i4pzLREB6+KyS8JZycDCEF0MHvChsPAQ2u/b48DTLQVYv7lEOOqhWnQcUJCsWpMFERJK4CB36zlnNcFOddFUROBI9M6KiucFwBG0dUjYvKJt9WzcA3VS2agsXVagXvN6S28FPcR52oABKtAYL7gLNbg0kVADkBPJqrHOW+Da7RunqhqnE70HgCZKDDxXVwcaFigW8DFgehohVQdJB4D/A4DByNYeNmoMEPVzQ6ZLwTYNwALpqUQ46AxZOhohlM3EyZ6PDQIeTOSkcGdDwSNvaCRuOyybeGDHSwqKta7FUsStWKm0DFuUBxACQ6QPTL2wBIZfBoAhyNYaNE1diraHQ14w0B4wBctAAL1Ij0jl2vgVrREipuDxQ1QKIDRL9cBSC7wOPuwFERNmqqGpVAw33Ml4LG3SAD3wguqigWez0WR8CiGlRUVCmalDsKgOIoSLwRRODtO0w7fJSBR4PSSgQ6uGE3CtdSNWqARuK6RxWNt4QMdOWifjmklseiahmkolpRFS5OUikeDBRdTXlDANkDHFerGwllg58MGbkejYplk2+hZOAb+C2qlESOlkNKPBYXKhZtoeIEmHgASHRg6EBSDTyugo67w8Zej0YDf8ZlSsYdQANdtbgOLI6WQipAxa2A4g1gAtTh5R0ggL8VdNwJOPJgo4mqcSFovK2agW+oWlT1WpwAFmm14i5Q0dBDwd9j8+7qxnMAgd8WOg4CRyvYKFU1WoJGI2/G26kZ+MZGTpROLa0JFierFW2gIhMoHuSZQAeHDiJPhI+mno4DwFEFNiqrGq0UjSeUTc6GDDxYubisJHI5XFRQLDpYNPsOdJjo0HEZbDwQNGoqGpdAxkUlk0coGfgGykVxSeQNwOIRUPFQmOgA0QHkbaHjEbDxbNAoeayPVzLwjeDiNLA44rF4ElS8OUx0kOiXM6HhudDxUNjI9WicABpvCxl4EFzs9Vvs9lqUBmVdABZ1oeKZQNFVCSIC81sBEQPcoeM88LgDcByCjcagURrUtaNsctSXcUvIwBvCRTPVolIpBC3Uim8CFW+vSrwbKHQweZ7acWPYqKFqcK3SyQ3VjD2ZKPSf/9UOMvBmZZGqcFFZtWhWCtkFFxmeily4uDlYoMNEh443hY1bgEaWb2MHaDSEjOBxjqgZJ0BGznPgPatjK8jAd1QuWngtdoHFmVDxHKDA00HibHB41+FsONmXcDMQYboZdFwJHLVgYw9o1PZm3ErJMFaOFpAx3jgFcQ9c7FItGoFFjXJIU7C4AVA8EiZaAkSf5HrstTgCJan39WQAwc4NFa2gwxM+yDnrihc45nUpEzTmdc8BDWQ9vz/+wAwZRrmbjWNw5GSUPbeJXsf8ubtvxH7vPB9Enpf7u9S/3yedcId6cQvV4hTFIg8smqsVKbC4GVDgHUDiMnC4exnmItWgpjpysfLBdBOVI/Wa1vZr7FI1KpROShSN26sZzupQW8XAxWWRZnBRU7U4rFjsUCveQKl4jDpxFCSawkP3aVwGKIdHrF8DHpeXVqqUUo7DBh8onew2gpaUTTLKKu0hI7C61AKN8aHqCXKPcxJcHPFZNFMscsCC75FkiacpE02gosNEndes0sburBN89HN2EnBgJzSgFmjkvm7z+hQsoRRABjOjuHzilE6csgmFSidY1Qf4yia+35s/d0smifeE71b2OH1hb6BenOq3eBxY3K8EgrsCxRGYqPY6dXB4a/XjiNJxssrBdGEp5awSytHSSa2ySW47a2GXyd4oci5diWuoGHiQ5+I0v0VO7gWF7qsQLIrUimcqFbeHiSqvzftBBOjBI87vAB97weOm0MFnl5+CwFGgbBSXTsIejaJcjFJfxomtrFzyLT8KGXgHuLhKtTiiWNRQK05WKnAnheISmLgfRKDHcD8LPm4OHXxHz0YNr8YR0MjxZ5yoZtSFjIwV5Ahk4EFwUaxavCNYxKDiXYHidJi4DiTuAAx4AxjgJ4HHjaHjdsBxVNV4GmgcKJnkQUbmt30vZIxPPhO7Yt7FXrg4Wgo5Ua3At4OLine truncated

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 23 KiB

@@ -1306,10 +1306,9 @@ export const PromptInputSubmit = ({
<InputGroupButton
aria-label={ariaLabel}
className={cn(
// 皮肤本体在 globals.css 的 .agent-composer-send(主题色扁平渐变
// 圆钮);流式态切 .agent-composer-send--stop 低对比停止钮。
"agent-composer-send transition-all duration-300",
status === "streaming" && "agent-composer-send--stop",
status === "streaming" &&
"animate-breathe-stop bg-primary/10 text-primary border-primary/25 shadow-[0_0_0_0_rgba(217,33,33,0.25)] hover:bg-primary/14",
className,
)}
size={size}
@@ -0,0 +1,326 @@
"use client";
import { CheckIcon, CopyIcon, MonitorUpIcon, UnplugIcon } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import { PromptInputButton } from "@/components/ai-elements/prompt-input";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import {
createBrowserSkillPairing,
getBrowserSkillStatus,
identifyBrowserSkillExtension,
pairBrowserSkillThroughExtension,
reconnectBrowserSkillThroughExtension,
revokeBrowserSkillDevice,
subscribeBrowserSkillExtensionEvents,
type BrowserSkillStatus,
} from "@/core/browser-skill/api";
import { shouldPairCurrentBrowser } from "@/core/browser-skill/connection";
import { cn } from "@/lib/utils";
import { Tooltip } from "./tooltip";
const ACTIVE_INSTANCE_STORAGE_KEY = "cmzs.browser-skill.active-instance-id";
function storedInstanceId(): string | null {
if (typeof window === "undefined") return null;
return window.localStorage.getItem(ACTIVE_INSTANCE_STORAGE_KEY);
}
function rememberInstanceId(instanceId: string | null) {
if (instanceId) {
window.localStorage.setItem(ACTIVE_INSTANCE_STORAGE_KEY, instanceId);
} else {
window.localStorage.removeItem(ACTIVE_INSTANCE_STORAGE_KEY);
}
}
export function BrowserSkillControl({
active,
onActiveChange,
disabled,
compact,
}: {
active: boolean;
onActiveChange: (active: boolean) => void;
disabled?: boolean;
compact?: boolean;
}) {
const [status, setStatus] = useState<BrowserSkillStatus | null>();
const [open, setOpen] = useState(false);
const [busy, setBusy] = useState(false);
const [pairingLink, setPairingLink] = useState("");
const operationInFlight = useRef(false);
const activatedInstanceId = useRef<string | null>(storedInstanceId());
const onActiveChangeRef = useRef(onActiveChange);
useEffect(() => {
onActiveChangeRef.current = onActiveChange;
}, [onActiveChange]);
const refresh = useCallback(async () => {
try {
const next = await getBrowserSkillStatus();
setStatus(next);
if (
!next?.connected ||
(activatedInstanceId.current != null &&
next.instance_id !== activatedInstanceId.current)
) {
onActiveChangeRef.current(false);
}
} catch {
setStatus((current) => current ?? null);
onActiveChangeRef.current(false);
}
}, []);
useEffect(() => {
void refresh();
let replacementRefreshTimer: number | undefined;
const unsubscribe = subscribeBrowserSkillExtensionEvents(() => {
// A reconnect can close the previous socket before the same browser
// instance finishes its new handshake. Re-check the stable instance id
// after that handshake settles instead of racing it.
if (replacementRefreshTimer !== undefined) {
window.clearTimeout(replacementRefreshTimer);
}
replacementRefreshTimer = window.setTimeout(() => {
replacementRefreshTimer = undefined;
void refresh();
}, 1000);
});
return () => {
if (replacementRefreshTimer !== undefined) {
window.clearTimeout(replacementRefreshTimer);
}
unsubscribe();
};
}, [refresh]);
if (status == null || !status.enabled) return null;
const connected = status?.connected === true;
const paired = status?.paired === true;
const handleToggle = () => {
if (active) {
onActiveChange(false);
return;
}
void activateCurrentBrowser();
};
const confirmConnection = async (
expectedInstanceId: string,
attempts = 3,
intervalMs = 500,
) => {
for (let attempt = 0; attempt < attempts; attempt += 1) {
const next = await getBrowserSkillStatus();
setStatus(next);
if (next?.connected === true && next.instance_id === expectedInstanceId) {
return true;
}
if (attempt < attempts - 1) {
await new Promise((resolve) => window.setTimeout(resolve, intervalMs));
}
}
return false;
};
const requireInstanceId = (instanceId?: string) => {
if (!instanceId) {
throw new Error("当前浏览器中的 CMZS 插件版本过旧或插件 ID 配置不匹配,请重新加载最新插件");
}
return instanceId;
};
const waitUntilIdle = async () => {
for (let attempt = 0; attempt < 30; attempt += 1) {
const next = await getBrowserSkillStatus();
setStatus(next);
if (!next?.busy) return true;
await new Promise((resolve) => window.setTimeout(resolve, 2000));
}
return false;
};
const activateCurrentBrowser = async () => {
if (operationInFlight.current) return;
operationInFlight.current = true;
setBusy(true);
setOpen(false);
setPairingLink("");
try {
let current: BrowserSkillStatus | null | undefined = status;
if (current?.busy) {
toast.info("当前浏览器任务结束后将自动切换到此浏览器");
if (!(await waitUntilIdle())) {
throw new Error("等待当前浏览器任务结束超时,请稍后重试");
}
current = await getBrowserSkillStatus();
setStatus(current);
}
const identity = await identifyBrowserSkillExtension();
const expectedInstanceId = requireInstanceId(identity.instance_id);
if (current?.connected && current.instance_id === expectedInstanceId) {
activatedInstanceId.current = expectedInstanceId;
rememberInstanceId(expectedInstanceId);
onActiveChange(true);
setOpen(false);
toast.success("本机浏览器已启用");
return;
}
if (!shouldPairCurrentBrowser(current?.paired === true, identity)) {
try {
await reconnectBrowserSkillThroughExtension();
if (await confirmConnection(expectedInstanceId)) {
activatedInstanceId.current = expectedInstanceId;
rememberInstanceId(expectedInstanceId);
onActiveChange(true);
setOpen(false);
toast.success("已切换到当前浏览器");
return;
}
} catch {
// The stored credential may have been invalidated by another browser.
// Fall through to a fresh one-time pairing for this installation.
}
}
await pairCurrentBrowser();
} catch (error) {
// Automatic pairing should remain a one-click flow. Keep the manual
// pairing dialog out of the normal path and report genuine failures as
// a toast instead.
setOpen(false);
toast.error(error instanceof Error ? error.message : "无法唤醒浏览器插件");
} finally {
operationInFlight.current = false;
setBusy(false);
}
};
const pairCurrentBrowser = async () => {
const link = await createBrowserSkillPairing();
const identity = await pairBrowserSkillThroughExtension(link);
const instanceId = requireInstanceId(identity.instance_id);
if (!(await confirmConnection(instanceId))) {
throw new Error("插件已授权但未能连接,请检查插件状态");
}
activatedInstanceId.current = instanceId;
rememberInstanceId(instanceId);
onActiveChange(true);
setOpen(false);
setPairingLink("");
toast.success("本机浏览器已连接");
};
const copyPairingLink = async () => {
setBusy(true);
try {
const link = await createBrowserSkillPairing();
await navigator.clipboard.writeText(link);
setPairingLink(link);
toast.success("配对链接已复制,5 分钟内有效且仅可使用一次");
await refresh();
} catch (error) {
toast.error(error instanceof Error ? error.message : "复制配对链接失败");
} finally {
setBusy(false);
}
};
const revoke = async () => {
setBusy(true);
try {
await revokeBrowserSkillDevice();
activatedInstanceId.current = null;
rememberInstanceId(null);
setPairingLink("");
onActiveChange(false);
await refresh();
toast.success("浏览器授权已撤销");
} catch (error) {
toast.error(error instanceof Error ? error.message : "撤销授权失败");
} finally {
setBusy(false);
}
};
return (
<>
<Tooltip content={connected ? (active ? "本机浏览器已启用" : "启用本机浏览器") : "连接本机浏览器"}>
<PromptInputButton
type="button"
aria-pressed={active}
disabled={disabled}
className={cn(
"gap-1.5! rounded-full px-3! shadow-none",
compact && "rounded-lg px-2.5! py-1! text-[11px] font-medium",
active
? "border border-emerald-300 bg-emerald-50 text-emerald-700 hover:bg-emerald-100 dark:border-emerald-700 dark:bg-emerald-950/40 dark:text-emerald-200"
: "border-0 bg-transparent text-muted-foreground hover:bg-muted/70 hover:text-foreground dark:text-slate-400 dark:hover:bg-slate-700/70 dark:hover:text-slate-100",
)}
onClick={handleToggle}
>
{active && <CheckIcon className="size-3" />}
<MonitorUpIcon className="size-3" />
<span className="text-xs font-normal">本机浏览器</span>
</PromptInputButton>
</Tooltip>
<Dialog open={open} onOpenChange={setOpen}>
<DialogContent className="sm:max-w-md">
<DialogHeader>
<DialogTitle>连接本机浏览器</DialogTitle>
<DialogDescription>
点击后会自动连接已安装的 CMZS 插件。一个账号只绑定一台浏览器。
</DialogDescription>
</DialogHeader>
<div className="rounded-xl border bg-muted/30 p-4">
<div className="flex items-center justify-between gap-3">
<span className="font-medium">连接状态</span>
<span className={cn("text-sm", connected ? "text-emerald-600 dark:text-emerald-300" : "text-muted-foreground")}>
{connected ? "已连接" : paired ? "已配对,等待扩展上线" : "未配对"}
</span>
</div>
{status?.device?.label && (
<p className="mt-2 text-sm text-muted-foreground">{status.device.label}</p>
)}
{pairingLink && (
<p className="mt-3 break-all rounded-lg bg-background px-3 py-2 text-xs text-muted-foreground">
{pairingLink}
</p>
)}
</div>
<DialogFooter className="sm:justify-between">
{paired ? (
<Button type="button" variant="ghost" className="text-destructive hover:text-destructive" disabled={busy} onClick={revoke}>
<UnplugIcon className="size-4" />
撤销授权
</Button>
) : <span />}
<Button type="button" disabled={busy} onClick={copyPairingLink}>
<CopyIcon className="size-4" />
{pairingLink ? "复制配对链接" : "手动复制配对链接"}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</>
);
}
@@ -0,0 +1,271 @@
"use client";
import {
ExternalLinkIcon,
MousePointer2Icon,
PictureInPicture2Icon,
SquareIcon,
XIcon,
} from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { Button } from "@/components/ui/button";
import {
browserTaskAction,
getBrowserTaskStatus,
type BrowserTaskPreview as PreviewFrame,
type BrowserTaskStatus,
} from "@/core/browser-skill/api";
import { cn } from "@/lib/utils";
interface DocumentPictureInPictureApi {
requestWindow(options: { width: number; height: number }): Promise<Window>;
}
const ACTIVE_PREVIEW_INTERVAL_MS = 2500;
const ACTIVE_STATUS_INTERVAL_MS = 15_000;
const IDLE_STATUS_INTERVAL_MS = 10_000;
const HIDDEN_STATUS_INTERVAL_MS = 30_000;
function pictureInPictureApi(): DocumentPictureInPictureApi | undefined {
return (window as Window & { documentPictureInPicture?: DocumentPictureInPictureApi })
.documentPictureInPicture;
}
function copyStyles(target: Document) {
const base = target.createElement("base");
base.href = document.baseURI;
target.head.append(base);
for (const sheet of Array.from(document.styleSheets)) {
if (sheet.href) {
const link = target.createElement("link");
link.rel = "stylesheet";
link.href = sheet.href;
target.head.append(link);
continue;
}
try {
const style = target.createElement("style");
style.textContent = Array.from(sheet.cssRules, (rule) => rule.cssText).join("\n");
target.head.append(style);
} catch {
// Cross-origin stylesheets cannot expose cssRules; linked styles above remain usable.
}
}
const reset = target.createElement("style");
reset.textContent = "html,body,#browser-task-pip{width:100%;height:100%;margin:0;overflow:hidden;background:#0f172a}";
target.head.append(reset);
}
export function BrowserTaskPreview({
threadId,
enabled,
}: {
threadId?: string;
enabled: boolean;
}) {
const [status, setStatus] = useState<BrowserTaskStatus | null>(null);
const [frame, setFrame] = useState<PreviewFrame | null>(null);
const [pipTarget, setPipTarget] = useState<HTMLElement | null>(null);
const pipWindow = useRef<Window | null>(null);
const closePip = useCallback(() => {
setPipTarget(null);
pipWindow.current?.close();
pipWindow.current = null;
}, []);
useEffect(() => closePip, [closePip]);
useEffect(() => {
if (!enabled || !threadId) {
setStatus(null);
setFrame(null);
closePip();
return;
}
let alive = true;
let timer = 0;
let failures = 0;
let hasActiveTask = false;
let currentStatus: BrowserTaskStatus | null = null;
let statusCheckedAt = 0;
let polling = false;
const poll = async () => {
if (polling) return;
polling = true;
try {
const now = Date.now();
// Preview POSTs carry live frames. Status GETs only discover or
// periodically reconcile the session, so keep them on a slower cadence.
if (!currentStatus?.selected || now - statusCheckedAt >= ACTIVE_STATUS_INTERVAL_MS) {
currentStatus = await getBrowserTaskStatus(threadId);
statusCheckedAt = now;
}
if (!alive) return;
failures = 0;
hasActiveTask = Boolean(currentStatus.connected && currentStatus.selected);
setStatus(currentStatus);
if (hasActiveTask && !document.hidden) {
try {
const preview = await browserTaskAction<PreviewFrame>(threadId, "preview");
if (alive) setFrame(preview);
} catch {
// A daemon reconnect can invalidate a session between the status
// GET and preview POST. Treat it as an ended task, not a page-level
// error, and wait for the next BrowserSkill call to create one.
hasActiveTask = false;
currentStatus = null;
statusCheckedAt = 0;
if (alive) {
setStatus(null);
setFrame(null);
}
}
} else {
setFrame(null);
}
} catch {
failures += 1;
if (alive) {
setStatus(null);
setFrame(null);
}
} finally {
polling = false;
if (alive) {
const delay = failures
? Math.min(30_000, 2000 * 2 ** (failures - 1))
: document.hidden
? HIDDEN_STATUS_INTERVAL_MS
: hasActiveTask
? ACTIVE_PREVIEW_INTERVAL_MS
: IDLE_STATUS_INTERVAL_MS;
timer = window.setTimeout(poll, delay);
}
}
};
const onVisibilityChange = () => {
window.clearTimeout(timer);
timer = window.setTimeout(poll, 0);
};
document.addEventListener("visibilitychange", onVisibilityChange);
void poll();
return () => {
alive = false;
window.clearTimeout(timer);
document.removeEventListener("visibilitychange", onVisibilityChange);
};
}, [closePip, enabled, threadId]);
const openPip = async () => {
const api = pictureInPictureApi();
if (!api || !frame) return;
const child = await api.requestWindow({ width: 420, height: 320 });
copyStyles(child.document);
child.document.title = frame.title || "本机浏览器任务";
const target = child.document.createElement("div");
target.id = "browser-task-pip";
child.document.body.append(target);
child.addEventListener("pagehide", closePip, { once: true });
pipWindow.current = child;
setPipTarget(target);
};
if (!enabled || !threadId || !status?.selected || !frame) return null;
const pointer = frame.pointer;
const viewportWidth = Math.max(1, frame.viewport?.width ?? 1);
const viewportHeight = Math.max(1, frame.viewport?.height ?? 1);
const pointerLabel = pointer
? pointer.action === "click"
? "正在点击"
: pointer.action === "wheel"
? "正在滚动"
: "正在移动"
: "浏览器正在执行";
const panel = (
<aside
className={cn(
"overflow-hidden border border-slate-700 bg-slate-950 text-white shadow-2xl",
pipTarget
? "flex size-full flex-col rounded-none"
: "fixed right-5 bottom-44 z-40 w-[360px] max-w-[calc(100vw-2.5rem)] rounded-xl",
)}
aria-label="本机浏览器任务预览"
>
<header className="flex h-10 shrink-0 items-center gap-2 px-3 text-xs">
<span className="size-2 rounded-full bg-emerald-400" />
<div className="min-w-0 flex-1">
<strong className="block truncate font-medium">
{frame.title || "本机浏览器正在执行"}
</strong>
<span className="block truncate text-[10px] text-sky-200/80">{pointerLabel}</span>
</div>
<Button
size="icon"
variant="ghost"
className="size-7 text-white/80 hover:bg-slate-800 hover:text-white"
onClick={() => void browserTaskAction(threadId, "focus")}
title="定位浏览器窗口"
>
<ExternalLinkIcon className="size-3.5" />
</Button>
{pictureInPictureApi() && (
<Button
size="icon"
variant="ghost"
className="size-7 text-white/80 hover:bg-slate-800 hover:text-white"
onClick={() => (pipTarget ? closePip() : void openPip())}
title={pipTarget ? "退出画中画" : "画中画"}
>
{pipTarget ? <XIcon className="size-3.5" /> : <PictureInPicture2Icon className="size-3.5" />}
</Button>
)}
<Button
size="icon"
variant="ghost"
className="size-7 text-white/80 hover:bg-red-950 hover:text-white"
onClick={() => void browserTaskAction(threadId, "stop").then(() => setStatus(null))}
title="停止浏览器任务"
>
<SquareIcon className="size-3.5" />
</Button>
</header>
<button
type="button"
className="relative min-h-0 flex-1 overflow-hidden bg-black p-0"
onClick={() => void browserTaskAction(threadId, "focus")}
title="点击定位浏览器窗口"
>
<img
src={`data:image/${frame.format};base64,${frame.image_base64}`}
alt="浏览器实时预览"
className={cn("block w-full object-contain", pipTarget ? "h-full" : "aspect-video")}
/>
{pointer && (
<span
className="pointer-events-none absolute z-10 transition-[left,top] duration-300 ease-[cubic-bezier(0.16,1,0.3,1)]"
style={{
left: `${Math.min(100, Math.max(0, (pointer.x / viewportWidth) * 100))}%`,
top: `${Math.min(100, Math.max(0, (pointer.y / viewportHeight) * 100))}%`,
}}
aria-hidden="true"
>
{pointer.action === "click" && (
<span
key={pointer.sequence}
className="absolute -left-3 -top-3 size-7 animate-ping rounded-full border-2 border-sky-300/80"
/>
)}
<MousePointer2Icon className="size-6 -translate-y-0.5 fill-sky-400 text-white drop-shadow-[0_2px_3px_rgba(0,0,0,0.9)]" />
</span>
)}
</button>
</aside>
);
return pipTarget ? createPortal(panel, pipTarget) : panel;
}
@@ -85,6 +85,7 @@ import { desensitize } from "@/lib/desensitize";
import { cn } from "@/lib/utils";
import { AgentSelector } from "@/components/workspace/agents/agent-selector";
import { BrowserSkillControl } from "@/components/workspace/browser-skill-control";
import {
ModelSelector,
@@ -138,6 +139,7 @@ export function InputBox({
agentSelector,
knowledgeSelector,
knowledgeRetrieval,
browserSkill,
knowledgeSuggestedQuestions = [],
extraModels = [],
placeholder,
@@ -170,6 +172,10 @@ export function InputBox({
* Other values scale the landing / compact-landing / conversation base.
*/
composerHeightPercent?: number;
browserSkill?: {
active: boolean;
onActiveChange: (active: boolean) => void;
};
disabled?: boolean;
context: Omit<
AgentThreadContext,
@@ -831,17 +837,7 @@ export function InputBox({
/>
)}
<div
className={cn(
usesGeminiFrame && "gemini-gradient-border gemini-shadow",
// 对话态(含紧凑会话):文心一言式外壳 —— 空闲主题色浅边白卡,
// 生成中主题色渐变流光描边(globals.css .composer-shell)。
// 落地态(Gemini 彩灯)与紧凑落地白卡维持各自形态。
!usesGeminiFrame &&
!(compact && isNewThread) &&
(composerStatus === "streaming"
? "composer-shell composer-shell--streaming"
: "composer-shell composer-shell--idle"),
)}
className={cn(usesGeminiFrame && "gemini-gradient-border gemini-shadow")}
data-frame-radius={
usesGeminiFrame && frameRadius !== "default" ? frameRadius : undefined
}
@@ -852,20 +848,19 @@ export function InputBox({
}
className={cn(
"transition-all duration-300 ease-out",
// 内部 input-group:紧凑落地白卡与对话态外壳都去掉自带描边/底色,
// 只保留外层一层描边(Gemini 落地态维持现状的双层结构不动)。
// 紧凑落地态(CM 参考图):表单作为白卡,内部 input-group 透明无边框,
// 避免双层描边;圆角对齐 2xl。
compact && isNewThread
? "*:data-[slot='input-group']:rounded-2xl *:data-[slot='input-group']:border-0 *:data-[slot='input-group']:bg-transparent *:data-[slot='input-group']:shadow-none"
: usesGeminiFrame
? "*:data-[slot='input-group']:rounded-3xl"
: "*:data-[slot='input-group']:rounded-3xl *:data-[slot='input-group']:border-0 *:data-[slot='input-group']:bg-transparent *:data-[slot='input-group']:shadow-none",
: "*:data-[slot='input-group']:rounded-3xl",
// 紧凑落地态(CM 参考图):白卡 + 细边 + 轻阴影 + 主色聚焦环(无旋转彩边)
compact && isNewThread
? "rounded-2xl border border-gray-200/70 bg-white overflow-hidden shadow-[0_4.5px_22px_rgba(0,0,0,0.025)] transition-all duration-300 focus-within:border-[rgb(var(--legacy-primary-rgb)/35%)] focus-within:ring-4 focus-within:ring-[rgb(var(--legacy-primary-rgb)/5%)] focus-within:shadow-[0_10px_35px_rgb(var(--legacy-primary-rgb)/4%)] dark:border-slate-700 dark:bg-slate-800"
: // 表单本体一律透明:落地态由外层 gemini-gradient-border 供白卡 +
// 旋转彩边,对话态由外层 composer-shell 供边框/底色/阴影,
// 避免双层描边。
"rounded-3xl bg-transparent border-0 shadow-none ring-0",
: // landing 态:透明,让外层 gemini-gradient-border 提供白卡 + 旋转描边
// 对话态:常规白卡 + 中性边框 + 轻阴影
usesGeminiFrame
? "rounded-3xl bg-transparent border-0 shadow-none ring-0"
: "rounded-3xl bg-background border border-border shadow-sm transition-shadow hover:shadow-md focus-within:border-ring/40 focus-within:shadow-md focus-within:ring-1 focus-within:ring-ring/20 dark:bg-slate-800 dark:border-slate-700",
className,
)}
disabled={disabled}
@@ -922,6 +917,14 @@ export function InputBox({
)}
/>
)}
{!frontendRagActive && browserSkill && (
<BrowserSkillControl
active={browserSkill.active}
onActiveChange={browserSkill.onActiveChange}
disabled={disabled}
compact={compact}
/>
)}
{!frontendRagActive && !hideContextTools && <Tooltip
content={
context.writing_mode
@@ -1475,9 +1478,11 @@ export function InputBox({
</ModelSelectorContent>
</ModelSelector>}
<PromptInputSubmit
// 皮肤(主题色扁平渐变圆钮 / 流式低对比停止钮)统一由 globals.css 的
// .agent-composer-send 提供,这里只保留紧凑落地档的小尺寸。
className={cn(compact && "size-7")}
className={cn(
"rounded-full border-border bg-background shadow-sm hover:bg-muted dark:bg-blue-500 dark:border-blue-500 dark:text-white dark:hover:bg-blue-400 dark:shadow-none",
compact &&
"size-7 rounded-lg border-0 bg-[var(--primary)] text-[var(--button-foreground,var(--primary-foreground))] hover:opacity-90 disabled:bg-muted disabled:text-muted-foreground/40 dark:bg-[var(--primary)] dark:border-0 dark:text-[var(--button-foreground,var(--primary-foreground))] dark:hover:opacity-90 dark:disabled:bg-muted",
)}
disabled={disabled}
onClick={(event) => {
if (composerStatus !== "streaming") return;
@@ -1486,6 +1491,7 @@ export function InputBox({
event.preventDefault();
handleStopCurrentGeneration();
}}
variant="outline"
status={composerStatus}
{...(documentRewriteActive
? { "aria-label": "停止 AI 全文改写" }
@@ -14,6 +14,7 @@ import {
NotebookPenIcon,
SearchIcon,
SquareTerminalIcon,
ImageIcon,
WrenchIcon,
} from "lucide-react";
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type AnchorHTMLAttributes } from "react";
@@ -47,6 +48,7 @@ import {
namedStreamingWriteFilePath,
} from "@/core/artifacts/streaming-write-preview";
import { listStreamingWriteFileCalls } from "@/core/artifacts/parse-partial-json";
import { urlOfArtifact } from "@/core/artifacts/utils";
import { useI18n } from "@/core/i18n/hooks";
import {
normalizeSearchResults,
@@ -233,6 +235,7 @@ export function MessageGroup({
skillDisplayMap={skillDisplayMap}
displaySources={displaySources}
ragModeEnabled={ragModeEnabled}
threadId={threadId}
/>
),
)}
@@ -246,6 +249,7 @@ export function MessageGroup({
skillDisplayMap={skillDisplayMap}
displaySources={displaySources}
ragModeEnabled={ragModeEnabled}
threadId={threadId}
/>
</FlipDisplay>
)}
@@ -362,6 +366,7 @@ function ToolCall({
skillDisplayMap = new Map(),
displaySources = [],
ragModeEnabled = false,
threadId,
}: {
id?: string;
messageId?: string;
@@ -373,6 +378,7 @@ function ToolCall({
skillDisplayMap?: ReturnType<typeof buildSkillDisplayMap>;
displaySources?: RuntimeDisplaySource[];
ragModeEnabled?: boolean;
threadId?: string;
}) {
const { t } = useI18n();
const invokedSkillName = extractInvokedSkillName(name, args);
@@ -445,6 +451,23 @@ function ToolCall({
return <DeepResearchProgressStep id={id} args={args} />;
}
const screenshot = browserScreenshotResult(name, result);
if (screenshot && threadId) {
const imageUrl = urlOfArtifact({ filepath: screenshot.path, threadId });
return (
<ChainOfThoughtStep key={id} label="浏览器截图" icon={ImageIcon}>
<a href={imageUrl} target="_blank" rel="noopener noreferrer">
<img
alt="浏览器截图"
className="mt-2 max-h-[32rem] max-w-full rounded-lg border object-contain"
loading="lazy"
src={imageUrl}
/>
</a>
</ChainOfThoughtStep>
);
}
const searchItems = normalizeSearchResults(result);
if (ragModeEnabled && shouldUseDefaultCitationMode(name, result, args) && searchItems.length > 0) {
const defaultItems = syncDisplaySourceIndexes(
@@ -669,6 +692,29 @@ function ToolCall({
}
}
function browserScreenshotResult(
name: string,
result: string | Record<string, unknown> | undefined,
): { path: string } | null {
if (name !== "browser_skill" || result == null) return null;
let parsed: unknown = result;
if (typeof result === "string") {
try {
parsed = JSON.parse(result);
} catch {
return null;
}
}
if (!parsed || typeof parsed !== "object") return null;
const value = parsed as Record<string, unknown>;
return value.captured === true &&
value.display_to_user === true &&
typeof value.path === "string" &&
value.path.startsWith("/mnt/user-data/outputs/browser-screenshots/")
? { path: value.path }
: null;
}
function DeepResearchProgressStep({
id,
args,
+375
View File
@@ -0,0 +1,375 @@
import { apiFetch } from "@/core/api";
import { getBackendBaseURL } from "@/core/config";
import { env } from "@/env";
export interface BrowserSkillDevice {
device_id: string;
label: string;
last_connected_at?: string | null;
}
export interface BrowserSkillStatus {
enabled: boolean;
paired: boolean;
connected: boolean;
runtime_id?: string | null;
instance_id?: string | null;
busy?: boolean;
device?: BrowserSkillDevice | null;
}
export interface BrowserTaskStatus {
enabled: boolean;
connected: boolean;
selected: boolean;
task_id: string;
}
export interface BrowserTaskPreview {
image_base64: string;
format: "png" | "jpeg";
captured_at: string;
tab_id: number;
title?: string;
viewport?: { width: number; height: number };
pointer?: {
x: number;
y: number;
action: "click" | "hover" | "wheel";
sequence: number;
occurred_at: string;
};
}
interface BrowserSkillEnvelope<T> {
success: boolean;
data: T;
}
const EXTENSION_PAIR_REQUEST = "cmzs-browser-skill-pair-request";
const EXTENSION_RECONNECT_REQUEST = "cmzs-browser-skill-reconnect-request";
const EXTENSION_IDENTIFY_REQUEST = "cmzs-browser-skill-identify-request";
const EXTENSION_PAIR_RESPONSE = "cmzs-browser-skill-pair-response";
const EXTENSION_RECONNECT_RESPONSE = "cmzs-browser-skill-reconnect-response";
const EXTENSION_IDENTIFY_RESPONSE = "cmzs-browser-skill-identify-response";
interface ChromeExternalMessageResponse {
success?: boolean;
error?: string;
runtime_id?: string;
instance_id?: string;
browser?: "chrome" | "edge";
}
export interface BrowserSkillExtensionEvent {
type: "browser-skill-connection-replaced";
runtime_id?: string;
}
interface ChromeExternalPort {
onMessage: {
addListener(listener: (message: BrowserSkillExtensionEvent) => void): void;
removeListener(listener: (message: BrowserSkillExtensionEvent) => void): void;
};
onDisconnect: {
addListener(listener: () => void): void;
removeListener(listener: () => void): void;
};
disconnect(): void;
}
export interface BrowserSkillExtensionIdentity {
runtime_id?: string;
instance_id?: string;
browser?: "chrome" | "edge";
has_remote_pairing?: boolean;
config?: {
enabled?: boolean;
mode?: "local" | "remote";
localUrl?: string;
};
remote?: {
url?: string;
deviceId?: string;
expiresAt?: string;
renewAfter?: string;
} | null;
}
interface ChromeExternalRuntime {
lastError?: { message?: string };
sendMessage(
extensionId: string,
message: Record<string, unknown>,
callback: (response?: ChromeExternalMessageResponse) => void,
): void;
connect(extensionId: string, connectInfo: { name: string }): ChromeExternalPort;
}
function chromeExternalRuntime(): ChromeExternalRuntime | undefined {
return (
window as typeof window & {
chrome?: { runtime?: ChromeExternalRuntime };
}
).chrome?.runtime;
}
async function sendExternalExtensionMessage(
message: Record<string, unknown>,
): Promise<ChromeExternalMessageResponse | null> {
if (!env.BROWSER_EXTENSION_ID) return null;
const runtime = chromeExternalRuntime();
if (!runtime) return null;
return new Promise<ChromeExternalMessageResponse | null>((resolve, reject) => {
runtime.sendMessage(
env.BROWSER_EXTENSION_ID,
message,
(response) => {
// A missing or mismatched configured extension is unavailable. Never
// fall back to a different CMZS installation in the current browser.
if (runtime.lastError) {
resolve(null);
return;
}
if (response?.success === true) {
resolve(response);
return;
}
if (response?.error) {
reject(new Error(response.error));
return;
}
resolve(null);
},
);
});
}
function responseTypeFor(requestType: string): string {
if (requestType === EXTENSION_PAIR_REQUEST) return EXTENSION_PAIR_RESPONSE;
if (requestType === EXTENSION_RECONNECT_REQUEST) return EXTENSION_RECONNECT_RESPONSE;
return EXTENSION_IDENTIFY_RESPONSE;
}
async function sendExtensionBridgeMessage(
requestType: string,
requestId: string,
payload: Record<string, unknown>,
timeoutMs = 1800,
): Promise<ChromeExternalMessageResponse | null> {
const expectedResponseType = responseTypeFor(requestType);
return new Promise((resolve, reject) => {
const timer = window.setTimeout(() => {
window.removeEventListener("message", handleMessage);
resolve(null);
}, timeoutMs);
const finish = (response: ChromeExternalMessageResponse | null) => {
window.clearTimeout(timer);
window.removeEventListener("message", handleMessage);
resolve(response);
};
function handleMessage(event: MessageEvent) {
if (event.source !== window || event.origin !== window.location.origin) return;
const response = event.data as (ChromeExternalMessageResponse & {
type?: string;
requestId?: string;
}) | null;
if (response?.type !== expectedResponseType || response.requestId !== requestId) return;
if (response.success === true) {
finish(response);
} else {
window.clearTimeout(timer);
window.removeEventListener("message", handleMessage);
reject(new Error(response.error || "浏览器插件请求失败"));
}
}
window.addEventListener("message", handleMessage);
window.postMessage({ type: requestType, requestId, ...payload }, window.location.origin);
});
}
async function sendToConfiguredExtension(
requestType: string,
payload: Record<string, unknown>,
timeoutMs = 12_000,
): Promise<BrowserSkillExtensionIdentity> {
const requestId = crypto.randomUUID();
const directResponse = await sendExternalExtensionMessage({
type: requestType,
requestId,
...payload,
});
if (directResponse) {
return directResponse;
}
// Chrome and Edge can assign different IDs to the same unpacked build.
// The content-script bridge addresses the extension installed in the
// browser that owns this page, so automatic pairing still targets the
// current browser when the configured direct ID does not match.
const bridgeResponse = await sendExtensionBridgeMessage(
requestType,
requestId,
payload,
timeoutMs,
);
if (bridgeResponse) {
return bridgeResponse;
}
throw new BrowserSkillExtensionUnavailableError();
}
export class BrowserSkillExtensionUnavailableError extends Error {
constructor() {
super("未检测到 CMZS 插件,请先安装并启用");
this.name = "BrowserSkillExtensionUnavailableError";
}
}
export function subscribeBrowserSkillExtensionEvents(
onEvent: (event: BrowserSkillExtensionEvent) => void,
): () => void {
const handleWindowMessage = (event: MessageEvent) => {
if (event.source !== window || event.origin !== window.location.origin) return;
const message = event.data as BrowserSkillExtensionEvent | null;
if (message?.type === "browser-skill-connection-replaced") {
onEvent(message);
}
};
window.addEventListener("message", handleWindowMessage);
const runtime = chromeExternalRuntime();
if (!runtime || !env.BROWSER_EXTENSION_ID) {
return () => window.removeEventListener("message", handleWindowMessage);
}
let port: ChromeExternalPort;
try {
port = runtime.connect(env.BROWSER_EXTENSION_ID, {
name: "cmzs-browser-skill-status",
});
} catch {
return () => window.removeEventListener("message", handleWindowMessage);
}
const handleMessage = (message: BrowserSkillExtensionEvent) => {
if (message?.type === "browser-skill-connection-replaced") {
onEvent(message);
}
};
const handleDisconnect = () => undefined;
port.onMessage.addListener(handleMessage);
port.onDisconnect.addListener(handleDisconnect);
return () => {
window.removeEventListener("message", handleWindowMessage);
port.onMessage.removeListener(handleMessage);
port.onDisconnect.removeListener(handleDisconnect);
try {
port.disconnect();
} catch {
// The extension may already have restarted or been removed.
}
};
}
function publicBaseURL(): string {
const configured = getBackendBaseURL().replace(/\/+$/, "");
if (configured) return configured;
const markers = [
"/platform/",
"/page/",
"/chats/",
"/agents/",
"/task-management/",
"/dashboard/",
];
const pathname = window.location.pathname;
const markerIndex = markers
.map((marker) => pathname.indexOf(marker))
.filter((index) => index >= 0)
.sort((left, right) => left - right)[0];
const basePath = markerIndex === undefined ? "" : pathname.slice(0, markerIndex);
return `${window.location.origin}${basePath}`.replace(/\/+$/, "");
}
function endpoint(): string {
return `${publicBaseURL()}/api/v1/me/browser`;
}
async function parse<T>(response: Response): Promise<T> {
const payload = (await response.json()) as BrowserSkillEnvelope<T>;
return payload.data;
}
export async function getBrowserSkillStatus(): Promise<BrowserSkillStatus | null> {
const response = await apiFetch(endpoint());
if (response.status === 404) return null;
if (!response.ok) throw new Error("无法获取浏览器连接状态");
return parse<BrowserSkillStatus>(response);
}
export async function createBrowserSkillPairing(): Promise<string> {
const response = await apiFetch(endpoint(), {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ action: "pair", origin: publicBaseURL() }),
});
if (!response.ok) throw new Error("无法生成配对链接");
const data = await parse<{ pairing_link: string }>(response);
return data.pairing_link;
}
export async function pairBrowserSkillThroughExtension(
pairingLink: string,
timeoutMs = 12_000,
): Promise<BrowserSkillExtensionIdentity> {
return sendToConfiguredExtension(
EXTENSION_PAIR_REQUEST,
{ pairingLink },
timeoutMs,
);
}
export async function reconnectBrowserSkillThroughExtension(
timeoutMs = 12_000,
): Promise<BrowserSkillExtensionIdentity> {
return sendToConfiguredExtension(
EXTENSION_RECONNECT_REQUEST,
{},
timeoutMs,
);
}
export async function identifyBrowserSkillExtension(): Promise<BrowserSkillExtensionIdentity> {
return sendToConfiguredExtension(EXTENSION_IDENTIFY_REQUEST, {});
}
export async function revokeBrowserSkillDevice(): Promise<void> {
const response = await apiFetch(endpoint(), { method: "DELETE" });
if (!response.ok) throw new Error("撤销浏览器授权失败");
}
function taskEndpoint(threadId: string): string {
return `${publicBaseURL()}/api/threads/${encodeURIComponent(threadId)}/local-browser`;
}
export async function getBrowserTaskStatus(threadId: string): Promise<BrowserTaskStatus> {
const response = await apiFetch(taskEndpoint(threadId));
if (!response.ok) throw new Error("无法获取浏览器任务状态");
return parse<BrowserTaskStatus>(response);
}
export async function browserTaskAction<T>(
threadId: string,
action: "preview" | "focus" | "stop",
): Promise<T> {
const response = await apiFetch(taskEndpoint(threadId), {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ action }),
});
if (!response.ok) throw new Error(`浏览器任务操作失败:${action}`);
return parse<T>(response);
}
@@ -0,0 +1,44 @@
import assert from "node:assert/strict";
import test from "node:test";
// @ts-expect-error Node's strip-types test runner requires the explicit extension.
import {
hasRemoteBrowserPairing,
shouldPairCurrentBrowser,
} from "./connection.ts";
test("账号已配对但当前浏览器无远程凭证时必须重新配对", () => {
assert.equal(
shouldPairCurrentBrowser(true, {
has_remote_pairing: false,
config: { mode: "local" },
remote: null,
}),
true,
);
});
test("当前浏览器持有远程凭证时可以直接重连", () => {
assert.equal(
shouldPairCurrentBrowser(true, {
has_remote_pairing: true,
config: { mode: "remote" },
remote: { deviceId: "device-1" },
}),
false,
);
});
test("兼容尚未返回显式标志的插件版本", () => {
assert.equal(
hasRemoteBrowserPairing({
config: { mode: "remote" },
remote: { deviceId: "device-1" },
}),
true,
);
assert.equal(
hasRemoteBrowserPairing({ config: { mode: "local" }, remote: null }),
false,
);
});
@@ -0,0 +1,28 @@
export interface BrowserSkillExtensionConnectionState {
has_remote_pairing?: boolean;
config?: {
mode?: "local" | "remote";
};
remote?: unknown | null;
}
/** Whether the current browser installation still owns a remote credential. */
export function hasRemoteBrowserPairing(
identity: BrowserSkillExtensionConnectionState,
): boolean {
if (typeof identity.has_remote_pairing === "boolean") {
return identity.has_remote_pairing;
}
return identity.config?.mode === "remote" && identity.remote != null;
}
/**
* Account-level pairing is not enough: another Chrome/Edge installation may
* own it while this browser has already cleared its displaced credential.
*/
export function shouldPairCurrentBrowser(
serverPaired: boolean,
identity: BrowserSkillExtensionConnectionState,
): boolean {
return !serverPaired || !hasRemoteBrowserPairing(identity);
}
+2
View File
@@ -28,6 +28,8 @@ export interface AgentThreadContext extends Record<string, unknown> {
summarization_enabled?: boolean;
/** Per-conversation switch for knowledge-base RAG injection (phase 2). */
knowledge_rag_enabled?: boolean;
/** Per-request opt-in for the independent BrowserSkill tool. */
browser_skill_enabled?: boolean;
/** DeerFlow LLMWiki mapping ids selected for this conversation. */
llmwiki_knowledge_base_ids?: string[];
/** Assistant knowledge-base ids selected for this conversation. */
+2
View File
@@ -70,6 +70,8 @@ export const env = {
* type (A / B) so two deployments can show different left-nav items.
*/
COMPACT_MENU_TYPE: readRuntimeEnv("VITE_COMPACT_MENU_TYPE"),
/** CMZS extension id used by the one-click local-browser pairing flow. */
BROWSER_EXTENSION_ID: readRuntimeEnv("VITE_BROWSER_EXTENSION_ID") ?? "",
GITHUB_OAUTH_TOKEN: readRuntimeEnv("VITE_GITHUB_OAUTH_TOKEN"),
};
+15 -1
View File
@@ -26,6 +26,7 @@ import { ExportTrigger } from "@/components/workspace/export-trigger";
import { KnowledgeSaveTrigger } from "@/components/workspace/knowledge-save-trigger";
import { LlmWikiKnowledgeSelector } from "@/components/workspace/llmwiki-knowledge-selector";
import { InputBox } from "@/components/workspace/input-box";
import { BrowserTaskPreview } from "@/components/workspace/browser-task-preview";
import {
MessageList,
MESSAGE_LIST_DEFAULT_PADDING_BOTTOM,
@@ -212,6 +213,7 @@ export default function ChatPage() {
// In-memory only — explicitly overrides any value persisted by the old
// per-thread header toggle so stale localStorage can't re-enable it.
const [knowledgeRagEnabled] = useKnowledgeRagEnabled();
const [browserSkillActive, setBrowserSkillActive] = useState(false);
const assistantKnowledgeAccess = useAssistantKnowledgeAccess();
const [selectedKnowledgeBaseIds, setSelectedKnowledgeBaseIds] = useState<string[]>([]);
const [selectedAssistantKnowledgeBaseIds, setSelectedAssistantKnowledgeBaseIds] = useState<string[]>([]);
@@ -262,12 +264,14 @@ export default function ChatPage() {
() => ({
...settings.context,
knowledge_rag_enabled: knowledgeRagEnabled,
browser_skill_enabled: browserSkillActive,
...knowledgeRetrievalRequestContext,
}),
[
settings.context,
effectiveKnowledgeRetrievalMode,
knowledgeRagEnabled,
browserSkillActive,
knowledgeRetrievalRequestContext,
],
);
@@ -1175,6 +1179,10 @@ export default function ChatPage() {
</div>
</header>
<main className="flex min-h-0 max-w-full grow flex-col">
<BrowserTaskPreview
threadId={threadId}
enabled={browserSkillActive}
/>
<div className="flex size-full justify-center">
{/*
Landing state covers two cases: (a) a brand-new chat that
@@ -1278,8 +1286,10 @@ export default function ChatPage() {
</div>
)}
<InputBox
className="w-full"
className="w-full *:data-[slot='input-group']:border-input *:data-[slot='input-group']:ring-[3px] *:data-[slot='input-group']:ring-ring/50"
isNewThread={isLandingState}
mirrorLandingFrame={!isLandingState}
frameRadius={isLandingState ? "default" : "tight"}
placeholder={
isLandingState
? qaPageDescription?.input_placeholder
@@ -1297,6 +1307,10 @@ export default function ChatPage() {
: "ready"
}
context={settings.context}
browserSkill={{
active: browserSkillActive,
onActiveChange: setBrowserSkillActive,
}}
extraModels={SINGLE_ROUND_RAG_MODELS}
disabled={
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" ||
@@ -240,8 +240,11 @@ function SentimentComposer({
/>
</PromptInputBody>
<PromptInputFooter className="flex items-center justify-end px-2 pt-1 pb-1.5">
{/* 皮肤(主题色渐变圆钮)由全局 .agent-composer-send 提供 */}
<PromptInputSubmit status={status} />
<PromptInputSubmit
className="rounded-full border-border bg-background shadow-sm hover:bg-muted dark:border-blue-500 dark:bg-blue-500 dark:text-white dark:hover:bg-blue-400 dark:shadow-none"
variant="outline"
status={status}
/>
</PromptInputFooter>
</PromptInput>
);
+21 -183
View File
@@ -1422,21 +1422,20 @@
}
/* ── Landing 态输入框 Gemini 风格:旋转彩色描边 + 多色柔光阴影 ───────── */
/* 关键:::before 转动的彩色锥形渐变 + ::after 内嵌白板遮罩中央,只在边缘可见模糊彩光。
* 彩灯颜色跟随当前主题(--legacy-primary-csv:浅红主题=红色系彩灯,浅蓝主题=蓝色系),
* 明暗档交替让旋转肉眼可辨;智能体专属落地页按 [data-agent-style] 的
* --agent-gemini-gradient 覆盖(风格 0 经典红无该变量,同样吃到主题色兜底)。 */
/* 关键:::before 转动的彩色锥形渐变 + ::after 内嵌白板遮罩中央,只在 1.5px 边缘可见模糊彩光 */
.gemini-shadow {
box-shadow:
0 0 20px rgba(var(--legacy-primary-csv), 0.12),
0 0 40px rgba(var(--legacy-primary-csv), 0.07),
0 0 60px rgba(var(--legacy-primary-csv), 0.05);
0 0 20px rgba(66, 133, 244, 0.10),
0 0 40px rgba(219, 68, 55, 0.05),
0 0 60px rgba(244, 180, 0, 0.05),
0 0 80px rgba(15, 157, 88, 0.05);
}
.dark .gemini-shadow {
box-shadow:
0 0 24px rgba(var(--legacy-primary-csv), 0.22),
0 0 48px rgba(var(--legacy-primary-csv), 0.12),
0 0 72px rgba(var(--legacy-primary-csv), 0.08);
0 0 24px rgba(66, 133, 244, 0.20),
0 0 48px rgba(219, 68, 55, 0.10),
0 0 72px rgba(244, 180, 0, 0.10),
0 0 96px rgba(15, 157, 88, 0.10);
}
.gemini-gradient-border {
@@ -1456,27 +1455,27 @@
top: -50%;
left: -50%;
/* 智能体专属落地页风格可按 [data-agent-style] 覆盖为该智能体的专属渐变;
缺省回退到主题色明暗交替的旋转彩灯。blur 收窄让色环更饱和清晰。 */
缺省回退到全局 Gemini 四色(其他页面零变化)。 */
background: var(
--agent-gemini-gradient,
conic-gradient(
from 0deg,
rgba(var(--legacy-primary-csv), 0.95) 0deg,
rgba(var(--legacy-primary-csv), 0.25) 90deg,
rgba(var(--legacy-primary-csv), 0.9) 180deg,
rgba(var(--legacy-primary-csv), 0.3) 270deg,
rgba(var(--legacy-primary-csv), 0.95) 360deg
#4285F4 0deg,
#DB4437 90deg,
#F4B400 180deg,
#0F9D58 270deg,
#4285F4 360deg
)
);
z-index: -2;
animation: gemini-rotate 4s linear infinite;
filter: blur(6px);
filter: blur(10px);
pointer-events: none;
}
.gemini-gradient-border::after {
content: '';
position: absolute;
inset: 2px;
inset: 2.5px;
background: var(--background);
border-radius: calc(var(--agent-composer-radius, 1.5rem) - 2px);
z-index: -1;
@@ -1501,63 +1500,6 @@
}
}
/* ── 对话态输入框外壳(文心一言式) ─────────────────────────────────────
* 空闲:主题色浅调细边白卡,悬停/聚焦时边色加深;生成中:细边转透明,
* 由 border-box 渐变环接管并沿边框流动(与落地彩灯同一主题色来源)。
* 外壳类挂在 input-box.tsx 的包裹 div 上,仅对话态(含紧凑会话)使用;
* Gemini 落地彩灯与紧凑落地白卡不走这套。表单本体与内部 input-group
* 透明无边框,避免双层描边。颜色取 --legacy-primary-csv(Firefox 102 兼容)。 */
.composer-shell {
border-radius: 1.5rem;
}
.composer-shell--idle {
border: 1.5px solid rgba(var(--legacy-primary-csv), 0.26);
background: var(--background);
box-shadow: 0 2px 12px rgba(var(--legacy-primary-csv), 0.06);
transition:
border-color 0.25s ease,
box-shadow 0.25s ease;
}
.composer-shell--idle:hover {
border-color: rgba(var(--legacy-primary-csv), 0.36);
}
.composer-shell--idle:focus-within {
border-color: rgba(var(--legacy-primary-csv), 0.48);
box-shadow: 0 4px 18px rgba(var(--legacy-primary-csv), 0.1);
}
.dark .composer-shell--idle {
border-color: rgba(var(--legacy-primary-csv), 0.4);
}
/* 生成中:渐变环图宽 200%,background-position 滚动一个图宽即无缝循环;
* 渐变首尾同色保证衔接。 */
.composer-shell--streaming {
border: 2px solid transparent;
background-image:
linear-gradient(var(--background), var(--background)),
linear-gradient(
115deg,
rgba(var(--legacy-primary-csv), 0.9),
rgba(var(--legacy-primary-csv), 0.28) 25%,
rgba(var(--legacy-primary-csv), 0.85) 50%,
rgba(var(--legacy-primary-csv), 0.28) 75%,
rgba(var(--legacy-primary-csv), 0.9)
);
background-origin: padding-box, border-box;
background-clip: padding-box, border-box;
background-size: auto, 200% 100%;
animation: composer-stream-slide 3s linear infinite;
}
@keyframes composer-stream-slide {
to {
background-position: 0 0, 200% 0;
}
}
@media (prefers-reduced-motion: reduce) {
.composer-shell--streaming {
animation: none;
}
}
* {
outline-color: var(--ring);
}
@@ -2696,11 +2638,8 @@ html[data-df-framed="1"] [class~="bg-card/90"] {
color: inherit;
}
/* 面板整列卡片跟随专属风格:不只是描边换色——每套风格一种「卡片形态」,
底色 / 边框形态 / 阴影 / 圆角全部不同(与输入框、工具条形态同一节奏)。
0 经典红 白卡专属描边(基准) / 1 蓝 无边框浅蓝软卡+柔影 /
2 绿 白卡+顶部3px强调线+直角 / 3 金 虚线凭证卡+米金浅底 /
4 紫 渐变玻璃卡 / 5 青 贴纸硬偏移阴影。 */
/* 面板整列卡片跟随专属风格:描边染色 + 每风格一套卡片圆角
(与输入框圆角同一节奏)+ 小节标题图标染色。 */
[data-agent-style] .agent-detail-section {
border-color: var(--agent-accent-border);
border-radius: var(--agent-detail-card-radius, 0.75rem);
@@ -2726,45 +2665,6 @@ html[data-df-framed="1"] [class~="bg-card/90"] {
[data-agent-style="5"] {
--agent-detail-card-radius: 0.5rem;
}
[data-agent-style="1"] .agent-detail-section {
border-color: transparent;
background-color: var(--agent-accent-soft);
box-shadow: 0 4px 18px rgb(43 129 255 / 0.1);
}
.dark [data-agent-style="1"] .agent-detail-section {
box-shadow: 0 4px 18px rgb(43 129 255 / 0.18);
}
[data-agent-style="2"] .agent-detail-section {
border-top: 3px solid var(--agent-accent);
background-color: var(--background);
}
[data-agent-style="3"] .agent-detail-section {
border: 1.5px dashed rgb(217 119 6 / 0.45);
background-color: var(--agent-accent-soft);
}
.dark [data-agent-style="3"] .agent-detail-section {
border-color: rgb(252 211 77 / 0.45);
}
[data-agent-style="4"] .agent-detail-section {
background-image: linear-gradient(
180deg,
var(--agent-accent-soft),
transparent 72%
);
}
[data-agent-style="5"] .agent-detail-section {
border-color: rgb(8 145 178 / 0.45);
box-shadow: 4px 4px 0 rgb(8 145 178 / 0.15);
}
.dark [data-agent-style="5"] .agent-detail-section {
border-color: rgb(103 232 249 / 0.5);
box-shadow: 4px 4px 0 rgb(8 145 178 / 0.22);
}
/* 技能行卡片在浅底卡片内(蓝/金)回到白底,避免双层同色叠加 */
[data-agent-style="1"] .agent-skill-row,
[data-agent-style="3"] .agent-skill-row {
background-color: var(--background);
}
/* 技能行卡片:同色浅底 + 描边,圆角跟随该风格的卡片档位;悬停加深一档;
首字方块白底描边 + 同色字 */
@@ -2917,69 +2817,9 @@ html[data-df-framed="1"] [class~="bg-card/90"] {
10px 10px 0 rgb(8 145 178 / 0.22);
}
/* ── 发送按钮皮肤:主题色扁平渐变圆钮 + 生成中低对比停止钮 ─────────────
钩子类 agent-composer-send 由 prompt-input.tsx 挂在所有 composer 的提交
按钮上。底色与图标色读取当前主题 token,渐变只叠加中性明暗层,
因此内置主题与管理员自定义主题都会自动生成同色系渐变;
不使用内高光、悬浮位移或投影,避免按钮呈现玻璃/浮雕质感。
下方 [data-agent-style]
智能体专属落地页仍以更高优先级覆盖。 */
.agent-composer-send {
border: 0;
border-radius: 9999px;
color: var(--button-foreground, var(--primary-foreground));
background-color: var(--primary);
background-image: linear-gradient(
135deg,
rgba(255, 255, 255, 0.06) 0%,
rgba(255, 255, 255, 0.24) 100%
);
box-shadow: none;
transition:
opacity 180ms ease,
background-image 180ms ease;
}
.agent-composer-send:hover:not(:disabled) {
background-image: linear-gradient(
135deg,
rgba(0, 0, 0, 0.1) 0%,
rgba(255, 255, 255, 0.1) 100%
);
}
.agent-composer-send:active:not(:disabled) {
opacity: 0.82;
}
.agent-composer-send:focus-visible {
box-shadow: 0 0 0 3px rgba(var(--legacy-primary-csv), 0.18);
}
/* 图标跟随主题前景色,浅色主色主题会自动切换为深色图标。 */
.agent-composer-send svg {
color: currentColor;
}
.agent-composer-send--stop {
background-color: var(--primary);
background-image: linear-gradient(
135deg,
rgba(0, 0, 0, 0.08) 0%,
rgba(255, 255, 255, 0.12) 100%
);
box-shadow: none;
}
.agent-composer-send--stop:hover:not(:disabled) {
background-color: var(--primary);
background-image: linear-gradient(
135deg,
rgba(0, 0, 0, 0.16) 0%,
rgba(255, 255, 255, 0.04) 100%
);
}
.dark .agent-composer-send {
box-shadow: none;
}
/* 智能体专属「输入框内部控件」:发送按钮与工具条按钮的形状/阴影也按风格
区分,和框体形态呼应。钩子类 agent-composer-send / agent-composer-tool
由 prompt-input.tsx 挂载(默认皮肤见上方全局渐变圆钮);只作用于 landing 列
由 prompt-input.tsx 挂载(全局标记,无默认样式);只作用于 landing 列
(.agent-style-scope),对话态与通用问答不受影响。 */
/* 框体圆角补齐:蓝/紫两档用更紧凑的中圆角,与其余三档拉开差距 */
@@ -2989,11 +2829,9 @@ html[data-df-framed="1"] [class~="bg-card/90"] {
}
/* 发送按钮:圆 / 方随风格轮换,阴影呼应框体主题;landing 列内统一改为
智能体专属色实底(覆盖全局渐变圆钮皮肤,需同时清掉渐变图),颜色随
--agent-accent 走 */
智能体专属色实底(覆盖白底 outline 皮肤),颜色随 --agent-accent 走 */
[data-agent-style] .agent-style-scope .agent-composer-send {
background-color: var(--agent-accent);
background-image: none;
border-color: transparent;
color: #ffffff;
}
@@ -46,6 +46,7 @@ from app.gateway.routers import (
auth,
service_tokens,
browser_context,
browser_skill,
business_mapping,
channels,
chat_archive,
@@ -1634,8 +1635,38 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
if getattr(app.state, "worker_pool", None) is not None:
app.state.worker_pool.start()
# BrowserSkill is an independent browser-control channel. It does not
# share the legacy BrowserContext router, tools, or state. The daemon
# itself remains lazy and starts only when an extension connects or a
# browser tool call is made.
from deerflow.integrations.browser_skill.manager import BrowserSkillManager
from deerflow.integrations.browser_skill.runtime import set_browser_skill_manager
from deerflow.persistence.engine import get_session_factory
browser_skill_config = app.state.config.browser_skill
app.state.browser_skill_manager = None
set_browser_skill_manager(None)
if browser_skill_config.enabled:
session_factory = get_session_factory()
if session_factory is None:
raise RuntimeError("BrowserSkill requires an initialized database session factory")
browser_skill_manager = BrowserSkillManager(
enabled=True,
binary=browser_skill_config.binary,
max_connections=browser_skill_config.max_connections,
session_factory=session_factory,
)
app.state.browser_skill_manager = browser_skill_manager
set_browser_skill_manager(browser_skill_manager)
logger.info("BrowserSkill manager initialized")
yield
browser_skill_manager = getattr(app.state, "browser_skill_manager", None)
if browser_skill_manager is not None:
await browser_skill_manager.close()
set_browser_skill_manager(None)
artifact_library_tasks = list(
getattr(app.state, "artifact_library_sync_tasks", {}).values()
)
@@ -2096,6 +2127,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Optional Chrome-extension page capture API is mounted at /api/browser-context
app.include_router(browser_context.router)
# BrowserSkill is a separate productized browser-control stack. Its routes
# are absent when disabled, matching the frontend and tool-catalog gates.
if get_app_config().browser_skill.enabled:
app.include_router(browser_skill.router)
# Thread cleanup API is mounted at /api/threads/{thread_id}
from app.gateway.routers import analysis_projects
@@ -78,6 +78,7 @@ _PUBLIC_EXACT_PATHS: frozenset[str] = frozenset(
"/api/writing/export/docx",
# 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。
"/api/parallel-agents/auth/token",
"/api/v1/local-browser/extension/authorize",
}
)
@@ -138,6 +138,10 @@ _AUTH_EXEMPT_PATHS: frozenset[str] = frozenset(
"/api/v1/auth/logout",
"/api/v1/auth/register",
"/api/v1/auth/initialize",
# BrowserSkill exchanges its one-time pairing credential before it has
# a DeerFlow browser session or CSRF cookie. Device authentication is
# performed by the dedicated BrowserSkill manager instead.
"/api/v1/local-browser/extension/authorize",
# 并行多智能体面板专用取 token 端点(公共 + CSRF 豁免)。
"/api/parallel-agents/auth/token",
}
@@ -0,0 +1,168 @@
"""Independent BrowserSkill pairing, status, and extension transport APIs."""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, HTTPException, Request, WebSocket
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user
from deerflow.integrations.browser_skill.runtime import get_browser_skill_manager
from deerflow.persistence.engine import get_session_factory
router = APIRouter(tags=["browser-skill"])
class PairBrowserRequest(BaseModel):
action: str = "pair"
origin: str = Field(description="Public frontend origin, including any deployment base path")
class AuthorizeBrowserRequest(BaseModel):
action: str
next_token: str
label: str = "BrowserSkill Extension"
class BrowserTaskActionRequest(BaseModel):
action: str
def _bearer(value: str | None) -> str:
if not value:
raise HTTPException(status_code=401, detail="missing device credential")
scheme, _, credential = value.partition(" ")
if scheme.lower() != "bearer" or not credential:
raise HTTPException(status_code=401, detail="invalid device credential")
return credential.strip()
async def _user_id(request: Request) -> str:
user_id = await get_current_user(request)
if not user_id:
raise HTTPException(status_code=401, detail="authentication required")
return user_id
@router.get("/api/v1/me/browser")
async def browser_status(request: Request) -> dict[str, Any]:
manager = get_browser_skill_manager()
async with get_session_factory()() as session:
return {"success": True, "data": await manager.status(session, await _user_id(request))}
@router.post("/api/v1/me/browser")
async def pair_browser(request: Request, body: PairBrowserRequest) -> dict[str, Any]:
if body.action != "pair":
raise HTTPException(status_code=400, detail="unsupported browser action")
manager = get_browser_skill_manager()
try:
async with get_session_factory()() as session:
link = await manager.create_pairing(session, await _user_id(request), body.origin)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
return {"success": True, "data": {"pairing_link": link}}
@router.delete("/api/v1/me/browser")
async def revoke_browser(request: Request) -> dict[str, bool]:
manager = get_browser_skill_manager()
async with get_session_factory()() as session:
await manager.revoke(session, await _user_id(request))
return {"success": True}
@router.get("/api/threads/{thread_id}/local-browser")
async def browser_task_status(request: Request, thread_id: str) -> dict[str, Any]:
manager = get_browser_skill_manager()
return {
"success": True,
"data": manager.task_status(
user_id=await _user_id(request), thread_id=thread_id
),
}
@router.post("/api/threads/{thread_id}/local-browser")
async def browser_task_action(
request: Request, thread_id: str, body: BrowserTaskActionRequest
) -> dict[str, Any]:
manager = get_browser_skill_manager()
try:
data = await manager.task_action(
user_id=await _user_id(request),
thread_id=thread_id,
action=body.action,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
return {"success": True, "data": data}
@router.post("/api/v1/local-browser/extension/authorize")
async def authorize_browser(request: Request, body: AuthorizeBrowserRequest) -> dict[str, Any]:
manager = get_browser_skill_manager()
credential = _bearer(request.headers.get("authorization"))
try:
async with get_session_factory()() as session:
return await manager.authorize(
session,
credential=credential,
action=body.action,
next_token=body.next_token,
label=body.label,
)
except PermissionError as exc:
raise HTTPException(status_code=401, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
def _browser_device_credential(subprotocol: str) -> str:
"""Extract the stored device credential from a WebSocket subprotocol."""
if subprotocol.startswith("bsk-auth."):
return subprotocol.removeprefix("bsk-auth.")
scheme, separator, credential = subprotocol.partition(" ")
if separator and scheme.lower() == "bearer":
return credential.strip()
return subprotocol
@router.websocket("/api/v1/local-browser/extension")
async def browser_extension_socket(websocket: WebSocket) -> None:
manager = get_browser_skill_manager()
requested = [
item.strip()
for item in websocket.headers.get("sec-websocket-protocol", "").split(",")
if item.strip()
]
authenticated = None
accepted_protocol = None
async with get_session_factory()() as session:
for candidate in reversed(requested):
authenticated = await manager.authenticate_device(
session,
_browser_device_credential(candidate),
)
if authenticated is not None:
accepted_protocol = candidate
break
if authenticated is None or accepted_protocol is None:
await websocket.close(code=1008, reason="invalid device authorization")
return
await manager.proxy_extension(
websocket,
user_id=authenticated.user_id,
device_id=authenticated.device_id,
subprotocol=accepted_protocol,
origin=websocket.headers.get("origin", "chrome-extension://browser-skill"),
)
@@ -527,6 +527,10 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"reasoning_effort",
"rag_mode_enabled",
"knowledge_rag_enabled",
# Per-request BrowserSkill opt-in. Without this whitelist entry the
# frontend flag is present in the HTTP payload but silently disappears
# before the lead-agent graph resolves its tool surface.
"browser_skill_enabled",
"llmwiki_knowledge_base_ids",
"assistant_knowledge_base_ids",
"knowledge_space_strict",
+10 -2
View File
@@ -304,6 +304,14 @@ browser_context:
enabled: true
actions_enabled: true
# BrowserSkill 浏览器扩展集成,与上面的 browser_act/browser_context 完全独立。
# 生产部署建议通过 BROWSERSKILL_ENABLED / BROWSERSKILL_BINARY 环境变量覆盖。
# binary 必须指向与后端运行系统及 CPU 架构匹配的 bsk 可执行文件。
browser_skill:
enabled: false
binary: null
max_connections: 100
# 上传文件识别配置。
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。
@@ -666,8 +674,8 @@ llmwiki:
# The restored WeKnora deployment exposes its API at the site root.
# The historical /deerflow-api prefix is a frontend-only route there and
# turns API mutations (POST/OPTIONS) into nginx 405 responses.
api_base_url: "http://119.254.86.251:8088"
web_base_url: "http://119.254.86.251:8088"
api_base_url: ""
web_base_url: ""
admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX"
# 高风险能力,默认关闭。连接参数仅允许后端配置;生产环境请使用环境变量。
@@ -1147,6 +1147,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
allow_agent_orchestration=agent_id in _COORDINATOR_AGENT_IDS,
app_config=resolved_app_config,
write_file_compatibility=write_file_compatibility,
browser_skill_requested=bool(cfg.get("browser_skill_enabled", False)),
)
)
if agent_id == FORCED_RESEARCH_AGENT_ID:
@@ -12,6 +12,7 @@ from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.api_read_cache_config import ApiReadCacheConfig
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.browser_skill_config import BrowserSkillConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.concurrency_config import ConcurrencyConfig
from deerflow.config.database_config import DatabaseConfig
@@ -161,6 +162,7 @@ class AppConfig(BaseModel):
description="Optional Redis cache for homepage agents/skills lists; MySQL remains authoritative.",
)
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
browser_skill: BrowserSkillConfig = Field(default_factory=BrowserSkillConfig, description="Independent BrowserSkill extension integration")
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
@@ -0,0 +1,29 @@
import os
from pydantic import BaseModel, Field, model_validator
def _env_bool(name: str, default: bool = False) -> bool:
value = os.getenv(name)
return default if value is None else value.strip().lower() in {"1", "true", "yes", "on"}
class BrowserSkillConfig(BaseModel):
"""Independent BrowserSkill extension/daemon integration."""
enabled: bool = Field(default_factory=lambda: _env_bool("BROWSERSKILL_ENABLED"))
binary: str | None = Field(default_factory=lambda: os.getenv("BROWSERSKILL_BINARY") or None)
max_connections: int = Field(
default_factory=lambda: int(os.getenv("BROWSERSKILL_MAX_CONNECTIONS", "100"))
)
@model_validator(mode="after")
def apply_environment_overrides(self) -> "BrowserSkillConfig":
"""Let deployment environment variables override config.yaml values."""
if "BROWSERSKILL_ENABLED" in os.environ:
self.enabled = _env_bool("BROWSERSKILL_ENABLED")
if binary := os.getenv("BROWSERSKILL_BINARY"):
self.binary = binary
if max_connections := os.getenv("BROWSERSKILL_MAX_CONNECTIONS"):
self.max_connections = int(max_connections)
return self
@@ -0,0 +1,9 @@
from .manager import BrowserSkillManager, public_websocket_url
from .runtime import get_browser_skill_manager, set_browser_skill_manager
__all__ = [
"BrowserSkillManager",
"get_browser_skill_manager",
"public_websocket_url",
"set_browser_skill_manager",
]
@@ -0,0 +1,111 @@
import asyncio
import json
import logging
import os
import shutil
import tempfile
from pathlib import Path
logger = logging.getLogger(__name__)
_RPC_STREAM_LIMIT = 16 * 1024 * 1024
class BrowserSkillDaemon:
def __init__(self, binary: str):
self.binary = binary
self.home: Path | None = None
self.process: asyncio.subprocess.Process | None = None
self.ws_port: int | None = None
self._lock = asyncio.Lock()
async def ensure_started(self) -> int:
async with self._lock:
if self.process and self.process.returncode is None and self.ws_port:
return self.ws_port
binary = Path(self.binary).expanduser().resolve()
if not binary.is_file():
raise RuntimeError(f"BrowserSkill binary not found: {binary}")
self.home = Path(tempfile.mkdtemp(prefix="deerflow-bsk-"))
env = {**os.environ, "BSK_HOME": str(self.home)}
self.process = await asyncio.create_subprocess_exec(
str(binary),
"daemon",
"start",
"--foreground",
"--port",
"0",
"--daemon-idle",
"24h",
"--session-idle",
"30m",
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
state_file = self.home / "daemon.json"
for _ in range(100):
if self.process.returncode is not None:
stderr = await self.process.stderr.read() if self.process.stderr else b""
raise RuntimeError(f"BrowserSkill daemon exited: {stderr.decode(errors='replace')}")
if state_file.exists():
state = json.loads(state_file.read_text(encoding="utf-8"))
self.ws_port = int(state["ws_port"])
return self.ws_port
await asyncio.sleep(0.1)
raise RuntimeError("BrowserSkill daemon did not become ready")
async def close(self) -> None:
if self.process and self.process.returncode is None:
self.process.terminate()
try:
await asyncio.wait_for(self.process.wait(), timeout=5)
except TimeoutError:
self.process.kill()
await self.process.wait()
if self.home:
shutil.rmtree(self.home, ignore_errors=True)
self.process = None
self.ws_port = None
async def rpc(self, method: str, params: dict | None = None) -> object:
await self.ensure_started()
if self.home is None:
raise RuntimeError("BrowserSkill daemon home is unavailable")
request_id = os.urandom(12).hex()
# asyncio's default StreamReader limit is only 64 KiB. Screenshot RPC
# responses contain base64 in one newline-delimited JSON record, so even
# a tiny page exceeds that limit and readline() raises LimitOverrunError
# ("Separator is not found, and chunk exceed the limit"). Keep this
# bounded, but large enough for the tool's 8 MiB base64 ceiling.
reader, writer = await asyncio.open_unix_connection(
str(self.home / "run" / "daemon.sock"),
limit=_RPC_STREAM_LIMIT,
)
try:
payload = {
"id": request_id,
"method": method,
"params": params or {},
}
writer.write(json.dumps(payload).encode("utf-8") + b"\n")
await writer.drain()
line = await asyncio.wait_for(reader.readline(), timeout=35)
finally:
writer.close()
await writer.wait_closed()
if not line:
raise RuntimeError("BrowserSkill daemon closed the RPC connection")
response = json.loads(line)
if response.get("id") != request_id:
raise RuntimeError("BrowserSkill daemon returned a mismatched RPC response")
if response.get("error"):
error = response["error"]
if isinstance(error, dict):
message = error.get("message") or json.dumps(error, ensure_ascii=False)
else:
message = str(error)
raise RuntimeError(f"BrowserSkill daemon RPC failed: {message}")
return response.get("result")
@@ -0,0 +1,771 @@
import asyncio
import json
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from websockets.asyncio.client import connect
from deerflow.persistence.browser_skill import (
BrowserDeviceCredentialRow,
BrowserDeviceRow,
BrowserPairingRow,
)
from .daemon import BrowserSkillDaemon
from .security import new_token, token_hash
PAIRING_TTL = timedelta(minutes=5)
DEVICE_TTL = timedelta(days=90)
PREVIOUS_TOKEN_GRACE = timedelta(minutes=5)
ALLOWED_TOOL_METHODS = {
"snapshot",
"observe",
"navigate",
"navigate_back",
"navigate_forward",
"reload",
"click",
"fill",
"press",
"hover",
"wheel",
"scroll_to",
"select",
"tab_list",
"tab_create",
"tab_select",
"tab_close",
"tab_borrow",
"tab_return",
"get_html",
"evaluate",
"screenshot",
"focus",
"blur",
"console",
"network",
"wait_for_navigation",
"wait_ms",
"window_resize",
"emulate",
"request_help",
}
def _as_utc(value: datetime) -> datetime:
"""Normalize datetimes loaded from SQLite before Python-side comparison."""
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
class BrowserSkillManager:
def __init__(
self,
*,
enabled: bool,
binary: str | None,
max_connections: int,
session_factory: async_sessionmaker[AsyncSession],
) -> None:
self.binary = binary
self.daemon = BrowserSkillDaemon(binary) if binary else None
self.configured_enabled = enabled
self.max_connections = max_connections
self.session_factory = session_factory
self.connections: dict[str, Any] = {}
self.task_sessions: dict[tuple[str, str], str] = {}
self._ui_calls: dict[str, tuple[str, asyncio.Future[Any]]] = {}
self._browser_write_locks: dict[str, asyncio.Lock] = {}
self._extension_runtime_ids: dict[str, str] = {}
self._extension_instance_ids: dict[str, str] = {}
self._active_tool_calls: dict[str, int] = {}
self._connections_lock = asyncio.Lock()
@property
def enabled(self) -> bool:
return bool(self.configured_enabled and self.daemon and self.binary and Path(self.binary).expanduser().is_file())
async def create_pairing(self, session: AsyncSession, user_id: str, origin: str) -> str:
if not self.enabled:
raise RuntimeError("BrowserSkill is not configured")
now = datetime.now(UTC)
token = new_token()
pairing = await session.get(BrowserPairingRow, user_id)
if pairing is None:
pairing = BrowserPairingRow(
user_id=user_id,
token_hash=token_hash(token),
expires_at=now + PAIRING_TTL,
created_at=now,
)
session.add(pairing)
else:
pairing.token_hash = token_hash(token)
pairing.expires_at = now + PAIRING_TTL
pairing.created_at = now
await session.commit()
return f"{public_websocket_url(origin)}#{token}"
async def authorize(
self,
session: AsyncSession,
*,
credential: str,
action: str,
next_token: str,
label: str,
) -> dict[str, Any]:
if not self.enabled:
raise RuntimeError("BrowserSkill is disabled or its daemon binary is unavailable")
now = datetime.now(UTC)
presented_hash = token_hash(credential)
next_hash = token_hash(next_token)
credential_row: BrowserDeviceCredentialRow | None = None
if action == "pair":
pairing = await session.scalar(
select(BrowserPairingRow).where(
BrowserPairingRow.token_hash == presented_hash,
BrowserPairingRow.expires_at > now,
)
)
if pairing is None:
raise PermissionError("authorization expired or already used")
device = await session.get(BrowserDeviceRow, pairing.user_id)
if device is None:
device = BrowserDeviceRow(
user_id=pairing.user_id,
device_id=uuid.uuid4().hex,
label=label or "BrowserSkill Extension",
token_hash=next_hash,
expires_at=now + DEVICE_TTL,
created_at=now,
updated_at=now,
)
session.add(device)
else:
# Keep the account-level row alive, but do not rotate its legacy
# token: doing so made a Chrome pairing revoke Edge (and vice
# versa). Each browser installation now owns a separate row.
device.label = label or device.label
device.expires_at = now + DEVICE_TTL
device.revoked_at = None
device.updated_at = now
credential_row = await session.scalar(
select(BrowserDeviceCredentialRow).where(
BrowserDeviceCredentialRow.user_id == pairing.user_id,
BrowserDeviceCredentialRow.label == (label or "BrowserSkill Extension"),
)
)
if credential_row is None:
credential_row = BrowserDeviceCredentialRow(
credential_id=uuid.uuid4().hex,
user_id=pairing.user_id,
device_id=device.device_id,
label=label or "BrowserSkill Extension",
token_hash=next_hash,
expires_at=now + DEVICE_TTL,
created_at=now,
updated_at=now,
)
session.add(credential_row)
else:
self._rotate_credential(credential_row, next_hash, label, now, preserve_previous=False)
await session.delete(pairing)
elif action == "renew":
credential_row = await self._credential_by_token(session, presented_hash, now)
if credential_row is not None:
device = await session.get(BrowserDeviceRow, credential_row.user_id)
if device is None:
raise PermissionError("device authorization is invalid or expired")
self._rotate_credential(credential_row, next_hash, label, now)
device.label = label or device.label
device.expires_at = now + DEVICE_TTL
device.revoked_at = None
device.updated_at = now
else:
# Existing deployments may still present the token stored on the
# legacy account row. Accept it and migrate this browser during
# the next explicit pairing instead of breaking upgrades.
device = await self._device_by_token(session, presented_hash, now)
if device is None:
raise PermissionError("device authorization is invalid or expired")
self._rotate_device(device, next_hash, label, now)
else:
raise ValueError("unsupported authorization action")
await session.commit()
return {
"device_id": device.device_id,
"service_name": "DeerFlow",
"expires_at": (credential_row.expires_at if credential_row else device.expires_at).isoformat(),
"renew_after": (now + timedelta(days=30)).isoformat(),
}
@staticmethod
def _rotate_device(
device: BrowserDeviceRow,
next_hash: str,
label: str,
now: datetime,
*,
preserve_previous: bool = True,
) -> None:
device.previous_token_hash = device.token_hash if preserve_previous else None
device.previous_token_expires_at = (
now + PREVIOUS_TOKEN_GRACE if preserve_previous else None
)
device.token_hash = next_hash
device.expires_at = now + DEVICE_TTL
device.label = label or device.label
device.revoked_at = None
device.updated_at = now
@staticmethod
def _rotate_credential(
credential: BrowserDeviceCredentialRow,
next_hash: str,
label: str,
now: datetime,
*,
preserve_previous: bool = True,
) -> None:
credential.previous_token_hash = credential.token_hash if preserve_previous else None
credential.previous_token_expires_at = now + PREVIOUS_TOKEN_GRACE if preserve_previous else None
credential.token_hash = next_hash
credential.expires_at = now + DEVICE_TTL
credential.label = label or credential.label
credential.revoked_at = None
credential.updated_at = now
async def authenticate_device(self, session: AsyncSession, credential: str) -> BrowserDeviceRow | None:
presented_hash = token_hash(credential)
now = datetime.now(UTC)
credential_row = await self._credential_by_token(session, presented_hash, now)
if credential_row is not None:
device = await session.get(BrowserDeviceRow, credential_row.user_id)
if device is not None and device.revoked_at is None:
return device
return await self._device_by_token(session, presented_hash, now)
async def _credential_by_token(
self,
session: AsyncSession,
presented_hash: str,
now: datetime,
) -> BrowserDeviceCredentialRow | None:
return await session.scalar(
select(BrowserDeviceCredentialRow).where(
BrowserDeviceCredentialRow.revoked_at.is_(None),
BrowserDeviceCredentialRow.expires_at > now,
or_(
BrowserDeviceCredentialRow.token_hash == presented_hash,
(
(BrowserDeviceCredentialRow.previous_token_hash == presented_hash)
& (BrowserDeviceCredentialRow.previous_token_expires_at > now)
),
),
)
)
async def _device_by_token(self, session: AsyncSession, presented_hash: str, now: datetime) -> BrowserDeviceRow | None:
return await session.scalar(
select(BrowserDeviceRow).where(
BrowserDeviceRow.revoked_at.is_(None),
BrowserDeviceRow.expires_at > now,
or_(
BrowserDeviceRow.token_hash == presented_hash,
((BrowserDeviceRow.previous_token_hash == presented_hash) & (BrowserDeviceRow.previous_token_expires_at > now)),
),
)
)
async def status(self, session: AsyncSession, user_id: str) -> dict[str, Any]:
device = await session.get(BrowserDeviceRow, user_id)
now = datetime.now(UTC)
active = bool(device and device.revoked_at is None and _as_utc(device.expires_at) > now)
return {
"enabled": self.enabled,
"paired": active,
# A TCP/WebSocket transport is not usable until the authenticated
# extension handshake has registered its installation identity.
# This prevents the UI from briefly selecting a stale/unknown
# browser while a replacement Chrome/Edge socket is connecting.
"connected": active
and user_id in self.connections
and user_id in self._extension_instance_ids,
"runtime_id": self._extension_runtime_ids.get(user_id),
"instance_id": self._extension_instance_ids.get(user_id),
"busy": self._active_tool_calls.get(user_id, 0) > 0,
"device": None
if not active
else {
"device_id": device.device_id,
"label": device.label,
"last_connected_at": device.last_connected_at,
},
}
async def revoke(self, session: AsyncSession, user_id: str) -> None:
now = datetime.now(UTC)
device = await session.get(BrowserDeviceRow, user_id)
if device:
device.revoked_at = now
device.updated_at = now
credentials = await session.scalars(
select(BrowserDeviceCredentialRow).where(BrowserDeviceCredentialRow.user_id == user_id)
)
for credential in credentials:
credential.revoked_at = now
credential.updated_at = now
pairing = await session.get(BrowserPairingRow, user_id)
if pairing:
await session.delete(pairing)
await session.commit()
socket = self.connections.get(user_id)
if socket:
await socket.close(code=1008, reason="device authorization revoked")
self._clear_task_sessions(user_id)
async def proxy_extension(
self,
websocket: Any,
*,
user_id: str,
device_id: str,
subprotocol: str,
origin: str,
) -> None:
if not self.enabled or not self.daemon:
await websocket.close(code=1013, reason="BrowserSkill is unavailable")
return
try:
port = await self.daemon.ensure_started()
except RuntimeError:
await websocket.close(code=1013, reason="BrowserSkill daemon failed to start")
return
await websocket.accept(subprotocol=subprotocol)
async with self._connections_lock:
previous = self.connections.get(user_id)
if previous is None and len(self.connections) >= self.max_connections:
await websocket.close(code=1013, reason="BrowserSkill connection limit reached")
return
self.connections[user_id] = websocket
# Identity belongs to the current transport generation. Keeping the
# previous browser's IDs here makes a newly connected Edge appear as
# the Chrome socket it replaced until its handshake is processed.
self._extension_runtime_ids.pop(user_id, None)
self._extension_instance_ids.pop(user_id, None)
self._browser_write_locks.setdefault(user_id, asyncio.Lock())
# A replacement socket is a transport refresh, not the end of the
# browser task. Clearing here made every extension reconnect detach
# the task from its chat thread, so the preview endpoint immediately
# started returning 409 even though the browser session was alive.
await self._close_replaced_connection(previous, websocket)
async with self.session_factory() as session:
device = await session.get(BrowserDeviceRow, user_id)
if device:
device.last_connected_at = datetime.now(UTC)
device.updated_at = datetime.now(UTC)
await session.commit()
try:
async with connect(f"ws://127.0.0.1:{port}", origin=origin) as daemon_socket:
await self._bridge(websocket, daemon_socket, user_id, device_id)
finally:
await self._detach_extension_socket(user_id, websocket)
@staticmethod
async def _close_replaced_connection(previous: Any | None, current: Any) -> None:
if previous and previous is not current:
await previous.close(
code=4001,
reason="replaced_by_another_browser",
)
async def _detach_extension_socket(self, user_id: str, websocket: Any) -> None:
"""Detach a socket without letting an obsolete connection erase new state."""
disconnected_current_socket = False
async with self._connections_lock:
if self.connections.get(user_id) is websocket:
self.connections.pop(user_id, None)
disconnected_current_socket = True
# The superseded socket finishes after the replacement is already
# registered. It must not erase sessions created or retained by that
# newer connection.
if disconnected_current_socket:
self._fail_ui_calls(user_id, "Browser extension disconnected")
self._clear_task_sessions(user_id)
async def _bridge(self, browser: Any, daemon_socket: Any, user_id: str, device_id: str) -> None:
first_frame = True
async def browser_to_daemon() -> None:
nonlocal first_frame
while True:
message = await browser.receive()
if message["type"] == "websocket.disconnect":
return
data: str | bytes = message.get("text") or message.get("bytes") or ""
# ui.task_* is a host-to-extension side channel. Its replies
# belong to DeerFlow and must never reach the native daemon.
if isinstance(data, str) and self._consume_ui_reply(user_id, data):
continue
if first_frame and isinstance(data, str):
first_frame = False
try:
payload = json.loads(data)
if payload.get("method") == "system.handshake":
params = payload.setdefault("params", {})
registered = await self._register_extension_identity(
user_id,
params,
device_id,
websocket=browser,
)
if not registered:
return
data = json.dumps(payload)
except (TypeError, ValueError):
pass
await daemon_socket.send(data)
async def daemon_to_browser() -> None:
async for data in daemon_socket:
async with self._browser_write_locks[user_id]:
if isinstance(data, bytes):
await browser.send_bytes(data)
else:
await browser.send_text(data)
tasks = {
asyncio.create_task(browser_to_daemon()),
asyncio.create_task(daemon_to_browser()),
}
done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
for task in pending:
task.cancel()
await asyncio.gather(*done, *pending, return_exceptions=True)
def _consume_ui_reply(self, user_id: str, data: str) -> bool:
"""Resolve a direct extension UI call and keep it away from the daemon."""
try:
payload = json.loads(data)
except (TypeError, ValueError):
return False
if not isinstance(payload, dict):
return False
call_id = payload.get("id")
pending = self._ui_calls.get(call_id) if isinstance(call_id, str) else None
if pending is None or pending[0] != user_id:
return False
self._ui_calls.pop(call_id, None)
future = pending[1]
if future.done():
return True
error = payload.get("error")
if error:
if isinstance(error, dict):
code = str(error.get("code") or "extension_error")
message = str(error.get("message") or "Browser extension UI request failed")
future.set_exception(RuntimeError(f"{code}: {message}"))
else:
future.set_exception(RuntimeError(str(error)))
else:
future.set_result(payload.get("result"))
return True
def _fail_ui_calls(self, user_id: str, message: str) -> None:
for call_id, (owner, future) in list(self._ui_calls.items()):
if owner != user_id:
continue
self._ui_calls.pop(call_id, None)
if not future.done():
future.set_exception(RuntimeError(message))
async def _call_extension_ui(self, *, user_id: str, method: str, session_id: str) -> Any:
"""Call the pinned BrowserSkill extension's UI side channel directly."""
websocket = self.connections.get(user_id)
if websocket is None:
raise RuntimeError("Browser extension is offline")
if sum(owner == user_id for owner, _ in self._ui_calls.values()) >= 8:
raise RuntimeError("Browser preview is busy")
call_id = f"df-ui-{uuid.uuid4().hex}"
future = asyncio.get_running_loop().create_future()
self._ui_calls[call_id] = (user_id, future)
try:
async with self._browser_write_locks.setdefault(user_id, asyncio.Lock()):
await websocket.send_json(
{
"id": call_id,
"method": method,
"params": {"session_id": session_id},
}
)
return await asyncio.wait_for(future, timeout=5)
except TimeoutError as exc:
raise RuntimeError("Browser preview timed out; check the pinned BrowserSkill extension") from exc
finally:
self._ui_calls.pop(call_id, None)
def _clear_task_sessions(self, user_id: str) -> None:
for key in [key for key in self.task_sessions if key[0] == user_id]:
self.task_sessions.pop(key, None)
async def _register_extension_runtime(self, user_id: str, runtime_id: str) -> None:
"""Invalidate sessions when the extension background runtime is recreated."""
previous = self._extension_runtime_ids.get(user_id)
self._extension_runtime_ids[user_id] = runtime_id
if previous is None or previous == runtime_id:
return
self._fail_ui_calls(user_id, "Browser extension reloaded; browser task restarted")
stale_sessions = {
session_id
for (owner, _thread_id), session_id in self.task_sessions.items()
if owner == user_id
}
for session_id in stale_sessions:
await self._stop_stale_task_session(session_id)
self._clear_task_sessions(user_id)
async def _register_extension_identity(
self,
user_id: str,
params: dict[str, Any],
fallback_device_id: str,
*,
websocket: Any | None = None,
) -> bool:
"""Keep each browser installation distinct from the account device."""
# A replaced socket may still yield a buffered handshake while it is
# shutting down. Never let that obsolete Chrome/Edge connection
# overwrite the identity of the socket that replaced it.
async with self._connections_lock:
if websocket is not None and self.connections.get(user_id) is not websocket:
return False
runtime_id = params.pop("runtime_id", None)
if isinstance(runtime_id, str) and runtime_id:
await self._register_extension_runtime(user_id, runtime_id)
# ``runtime_id`` belongs to DeerFlow's transport lifecycle and is not
# part of the BrowserSkill daemon handshake. ``instance_id`` is part of
# that handshake, however: the daemon uses it to bind sessions to the
# exact Chrome/Edge installation. Keep it in ``params`` while also
# recording it for the account status and session.start calls.
instance_id = params.get("instance_id")
if not isinstance(instance_id, str) or not instance_id:
instance_id = fallback_device_id
self._extension_instance_ids[user_id] = instance_id
# BrowserSkill controls Chromium through the Chrome extension API. Its
# protocol currently validates this field as ``chrome`` even when the
# extension is hosted by Microsoft Edge. Keep the real browser family
# in DeerFlow's device/instance metadata, but normalize the downstream
# daemon handshake so Edge is not disconnected after its first frame.
browser = params.get("browser")
if isinstance(browser, dict) and browser.get("name") == "edge":
browser["name"] = "chrome"
return True
@staticmethod
def _is_missing_session_error(error: RuntimeError) -> bool:
message = str(error).lower()
return "session" in message and any(
marker
in message
for marker in (
"unknown",
"not found",
"not_found",
"not registered",
"already stopped",
)
)
@staticmethod
def _is_missing_controlled_tab_error(error: RuntimeError) -> bool:
"""Return true when the user closed the tab owned by a live session."""
message = " ".join(str(error).lower().replace("_", " ").split())
return "session" in message and any(
marker
in message
for marker in (
"has no controlled tab",
"controlled tab not found",
"controlled tab was closed",
)
)
@staticmethod
def _is_orphaned_agent_window_error(error: RuntimeError) -> bool:
"""Return true when a live session has lost its owned Chrome window."""
message = " ".join(str(error).lower().replace("_", " ").split())
return "window" in message and any(
marker in message
for marker in (
"no window with id",
"window not found",
"window does not exist",
"window was closed",
)
)
async def _stop_stale_task_session(self, session_id: str) -> None:
"""Best-effort cleanup before replacing a session with a closed window."""
if not self.daemon:
return
try:
await self.daemon.rpc("session.stop", {"session_id": session_id})
except RuntimeError:
# The daemon may already have discarded the orphaned session. A
# failed cleanup must not prevent creating its replacement.
pass
async def _start_task_session(self, user_id: str) -> str:
if not self.daemon:
raise RuntimeError("BrowserSkill is not configured")
async with self.session_factory() as session:
device = await session.get(BrowserDeviceRow, user_id)
now = datetime.now(UTC)
if device is None or device.revoked_at is not None or _as_utc(device.expires_at) <= now:
raise RuntimeError("Pair a browser before using BrowserSkill")
device_id = device.device_id
browser_instance_id = self._extension_instance_ids.get(user_id) or device_id
started = await self.daemon.rpc(
"session.start",
{"focused": False, "browser_instance_id": browser_instance_id},
)
session_id = started.get("session_id") if isinstance(started, dict) else None
if not session_id:
raise RuntimeError("invalid BrowserSkill session response")
return session_id
async def call_tool(
self,
*,
user_id: str,
thread_id: str,
method: str,
params: dict[str, Any],
) -> Any:
self._active_tool_calls[user_id] = self._active_tool_calls.get(user_id, 0) + 1
try:
return await self._call_tool_impl(
user_id=user_id,
thread_id=thread_id,
method=method,
params=params,
)
finally:
remaining = self._active_tool_calls.get(user_id, 1) - 1
if remaining > 0:
self._active_tool_calls[user_id] = remaining
else:
self._active_tool_calls.pop(user_id, None)
async def _call_tool_impl(
self,
*,
user_id: str,
thread_id: str,
method: str,
params: dict[str, Any],
) -> Any:
if user_id not in self.connections:
raise RuntimeError("Browser extension is offline")
if not self.daemon:
raise RuntimeError("BrowserSkill is not configured")
method = method.removeprefix("tool.")
if method not in ALLOWED_TOOL_METHODS:
raise ValueError(f"unsupported browser method: {method}")
key = (user_id, thread_id)
for attempt in range(2):
session_id = self.task_sessions.get(key)
if not session_id:
session_id = await self._start_task_session(user_id)
self.task_sessions[key] = session_id
call_params = {**params, "session_id": session_id}
# WeKnora exposes a character budget to the model while the bsk
# protocol expects a token budget for DOM observations.
if method in {"observe", "snapshot"} and "max_text_chars" in call_params:
max_text_chars = int(call_params.pop("max_text_chars"))
call_params["max_tokens"] = max(1, (max_text_chars + 3) // 4)
try:
return await self.daemon.rpc(f"tool.{method}", call_params)
except RuntimeError as exc:
missing_session = self._is_missing_session_error(exc)
orphaned_window = self._is_orphaned_agent_window_error(exc)
missing_controlled_tab = self._is_missing_controlled_tab_error(exc)
if attempt == 0 and (
missing_session or orphaned_window or missing_controlled_tab
):
self.task_sessions.pop(key, None)
if orphaned_window or missing_controlled_tab:
await self._stop_stale_task_session(session_id)
continue
raise
raise RuntimeError("BrowserSkill session recovery failed")
def task_status(self, *, user_id: str, thread_id: str) -> dict[str, Any]:
session_id = self.task_sessions.get((user_id, thread_id))
return {
"enabled": self.enabled,
"connected": user_id in self.connections
and user_id in self._extension_instance_ids,
"selected": bool(session_id),
"task_id": session_id or "",
}
async def task_action(self, *, user_id: str, thread_id: str, action: str) -> dict[str, Any]:
key = (user_id, thread_id)
session_id = self.task_sessions.get(key)
if not session_id:
raise RuntimeError("No active browser task for this conversation")
if user_id not in self.connections:
raise RuntimeError("Browser extension is offline")
if action == "preview":
return await self._call_extension_ui(
user_id=user_id,
method="ui.task_preview",
session_id=session_id,
)
if action == "focus":
return await self._call_extension_ui(
user_id=user_id,
method="ui.task_focus",
session_id=session_id,
)
if action == "stop":
if not self.daemon:
raise RuntimeError("BrowserSkill is not configured")
result = await self.daemon.rpc("session.stop", {"session_id": session_id})
self.task_sessions.pop(key, None)
return result if isinstance(result, dict) else {"stopped": True}
raise ValueError(f"unsupported browser task action: {action}")
async def close(self) -> None:
sockets = list(self.connections.values())
for user_id in list(self.connections):
self._fail_ui_calls(user_id, "BrowserSkill manager is shutting down")
self.connections.clear()
self.task_sessions.clear()
self._browser_write_locks.clear()
self._extension_runtime_ids.clear()
self._extension_instance_ids.clear()
for socket in sockets:
await socket.close(code=1001, reason="server shutting down")
if self.daemon:
await self.daemon.close()
def public_websocket_url(origin: str) -> str:
parsed = urlsplit(origin.strip())
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("origin must be an absolute http(s) URL")
scheme = "wss" if parsed.scheme == "https" else "ws"
base_path = parsed.path.rstrip("/")
path = f"{base_path}/api/v1/local-browser/extension"
return urlunsplit((scheme, parsed.netloc, path, "", ""))
@@ -0,0 +1,17 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .manager import BrowserSkillManager
_manager: "BrowserSkillManager | None" = None
def set_browser_skill_manager(manager: "BrowserSkillManager | None") -> None:
global _manager
_manager = manager
def get_browser_skill_manager() -> "BrowserSkillManager":
if _manager is None:
raise RuntimeError("BrowserSkill is disabled or not initialized")
return _manager
@@ -0,0 +1,182 @@
"""Typed BrowserSkill contract aligned with WeKnora v0.8.3."""
from __future__ import annotations
from typing import Any, Literal
from langchain.tools import ToolRuntime
from langgraph.typing import ContextT
from pydantic import BaseModel, ConfigDict, Field, model_validator
from deerflow.agents.thread_state import ThreadState
BrowserSkillMethod = Literal[
"observe",
"snapshot",
"screenshot",
"navigate",
"navigate_back",
"navigate_forward",
"reload",
"click",
"fill",
"press",
"hover",
"wheel",
"scroll_to",
"focus",
"blur",
"select",
"tab_list",
"tab_create",
"tab_select",
"tab_close",
"tab_borrow",
"tab_return",
"get_html",
"evaluate",
"console",
"network",
"wait_for_navigation",
"wait_ms",
"window_resize",
"emulate",
"request_help",
]
class BrowserSkillParams(BaseModel):
"""Superset of operation parameters; method-specific rules are validated below."""
model_config = ConfigDict(extra="forbid")
debug_surfaces: bool | None = None
max_depth: int | None = Field(default=None, ge=0)
probe_hover: bool | None = None
tab_id: int | None = Field(default=None, ge=1)
timeout_ms: int | None = Field(default=None, ge=1)
url: str | None = Field(default=None, min_length=1)
wait_until: Literal["load", "domcontentloaded", "networkidle", "commit"] | None = None
hard: bool | None = None
button: Literal["left", "middle", "right"] | None = None
click_count: int | None = Field(default=None, ge=1)
modifiers: list[Literal["alt", "ctrl", "meta", "shift"]] | None = None
ref: str | None = Field(default=None, min_length=1)
selector: str | None = Field(default=None, min_length=1)
clear_before: bool | None = None
value: str | None = Field(default=None, description="Required for fill; empty clears the field.")
hold_ms: int | None = Field(default=None, ge=0)
key: str | None = Field(default=None, min_length=1, description="Required for press, e.g. Enter or Ctrl+A.")
settle_ms: int | None = Field(default=None, ge=0)
delta_x: float | None = None
delta_y: float | None = None
values: list[str] | None = None
scope: Literal["user", "agent", "all"] | None = None
active: bool | None = None
index: int | None = Field(default=None, ge=0)
confirm: bool | None = None
max_bytes: int | None = Field(default=None, ge=0)
await_promise: bool | None = None
expression: str | None = Field(
default=None,
min_length=1,
description="Required for evaluate. Use an expression or an IIFE; return bounded JSON data.",
)
return_by_value: bool | None = None
include_stack: bool | None = None
limit: int | None = Field(default=None, ge=1)
max_text_chars: int | None = Field(default=None, ge=1)
since: int | None = Field(default=None, ge=0)
duration_ms: int | None = Field(default=None, ge=0, le=10_000)
height: int | None = Field(default=None, ge=100, le=7680)
width: int | None = Field(default=None, ge=100, le=7680)
off: bool | None = None
overrides: dict[str, Any] | None = None
prompt: str | None = Field(default=None, min_length=1)
targets: list[dict[str, str]] | None = None
title: str | None = None
_REQUIRED_FIELDS: dict[str, tuple[str, ...]] = {
"navigate": ("url",),
"fill": ("value",),
"press": ("key",),
"select": ("values",),
"tab_select": ("tab_id",),
"tab_close": ("tab_id",),
"tab_borrow": ("tab_id",),
"tab_return": ("tab_id",),
"evaluate": ("expression",),
"wait_ms": ("duration_ms",),
"window_resize": ("height", "width"),
"request_help": ("prompt",),
}
_ALLOWED_FIELDS: dict[str, frozenset[str]] = {
"observe": frozenset({"tab_id", "max_depth", "max_text_chars", "probe_hover", "debug_surfaces"}),
"snapshot": frozenset({"tab_id", "max_depth", "max_text_chars", "probe_hover", "debug_surfaces"}),
"screenshot": frozenset({"tab_id"}),
"navigate": frozenset({"url", "tab_id", "wait_until", "timeout_ms"}),
"navigate_back": frozenset({"tab_id", "wait_until", "timeout_ms"}),
"navigate_forward": frozenset({"tab_id", "wait_until", "timeout_ms"}),
"reload": frozenset({"tab_id", "hard", "wait_until", "timeout_ms"}),
"click": frozenset({"tab_id", "ref", "selector", "button", "click_count", "modifiers", "timeout_ms"}),
"fill": frozenset({"tab_id", "ref", "selector", "value", "clear_before", "timeout_ms"}),
"press": frozenset({"tab_id", "ref", "selector", "key", "hold_ms", "timeout_ms"}),
"hover": frozenset({"tab_id", "ref", "selector", "settle_ms", "timeout_ms"}),
"wheel": frozenset({"tab_id", "delta_x", "delta_y", "settle_ms"}),
"scroll_to": frozenset({"tab_id", "ref", "selector", "settle_ms", "timeout_ms"}),
"focus": frozenset({"tab_id", "ref", "selector", "timeout_ms"}),
"blur": frozenset({"tab_id", "ref", "selector", "timeout_ms"}),
"select": frozenset({"tab_id", "ref", "selector", "values", "timeout_ms"}),
"tab_list": frozenset({"scope", "active"}),
"tab_create": frozenset({"url", "active"}),
"tab_select": frozenset({"tab_id"}),
"tab_close": frozenset({"tab_id"}),
"tab_borrow": frozenset({"tab_id", "confirm"}),
"tab_return": frozenset({"tab_id"}),
"get_html": frozenset({"tab_id", "ref", "selector", "max_bytes"}),
"evaluate": frozenset({"tab_id", "expression", "await_promise", "return_by_value", "timeout_ms"}),
"console": frozenset({"tab_id", "limit", "since", "include_stack"}),
"network": frozenset({"tab_id", "limit", "since"}),
"wait_for_navigation": frozenset({"tab_id", "wait_until", "timeout_ms"}),
"wait_ms": frozenset({"duration_ms"}),
"window_resize": frozenset({"height", "width"}),
"emulate": frozenset({"tab_id", "off", "overrides"}),
"request_help": frozenset({"prompt", "targets", "title"}),
}
class BrowserSkillInput(BaseModel):
"""Model-visible BrowserSkill input with an explicit operation enum."""
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
# An explicit args_schema must include injected arguments. LangChain keeps
# this field out of the model-visible JSON schema and fills it in ToolNode.
runtime: ToolRuntime[ContextT, ThreadState]
method: BrowserSkillMethod = Field(description=("Browser operation. Use fill (not type), evaluate (not eval), wheel or scroll_to (not scroll), wait_ms or wait_for_navigation (not wait), and observe/snapshot/get_html for extraction."))
params: BrowserSkillParams = Field(default_factory=BrowserSkillParams)
@model_validator(mode="after")
def validate_method_requirements(self) -> BrowserSkillInput:
for name in _REQUIRED_FIELDS.get(self.method, ()):
if getattr(self.params, name) is None:
raise ValueError(f"{self.method} requires params.{name}")
unexpected = self.params.model_fields_set - _ALLOWED_FIELDS[self.method]
if unexpected:
names = ", ".join(sorted(unexpected))
raise ValueError(f"{self.method} does not accept params: {names}")
if self.method == "press" and self.params.key:
normalized_key = self.params.key.lower().replace("command", "meta").replace("control", "ctrl").replace(" ", "")
if normalized_key in {"ctrl+t", "meta+t", "ctrl+n", "meta+n", "ctrl+shift+n", "meta+shift+n"}:
raise ValueError("press cannot create browser tabs or windows; use tab_create so the new tab remains in agent scope")
return self
def params_dict(value: BrowserSkillParams | dict[str, Any] | None) -> dict[str, Any]:
if value is None:
return {}
if isinstance(value, BrowserSkillParams):
return value.model_dump(exclude_none=True)
return {key: item for key, item in value.items() if item is not None}
@@ -0,0 +1,17 @@
import hashlib
import re
import secrets
EXTENSION_ORIGIN = re.compile(r"^(chrome|edge)-extension://[a-p]{32}$")
def new_token() -> str:
return secrets.token_urlsafe(32)
def token_hash(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()
def is_extension_origin(origin: str | None) -> bool:
return bool(origin and EXTENSION_ORIGIN.fullmatch(origin))
@@ -0,0 +1,137 @@
"""Request-scoped BrowserSkill tool exposed only when the chat enables it."""
from __future__ import annotations
import asyncio
import base64
import hashlib
import json
from collections.abc import Mapping
from typing import Any
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
from deerflow.runtime.user_context import get_effective_user_id
from .runtime import get_browser_skill_manager
from .schema import BrowserSkillInput, BrowserSkillMethod, BrowserSkillParams, params_dict
_MAX_SCREENSHOT_BASE64 = 8 * 1024 * 1024
_MAX_SCREENSHOT_BYTES = 5 * 1024 * 1024
_MAX_SCREENSHOTS_PER_THREAD = 20
def _decode_screenshot(result: Any) -> tuple[bytes, str, dict[str, Any]]:
"""Validate a screenshot and split its bytes from context-safe metadata."""
if not isinstance(result, dict):
raise ValueError("Browser screenshot returned an invalid response")
image = result.get("image_base64")
image_format = str(result.get("format") or "").lower()
if image_format == "jpg":
image_format = "jpeg"
if not isinstance(image, str) or not image or len(image) > _MAX_SCREENSHOT_BASE64 or image_format not in {"png", "jpeg"}:
raise ValueError("Browser screenshot is invalid or too large")
try:
raw = base64.b64decode(image, validate=True)
except ValueError as exc:
raise ValueError("Browser screenshot contains invalid base64 data") from exc
if len(raw) > _MAX_SCREENSHOT_BYTES:
raise ValueError("Browser screenshot is invalid or too large")
metadata = {key: value for key, value in result.items() if key != "image_base64"}
return raw, image_format, metadata
def _write_screenshot_artifact(*, thread_id: str, user_id: str, raw: bytes, image_format: str, metadata: dict[str, Any]) -> str:
"""Persist screenshot bytes in the thread outputs and return lightweight metadata."""
from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths
digest = hashlib.sha256(raw).hexdigest()
extension = "jpg" if image_format == "jpeg" else image_format
relative_path = f"browser-screenshots/{digest}.{extension}"
output_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=user_id)
artifact_path = output_dir / relative_path
artifact_path.parent.mkdir(parents=True, exist_ok=True)
artifact_path.write_bytes(raw)
screenshots = sorted(
(path for path in artifact_path.parent.iterdir() if path.is_file()),
key=lambda path: path.stat().st_mtime,
reverse=True,
)
for stale_path in screenshots[_MAX_SCREENSHOTS_PER_THREAD:]:
stale_path.unlink(missing_ok=True)
metadata.update(
{
"captured": True,
"artifact_id": f"browser-screenshot:{digest}",
"path": f"{VIRTUAL_PATH_PREFIX}/outputs/{relative_path}",
"mime_type": f"image/{image_format}",
"format": image_format,
"byte_size": len(raw),
"sha256": digest,
"display_to_user": True,
}
)
return json.dumps(metadata, ensure_ascii=False, default=str)
async def _screenshot_artifact(result: Any, *, thread_id: str, user_id: str) -> str:
raw, image_format, metadata = _decode_screenshot(result)
return await asyncio.to_thread(
_write_screenshot_artifact,
thread_id=thread_id,
user_id=user_id,
raw=raw,
image_format=image_format,
metadata=metadata,
)
@tool("browser_skill", args_schema=BrowserSkillInput, parse_docstring=True)
async def browser_skill_tool(
runtime: ToolRuntime[ContextT, ThreadState],
method: BrowserSkillMethod,
params: BrowserSkillParams | dict[str, Any] | None = None,
) -> str:
"""Operate the user's paired browser through the independent BrowserSkill bridge.
Args:
method: Browser operation. Prefer observe then a returned ref for click/fill/focus.
Do not use evaluate merely to synthesize a CSS selector. Use screenshot only
when the user explicitly requests a capture; image bytes are intentionally
omitted from model context, so use observe to inspect page content.
params: Typed parameters required by the operation. When observe provides a ref,
pass that ref instead of constructing a positional nth-child selector.
"""
configurable = runtime.config.get("configurable", {}) if isinstance(runtime.config, Mapping) else {}
context = runtime.context if isinstance(runtime.context, Mapping) else {}
# LangGraph >= 1.1 keeps run context on ToolRuntime.context. Reading only
# configurable silently grouped every browser session under "default", so
# the per-thread preview endpoint could never discover the active task.
thread_id = str(context.get("thread_id") or configurable.get("thread_id") or "default")
try:
user_id = get_effective_user_id()
result = await get_browser_skill_manager().call_tool(
user_id=user_id,
thread_id=thread_id,
method=method,
params=params_dict(params),
)
if method == "screenshot":
return await _screenshot_artifact(result, thread_id=thread_id, user_id=user_id)
return json.dumps(result, ensure_ascii=False, default=str)
except (RuntimeError, ValueError) as exc:
return json.dumps(
{
"ok": False,
"error": {
"code": "browser_skill_error",
"message": str(exc) or exc.__class__.__name__,
"recovery": ("Do not retry repeatedly. Reconnect the browser extension or start a new browser session if the error persists."),
},
},
ensure_ascii=False,
)
@@ -0,0 +1,3 @@
from .model import BrowserDeviceCredentialRow, BrowserDeviceRow, BrowserPairingRow
__all__ = ["BrowserDeviceCredentialRow", "BrowserDeviceRow", "BrowserPairingRow"]
@@ -0,0 +1,54 @@
from datetime import datetime
from sqlalchemy import DateTime, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
class BrowserDeviceRow(Base):
__tablename__ = "browser_devices"
user_id: Mapped[str] = mapped_column(String(128), primary_key=True)
device_id: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
label: Mapped[str] = mapped_column(String(255), nullable=False, default="BrowserSkill Extension")
token_hash: Mapped[str] = mapped_column(String(64), nullable=False)
previous_token_hash: Mapped[str | None] = mapped_column(String(64))
previous_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
last_connected_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class BrowserDeviceCredentialRow(Base):
"""A renewable credential for one browser installation.
``BrowserDeviceRow`` remains the account-level device record for backwards
compatibility. Credentials live separately so Chrome and Edge can stay
paired to the same account without rotating each other's token.
"""
__tablename__ = "browser_device_credentials"
credential_id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(128), index=True, nullable=False)
device_id: Mapped[str] = mapped_column(String(64), index=True, nullable=False)
label: Mapped[str] = mapped_column(String(255), nullable=False)
token_hash: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
previous_token_hash: Mapped[str | None] = mapped_column(String(64), index=True)
previous_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class BrowserPairingRow(Base):
__tablename__ = "browser_pairings"
user_id: Mapped[str] = mapped_column(String(128), primary_key=True)
token_hash: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
@@ -35,7 +35,6 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.analysis_project_memory.model import (
AnalysisProjectMemoryItemRow,
@@ -43,6 +42,7 @@ from deerflow.persistence.analysis_project_memory.model import (
AnalysisProjectMemoryStateRow,
)
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
from deerflow.persistence.article_types.model import ArticleTypeRow
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
@@ -60,6 +60,11 @@ from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.browser_skill.model import (
BrowserDeviceCredentialRow,
BrowserDeviceRow,
BrowserPairingRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow, UserCompactMenuOverrideConfigRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -78,6 +83,11 @@ from deerflow.persistence.force_situation.model import (
ForceSituationDetailVersionRow,
)
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.goview.models import (
GoviewProjectContentRow,
GoviewProjectRow,
GoviewTemplateRow,
)
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.knowledge_research import KnowledgeResearchSessionRow
@@ -159,6 +169,7 @@ from deerflow.persistence.skills.model import (
SkillRow,
)
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
@@ -180,16 +191,10 @@ from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
from deerflow.persistence.goview.models import (
GoviewProjectContentRow,
GoviewProjectRow,
GoviewTemplateRow,
)
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_planning.model import (
@@ -226,6 +231,9 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"BrowserDeviceCredentialRow",
"BrowserDeviceRow",
"BrowserPairingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeVectorRow",
@@ -49,6 +49,32 @@ def _is_browser_context_tool(tool: object) -> bool:
return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
def _filter_legacy_browser_tools(
tool_configs: list[object],
*,
browser_skill_requested: bool,
browser_context_config: object | None,
) -> list[object]:
"""Resolve the two browser transports into one unambiguous tool surface.
BrowserSkill and the legacy Deer Web Clipper tools use different transports.
Exposing both makes the model prefer the more prescriptive legacy
``browser_act`` schema even when the user explicitly selected BrowserSkill.
"""
if browser_skill_requested:
return [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
if not getattr(browser_context_config, "enabled", False):
return [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
if not getattr(browser_context_config, "actions_enabled", True):
return [
tool
for tool in tool_configs
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE
and getattr(tool, "name", None) != "browser_act"
]
return tool_configs
def _enforce_agent_orchestration_scope(
tools: list[BaseTool],
*,
@@ -72,6 +98,7 @@ async def get_available_tools_async(
allow_agent_orchestration: bool = False,
app_config: AppConfig | None = None,
write_file_compatibility: bool | None = None,
browser_skill_requested: bool = False,
) -> list[BaseTool]:
"""Load tools without blocking the caller loop or its MCP discovery.
@@ -92,6 +119,7 @@ async def get_available_tools_async(
allow_agent_orchestration=allow_agent_orchestration,
app_config=app_config,
write_file_compatibility=write_file_compatibility,
browser_skill_requested=browser_skill_requested,
)
@@ -107,6 +135,7 @@ def get_available_tools(
allow_agent_orchestration: bool = False,
app_config: AppConfig | None = None,
write_file_compatibility: bool | None = None,
browser_skill_requested: bool = False,
) -> list[BaseTool]:
"""Get all available tools from config.
@@ -167,16 +196,11 @@ def get_available_tools(
]
browser_context_config = getattr(config, "browser_context", None)
if not getattr(browser_context_config, "enabled", False):
tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
elif not getattr(browser_context_config, "actions_enabled", True):
# browser_context is on, but interactive page actions are disabled — keep
# the read-only browser_fetch_page tool, drop only browser_act.
tool_configs = [
tool
for tool in tool_configs
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
]
tool_configs = _filter_legacy_browser_tools(
tool_configs,
browser_skill_requested=browser_skill_requested,
browser_context_config=browser_context_config,
)
# Do not expose host bash by default when LocalSandboxProvider is active.
if not is_host_bash_allowed(config):
@@ -201,6 +225,10 @@ def get_available_tools(
# Conditionally add tools based on config
builtin_tools = BUILTIN_TOOLS.copy()
if browser_skill_requested and getattr(config.browser_skill, "enabled", False):
from deerflow.integrations.browser_skill.tool import browser_skill_tool
builtin_tools.append(browser_skill_tool)
if allow_agent_orchestration:
builtin_tools.append(agent_orchestration_tool)
if not is_scheduled_run:
@@ -0,0 +1,161 @@
import asyncio
from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.integrations.browser_skill.manager import BrowserSkillManager
from deerflow.integrations.browser_skill.security import token_hash
from deerflow.persistence.base import Base
from deerflow.persistence.browser_skill import BrowserDeviceCredentialRow, BrowserPairingRow
@pytest.mark.asyncio
async def test_replaced_socket_cannot_overwrite_current_browser_identity() -> None:
manager = BrowserSkillManager.__new__(BrowserSkillManager)
current_socket = object()
replaced_socket = object()
manager.connections = {"user-1": current_socket}
manager._connections_lock = asyncio.Lock()
manager._extension_runtime_ids = {}
manager._extension_instance_ids = {}
manager._ui_calls = {}
manager.task_sessions = {}
stale_params = {
"runtime_id": "chrome-runtime",
"instance_id": "chrome-instance",
}
assert not await manager._register_extension_identity(
"user-1",
stale_params,
"device-1",
websocket=replaced_socket,
)
assert manager._extension_runtime_ids == {}
assert manager._extension_instance_ids == {}
edge_params = {
"runtime_id": "edge-runtime",
"instance_id": "edge-instance",
"browser": {"name": "edge", "version": "157.0.0.0"},
}
assert await manager._register_extension_identity(
"user-1",
edge_params,
"device-1",
websocket=current_socket,
)
assert manager._extension_runtime_ids == {"user-1": "edge-runtime"}
assert manager._extension_instance_ids == {"user-1": "edge-instance"}
assert edge_params["browser"]["name"] == "chrome"
@pytest.mark.asyncio
async def test_chrome_and_edge_pairing_tokens_do_not_revoke_each_other(tmp_path) -> None:
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'browser-skill.db'}")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
sessions = async_sessionmaker(engine, expire_on_commit=False)
manager = BrowserSkillManager.__new__(BrowserSkillManager)
manager.configured_enabled = True
manager.daemon = object()
manager.binary = __file__
async def pair(pair_token: str, next_token: str, label: str) -> None:
now = datetime.now(UTC)
async with sessions() as session:
existing = await session.get(BrowserPairingRow, "user-1")
if existing is None:
session.add(
BrowserPairingRow(
user_id="user-1",
token_hash=token_hash(pair_token),
expires_at=now + timedelta(minutes=5),
created_at=now,
)
)
else:
existing.token_hash = token_hash(pair_token)
existing.expires_at = now + timedelta(minutes=5)
await session.commit()
await manager.authorize(
session,
credential=pair_token,
action="pair",
next_token=next_token,
label=label,
)
await pair("pair-edge", "token-edge", "Edge · CMZS")
await pair("pair-chrome", "token-chrome", "Chrome · CMZS")
async with sessions() as session:
edge = await manager.authenticate_device(session, "token-edge")
chrome = await manager.authenticate_device(session, "token-chrome")
credentials = list(
await session.scalars(
select(BrowserDeviceCredentialRow).where(
BrowserDeviceCredentialRow.user_id == "user-1"
)
)
)
assert edge is not None
assert chrome is not None
assert edge.user_id == chrome.user_id == "user-1"
assert {credential.label for credential in credentials} == {
"Chrome · CMZS",
"Edge · CMZS",
}
await engine.dispose()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"recoverable_error",
[
"protocol_error: session has no controlled tab",
"session not registered or already stopped",
],
)
async def test_closed_controlled_tab_restarts_session_once(
recoverable_error: str,
) -> None:
class FakeDaemon:
def __init__(self) -> None:
self.calls: list[tuple[str, dict[str, object]]] = []
async def rpc(self, method: str, params: dict[str, object]) -> dict[str, object]:
self.calls.append((method, params))
if method == "tool.navigate" and params["session_id"] == "stale-session":
raise RuntimeError(recoverable_error)
return {"ok": True, "session_id": params.get("session_id")}
manager = BrowserSkillManager.__new__(BrowserSkillManager)
manager.connections = {"user-1": object()}
manager.daemon = FakeDaemon()
manager.task_sessions = {("user-1", "thread-1"): "stale-session"}
manager._active_tool_calls = {}
async def start_task_session(_user_id: str) -> str:
return "replacement-session"
manager._start_task_session = start_task_session
result = await manager.call_tool(
user_id="user-1",
thread_id="thread-1",
method="navigate",
params={"url": "https://www.baidu.com/"},
)
assert result == {"ok": True, "session_id": "replacement-session"}
assert manager.task_sessions[("user-1", "thread-1")] == "replacement-session"
navigate_calls = [call for call in manager.daemon.calls if call[0] == "tool.navigate"]
assert [call[1]["session_id"] for call in navigate_calls] == [
"stale-session",
"replacement-session",
]
+10
View File
@@ -8,6 +8,16 @@ RUNTIME_DIR="${ROOT_DIR}/.runtime"
LOG_DIR="${RUNTIME_DIR}/logs"
PID_DIR="${RUNTIME_DIR}/pids"
# Load developer-only backend overrides. This file is gitignored so local
# binary paths and other machine-specific values never enter source control.
BACKEND_LOCAL_ENV_FILE="${BACKEND_DIR}/.env.local"
if [[ -f "${BACKEND_LOCAL_ENV_FILE}" ]]; then
set -a
# shellcheck disable=SC1090
source "${BACKEND_LOCAL_ENV_FILE}"
set +a
fi
BACKEND_HOST="${BACKEND_HOST:-0.0.0.0}"
BACKEND_PORT="${BACKEND_PORT:-8001}"
FRONTEND_HOST="${FRONTEND_HOST:-0.0.0.0}"