-
问答前缀配置
+
问答前缀配置
管理员可在此配置问答页输入框中的「前缀开关」。开启后,用户在新会话中第一次提问时,
会将下方配置的前缀文本和原问题拼接后再发送给模型。
diff --git a/frontend-web/src/pages/PublicLeaderboardPage.tsx b/frontend-web/src/pages/PublicLeaderboardPage.tsx
new file mode 100644
index 00000000..df783113
--- /dev/null
+++ b/frontend-web/src/pages/PublicLeaderboardPage.tsx
@@ -0,0 +1,947 @@
+"use client";
+
+import {
+ Activity,
+ Award,
+ BarChart3,
+ Bot,
+ CalendarDays,
+ Crown,
+ Gauge,
+ Layers3,
+ Loader2,
+ Maximize2,
+ Medal,
+ MessageSquareText,
+ MousePointerClick,
+ Search,
+ Sparkles,
+ TrendingUp,
+ Users,
+ type LucideIcon,
+} from "lucide-react";
+import { useEffect, useId, useMemo, useState, type ReactNode } from "react";
+
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogHeader,
+ DialogTitle,
+ DialogTrigger,
+} from "@/components/ui/dialog";
+import { Input } from "@/components/ui/input";
+import { Pagination } from "@/components/ui/pagination";
+import { ScrollArea } from "@/components/ui/scroll-area";
+import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import type {
+ AdminLeaderboardDimension,
+ AdminLeaderboardOverview,
+ AdminLeaderboardQuestion,
+ AdminLeaderboardResponse,
+ AdminLeaderboardSkill,
+ AdminLeaderboardTrendPoint,
+ AdminLeaderboardUser,
+} from "@/core/admin/api";
+import { getBackendBaseURL } from "@/core/config";
+import { cn } from "@/lib/utils";
+
+type UserRankTab = "activity" | "questions" | "tokens" | "skills" | "efficiency";
+type LeaderboardAccent = "crimson" | "rose" | "amber" | "teal";
+
+const RANK_COLORS = ["#C1121F", "#E11D48", "#F59E0B", "#991B1B", "#B45309"] as const;
+const LEADERBOARD_PAGE_SIZE = 20;
+const RANGE_DAYS = 30;
+
+const accentClasses: Record = {
+ crimson: {
+ soft: "bg-[#FFF1F1] dark:bg-[#2563EB]/15",
+ text: "text-[#C1121F] dark:text-[#7FACF2]",
+ bar: "bg-[#C1121F] dark:bg-[#2563EB]",
+ border: "border-[#F1D6D6] dark:border-[#213D5E]",
+ },
+ rose: {
+ soft: "bg-[#FFF1F2] dark:bg-rose-400/15",
+ text: "text-[#E11D48] dark:text-rose-200",
+ bar: "bg-[#E11D48]",
+ border: "border-[#FECDD3] dark:border-rose-300/25",
+ },
+ amber: {
+ soft: "bg-[#FFFBEB] dark:bg-amber-300/15",
+ text: "text-[#B45309] dark:text-amber-100",
+ bar: "bg-[#F59E0B]",
+ border: "border-[#FDE68A] dark:border-amber-200/25",
+ },
+ teal: {
+ soft: "bg-[#ECFDF5] dark:bg-teal-300/15",
+ text: "text-[#0F766E] dark:text-teal-100",
+ bar: "bg-[#0F766E]",
+ border: "border-[#A7F3D0] dark:border-teal-200/25",
+ },
+};
+
+function fmtNumber(value?: number | null): string {
+ return Number(value ?? 0).toLocaleString();
+}
+
+function fmtTime(value?: string | null): string {
+ if (!value) return "-";
+ const date = new Date(value);
+ return Number.isNaN(date.getTime()) ? value : date.toLocaleString();
+}
+
+function fmtDate(value?: string | null): string {
+ if (!value) return "";
+ const date = new Date(value);
+ return Number.isNaN(date.getTime()) ? "" : date.toLocaleString("zh-CN", { hour12: false });
+}
+
+function clampText(value: string, max = 120): string {
+ const text = value.trim();
+ return text.length > max ? `${text.slice(0, max)}...` : text;
+}
+
+function userScore(user: AdminLeaderboardUser, tab: UserRankTab): number {
+ if (tab === "questions") return user.qa_count;
+ if (tab === "tokens") return user.total_tokens;
+ if (tab === "skills") return user.skill_call_count;
+ if (tab === "efficiency") {
+ return user.run_count > 0 ? Math.round((user.qa_count / user.run_count) * 100) : 0;
+ }
+ return user.run_count + user.tool_call_count;
+}
+
+function scoreLabel(user: AdminLeaderboardUser, tab: UserRankTab): string {
+ if (tab === "questions") return `${fmtNumber(user.qa_count)} 次问答`;
+ if (tab === "tokens") return `${fmtNumber(user.total_tokens)} tokens`;
+ if (tab === "skills") return `${fmtNumber(user.skill_call_count)} 次技能`;
+ if (tab === "efficiency") {
+ return `${(user.run_count > 0 ? user.qa_count / user.run_count : 0).toFixed(1)} 问答/运行`;
+ }
+ return `${fmtNumber(userScore(user, tab))} 次使用`;
+}
+
+function rankTabDescription(tab: UserRankTab): string {
+ if (tab === "questions") return "问答排行按用户提出有效问题的运行次数排序。";
+ if (tab === "tokens") return "消耗排行按模型输入与输出 Token 总量排序。";
+ if (tab === "skills") return "技能排行按带有技能名称的工具调用次数排序。";
+ if (tab === "efficiency") return "效率排行 = 问答次数 / 运行次数,用来衡量每次运行承载的问答密度。";
+ return "使用排行按运行次数与工具调用次数合计排序。";
+}
+
+function initials(user: AdminLeaderboardUser): string {
+ const source = user.email || user.user_id || "?";
+ return source.slice(0, 1).toUpperCase();
+}
+
+async function fetchPublicLeaderboard(days: number): Promise {
+ const res = await fetch(
+ `${getBackendBaseURL()}/api/public/leaderboard?days=${days}&limit=100`,
+ { headers: { Accept: "application/json" } },
+ );
+ if (!res.ok) {
+ const err = (await res.json().catch(() => ({}))) as { detail?: string };
+ throw new Error(err.detail ?? `加载排行榜失败 (HTTP ${res.status})`);
+ }
+ return (await res.json()) as AdminLeaderboardResponse;
+}
+
+export default function PublicLeaderboardPage() {
+ const [data, setData] = useState(null);
+ const [error, setError] = useState(null);
+ const [loading, setLoading] = useState(true);
+ const [tab, setTab] = useState("activity");
+ const [query, setQuery] = useState("");
+ const [userPage, setUserPage] = useState(1);
+ const days = RANGE_DAYS;
+
+ useEffect(() => {
+ let cancelled = false;
+ setLoading(true);
+ fetchPublicLeaderboard(days)
+ .then((res) => {
+ if (!cancelled) {
+ setData(res);
+ setError(null);
+ }
+ })
+ .catch((e) => {
+ if (!cancelled) setError(e instanceof Error ? e.message : String(e));
+ })
+ .finally(() => {
+ if (!cancelled) setLoading(false);
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [days]);
+
+ const pending = data?.status === "running" || data?.status === "refreshing";
+
+ // The snapshot may still be computing on first load — poll briefly until ready.
+ useEffect(() => {
+ if (!pending) return;
+ const timer = setInterval(() => {
+ fetchPublicLeaderboard(days)
+ .then((res) => setData(res))
+ .catch(() => undefined);
+ }, 3000);
+ return () => clearInterval(timer);
+ }, [pending, days]);
+
+ const rankedUsers = useMemo(() => {
+ const base = data?.users_by_activity ?? [];
+ const source =
+ tab === "questions"
+ ? data?.users_by_questions
+ : tab === "tokens"
+ ? data?.users_by_tokens
+ : tab === "skills"
+ ? [...base].sort((a, b) => b.skill_call_count - a.skill_call_count)
+ : tab === "efficiency"
+ ? [...base].sort((a, b) => userScore(b, "efficiency") - userScore(a, "efficiency"))
+ : data?.users_by_activity;
+ const needle = query.trim().toLowerCase();
+ return (source ?? []).filter((user) =>
+ needle ? `${user.email} ${user.user_id}`.toLowerCase().includes(needle) : true,
+ );
+ }, [data, query, tab]);
+
+ const overview = data?.overview;
+ const hasRows = rankedUsers.length > 0 || (overview?.total_runs ?? 0) > 0 || (overview?.total_users ?? 0) > 0;
+ const topUsers = rankedUsers.slice(0, 3);
+ const userPageCount = Math.max(1, Math.ceil(rankedUsers.length / LEADERBOARD_PAGE_SIZE));
+ const visibleUsers = useMemo(() => {
+ const start = (userPage - 1) * LEADERBOARD_PAGE_SIZE;
+ return rankedUsers.slice(start, start + LEADERBOARD_PAGE_SIZE);
+ }, [rankedUsers, userPage]);
+
+ useEffect(() => {
+ setUserPage(1);
+ }, [query, tab]);
+
+ useEffect(() => {
+ setUserPage((page) => Math.min(page, userPageCount));
+ }, [userPageCount]);
+
+ return (
+
+
+ {/* 标题区:仅展示,不含任何操作按钮 */}
+
+
+ 用户排行榜
+
+ 展示近 {data?.range_days ?? days} 天用户活跃、问答、Token 消耗、技能与工具热度、模型与 Agent 使用及近期问题。
+ {data?.since && data?.until && (
+
+ ({fmtDate(data.since).slice(0, 10)} ~ {fmtDate(data.until).slice(0, 10)})
+
+ )}
+
+
+
+ {loading && !data && (
+
+
+ 正在加载排行榜数据
+
+ )}
+
+ {error && (
+
+ 加载失败:{error}
+
+ )}
+
+ {data && overview && (
+
+ {/* 仅在首次统计、尚无数据时提示;已有数据时后台静默刷新,不再常驻横幅 */}
+ {pending && !hasRows && (
+
+
+ 排行榜正在后台统计,数据会自动刷新。
+
+ )}
+
+ {/* 概览指标 */}
+
+
+ {/* 衍生洞察 */}
+
+
+ {/* 榜单前三 */}
+
+
+
+
榜单前三
+
{rankTabDescription(tab)}
+
+
setTab(value as UserRankTab)}>
+
+ 使用排行
+ 问答排行
+ 消耗排行
+ 技能排行
+ 效率排行
+
+
+
+
+
+
+ {/* 完整排行榜 */}
+
+
+
+
+
+
完整排行榜
+
{rankTabDescription(tab)}
+
+
+
+
+ setQuery(event.target.value)}
+ placeholder="搜索用户"
+ className="h-9 w-56 bg-white pl-8 dark:bg-[#011226]"
+ />
+
+
+
+
+
+ 共 {fmtNumber(rankedUsers.length)} 位用户,当前第 {userPage} / {userPageCount} 页
+
+
+
+
+
+ {/* 时间趋势 */}
+
+
+
+
+
时间趋势
+
+ {data.since && data.until && (
+
+
+ {fmtDate(data.since).slice(0, 10)} 至 {fmtDate(data.until).slice(0, 10)}
+
+ )}
+
+
+
+
+ {/* 技能与工具热度 */}
+
+
+
+
+
+ {/* 模型 / Agent / 运行状态 */}
+
+
+
+
+
+
+ {/* 问题 */}
+
+
+
+
+
+ )}
+
+
+
+
+ );
+}
+
+function StatCard({
+ icon: Icon,
+ label,
+ value,
+ note,
+ accent,
+}: {
+ icon: LucideIcon;
+ label: string;
+ value: string;
+ note: string;
+ accent: LeaderboardAccent;
+}) {
+ const color = accentClasses[accent];
+ return (
+
+
+ {label}
+
+
+
+
+
+ {value}
+
+
{note}
+
+ );
+}
+
+function InsightStrip({ overview }: { overview: AdminLeaderboardOverview }) {
+ const activeUsers = Math.max(1, overview.active_users);
+ const totalRuns = Math.max(1, overview.total_runs);
+ const items = [
+ {
+ label: "人均使用",
+ value: overview.total_runs / activeUsers,
+ unit: "次/活跃用户",
+ description: "当前时间段内,平均每位活跃用户发起了多少次运行。",
+ },
+ {
+ label: "问答密度",
+ value: overview.total_questions / totalRuns,
+ unit: "问答/运行",
+ description: "问答次数 ÷ 运行次数,用来判断每次使用承载了多少有效提问。",
+ },
+ {
+ label: "平均 Token",
+ value: overview.total_tokens / totalRuns,
+ unit: "Token/运行",
+ description: "总 Token ÷ 运行次数,反映单次使用的模型消耗强度。",
+ },
+ {
+ label: "技能覆盖",
+ value: overview.total_tool_calls ? (overview.total_skill_calls / overview.total_tool_calls) * 100 : 0,
+ unit: "%",
+ description: "技能调用 ÷ 工具总调用,越高说明用户越多依赖已配置技能能力。",
+ },
+ {
+ label: "失败率",
+ value: (overview.error_runs / totalRuns) * 100,
+ unit: "%",
+ description: "失败运行 ÷ 总运行,用来快速发现稳定性或配置问题。",
+ },
+ ];
+ return (
+
+ {items.map((item) => (
+
+
{item.label}
+
+ {Number(item.value).toFixed(item.unit === "%" || String(item.value).includes(".") ? 1 : 0)}
+ {item.unit}
+
+
{item.description}
+
+ ))}
+
+ );
+}
+
+function PodiumCard({
+ user,
+ rank,
+ tab,
+}: {
+ user?: AdminLeaderboardUser;
+ rank: 1 | 2 | 3;
+ tab: UserRankTab;
+}) {
+ const styles = {
+ 1: "border-[#FEEFC3] bg-[#FEF7E0] text-[#B06000] dark:border-[#F4B400]/40 dark:bg-[#F4B400]/10 dark:text-[#FDD663]",
+ 2: "border-[#E5E7EB] bg-[#F8FAFC] text-[#4B5563] dark:border-[#6B7280]/40 dark:bg-white/5 dark:text-[#E5E7EB]",
+ 3: "border-[#FAD2CF] bg-[#FCE8E6] text-[#DB4437] dark:border-[#DB4437]/40 dark:bg-[#DB4437]/10 dark:text-[#F28B82]",
+ }[rank];
+
+ if (!user) {
+ return (
+
+ );
+ }
+
+ return (
+
+
+
+ {rank === 1 ? : }
+ 第 {rank} 名
+
+ {user.system_role === "admin" && 管理员}
+
+
+
+ {initials(user)}
+
+
{user.email || user.user_id}
+
{scoreLabel(user, tab)}
+
+ 运行 {fmtNumber(user.run_count)}
+ 问答 {fmtNumber(user.qa_count)}
+ 技能 {fmtNumber(user.skill_call_count)}
+
+
+
+ );
+}
+
+function UserTable({
+ users,
+ tab,
+ rankOffset = 0,
+}: {
+ users: AdminLeaderboardUser[];
+ tab: UserRankTab;
+ rankOffset?: number;
+}) {
+ const max = Math.max(1, ...users.map((user) => userScore(user, tab)));
+ if (users.length === 0) {
+ return 暂无用户统计
;
+ }
+
+ return (
+
+ {users.map((user, index) => {
+ const score = userScore(user, tab);
+ return (
+
+
+ {rankOffset + index + 1}
+
+
+
+ {user.email || user.user_id}
+ {user.system_role === "admin" && 管理员}
+
+
+ 最近活跃 {fmtTime(user.last_active_at)}
+
+
+
+
+
+ 会话 {fmtNumber(user.thread_count)} · LLM {fmtNumber(user.llm_call_count)}
+
+
+
+
{scoreLabel(user, tab)}
+
+ {fmtNumber(user.total_tokens)} tokens
+
+
+
+ );
+ })}
+
+ );
+}
+
+function SkillRanking({
+ title,
+ icon: Icon,
+ items,
+ empty,
+}: {
+ title: string;
+ icon: LucideIcon;
+ items: AdminLeaderboardSkill[];
+ empty: string;
+}) {
+ const max = Math.max(1, ...items.map((item) => item.call_count));
+ return (
+
+
+
+
+
{title}
+
+
{items.length} 项
+
+
+ {items.length === 0 &&
{empty}
}
+ {items.slice(0, 10).map((item, index) => (
+
+
{index + 1}
+
+
+ {item.name}
+
+ {fmtNumber(item.user_count)} 人
+
+
+
+
+
+
+ {fmtNumber(item.call_count)}
+
+
+ 失败 {fmtNumber(item.error_count)}
+
+
+
+ ))}
+
+
+ );
+}
+
+function DimensionRanking({
+ title,
+ icon: Icon,
+ items,
+ empty,
+}: {
+ title: string;
+ icon: LucideIcon;
+ items: AdminLeaderboardDimension[];
+ empty: string;
+}) {
+ const max = Math.max(1, ...items.map((item) => item.count));
+ return (
+
+
+
+
+
{title}
+
+
{items.length} 项
+
+
+ {items.length === 0 &&
{empty}
}
+ {items.slice(0, 8).map((item, index) => (
+
+
{index + 1}
+
+
+ {item.name}
+
+ {fmtNumber(item.user_count)} 人
+
+
+
+
+ 成功 {fmtNumber(item.success_count)} · 失败 {fmtNumber(item.error_count)} · {fmtNumber(item.total_tokens)} tokens
+
+
+
+
{fmtNumber(item.count)}
+
次
+
+
+ ))}
+
+
+ );
+}
+
+function QuestionPanel({
+ title,
+ items,
+}: {
+ title: string;
+ items: AdminLeaderboardQuestion[];
+}) {
+ return (
+
+
+
+
+
{title}
+
+
{items.length}
+
+
+
+ {items.length === 0 &&
暂无问题记录
}
+ {items.map((item, index) => (
+
+
{clampText(item.text, 180)}
+
+ {item.email || item.user_id || "未知用户"}
+ {fmtTime(item.asked_at)}
+
+
+ ))}
+
+
+
+ );
+}
+
+function TrendBars({ points }: { points: AdminLeaderboardTrendPoint[] }) {
+ type TrendMetric = keyof Pick;
+ const [metric, setMetric] = useState<"all" | TrendMetric>("all");
+ const chartId = useId().replace(/:/g, "");
+ if (points.length === 0) {
+ return 暂无趋势数据
;
+ }
+
+ const visible = points.slice(-32);
+ const series = [
+ { key: "runs", label: "运行", color: "#C1121F", soft: "bg-[#FFF1F1] text-[#C1121F] dark:text-[#7FACF2] dark:bg-[#2563EB]/15 dark:text-[#7FACF2]", description: "每天发起的运行次数" },
+ { key: "questions", label: "问答", color: "#0F766E", soft: "bg-[#ECFDF5] text-[#0F766E] dark:bg-teal-300/15 dark:text-teal-100", description: "每天产生的有效提问次数" },
+ { key: "users", label: "活跃", color: "#B45309", soft: "bg-[#FFFBEB] text-[#B45309] dark:bg-amber-300/15 dark:text-amber-100", description: "每天有行为的用户数量" },
+ { key: "tokens", label: "Token", color: "#7C3AED", soft: "bg-[#F5F3FF] text-[#7C3AED] dark:bg-violet-300/15 dark:text-violet-100", description: "每天模型输入输出 Token 总量" },
+ ] as const;
+ const activeMetric: TrendMetric = metric === "all" ? "runs" : metric;
+ const activeSeries = series.find((item) => item.key === activeMetric) ?? series[0];
+ const metricMax = Math.max(1, ...visible.map((point) => Number(point[activeMetric] ?? 0)));
+ const latest = visible[visible.length - 1];
+ const previous = visible[visible.length - 2];
+ const latestValue = Number(latest?.[activeMetric] ?? 0);
+ const previousValue = Number(previous?.[activeMetric] ?? 0);
+ const delta = latestValue - previousValue;
+ const width = 920;
+ const height = 280;
+ const padding = { top: 22, right: 28, bottom: 36, left: 48 };
+ const chartWidth = width - padding.left - padding.right;
+ const chartHeight = height - padding.top - padding.bottom;
+ const xFor = (index: number) =>
+ padding.left + (visible.length <= 1 ? chartWidth / 2 : (index / (visible.length - 1)) * chartWidth);
+ const yFor = (value: number, max: number) => padding.top + chartHeight - (value / max) * chartHeight;
+ const gridLines = [0, 0.25, 0.5, 0.75, 1];
+ const selectedPath = visible
+ .map((point, index) => {
+ const value = Number(point[activeMetric] ?? 0);
+ return `${index === 0 ? "M" : "L"} ${xFor(index).toFixed(2)} ${yFor(value, metricMax).toFixed(2)}`;
+ })
+ .join(" ");
+ const selectedArea = `${selectedPath} L ${xFor(visible.length - 1).toFixed(2)} ${padding.top + chartHeight} L ${xFor(0).toFixed(2)} ${padding.top + chartHeight} Z`;
+ const peak = visible.reduce(
+ (best, point) => (Number(point[activeMetric] ?? 0) > Number(best[activeMetric] ?? 0) ? point : best),
+ visible[0],
+ );
+ const displayedSeries = metric === "all" ? series : [activeSeries];
+ const totalFor = (key: TrendMetric) => visible.reduce((sum, point) => sum + Number(point[key] ?? 0), 0);
+ const metricCards = ({ large = false }: { large?: boolean } = {}) => (
+
+
+ {series.map((item) => (
+
+ ))}
+
+ );
+
+ const analysisDialog = (title: string, description: ReactNode) => (
+
+ );
+
+ const chart = (large = false) => (
+
+
+
+ );
+
+ return (
+
+
+ 每个折线点代表一天;点击指标卡可单独查看运行、问答、活跃或 Token,全部趋势会同时展示四条多色折线。
+
+
+
{metricCards()}
+ {analysisDialog(
+ "时间趋势分析",
+ <>当前查看 {metric === "all" ? "全部指标" : activeSeries.label},峰值 {peak.date} / {fmtNumber(Number(peak[activeMetric] ?? 0))}。>,
+ )}
+
+
+
+
最新 {metric === "all" ? "运行" : activeSeries.label}
+
{fmtNumber(latestValue)}
+
= 0 ? "text-[#C1121F] dark:text-[#7FACF2]" : "text-[#7F1D1D] dark:text-[#FCA5A5]")}>
+ 较上一日 {delta >= 0 ? "+" : ""}{fmtNumber(delta)}
+
+
+ 峰值 {peak.date.slice(5)} / {fmtNumber(Number(peak[activeMetric] ?? 0))}
+
+
+ {chart(false)}
+
+
+ );
+}
diff --git a/frontend-web/src/pages/PublicScheduledTaskPage.tsx b/frontend-web/src/pages/PublicScheduledTaskPage.tsx
index a57f653c..e3e251f3 100644
--- a/frontend-web/src/pages/PublicScheduledTaskPage.tsx
+++ b/frontend-web/src/pages/PublicScheduledTaskPage.tsx
@@ -6,16 +6,18 @@ import {
ChevronLeft,
ChevronRight,
Download,
+ ExternalLink,
FileText,
List,
Loader2,
+ Maximize2,
Pencil,
Trash2,
X,
} from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import type { PointerEvent as ReactPointerEvent } from "react";
-import { useParams } from "react-router-dom";
+import { useParams, useSearchParams } from "react-router-dom";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
@@ -127,6 +129,10 @@ async function fetchPublicTaskRuns(taskId: string): Promise
return fetchJson(`${publicBaseUrl()}/${encodeURIComponent(taskId)}/runs?limit=200`);
}
+async function fetchPublicRun(runId: string): Promise {
+ return fetchJson(`${publicBaseUrl()}/runs/${encodeURIComponent(runId)}`);
+}
+
async function fetchPublicRunFile(runId: string, index: number): Promise {
return fetchJson(
`${publicBaseUrl()}/runs/${encodeURIComponent(runId)}/files/${index}`,
@@ -667,12 +673,61 @@ type ContentTab = {
fileName?: string;
};
+type PublicFullscreenPreview =
+ | { mode: "html"; title: string; html: string }
+ | { mode: "markdown"; title: string; content: string }
+ | { mode: "text"; title: string; content: string }
+ | { mode: "file"; title: string; file: PublicFileContent };
+
+function PublicFullscreenPreviewContent({ preview }: { preview: PublicFullscreenPreview }) {
+ if (preview.mode === "html") {
+ return ;
+ }
+ if (preview.mode === "markdown") {
+ return (
+
+ );
+ }
+ if (preview.mode === "file") {
+ const content = preview.file.content || "无文件内容";
+ if (isMarkdownFile(preview.file)) {
+ return (
+
+ );
+ }
+ return {content};
+ }
+ return {preview.content || "无正文内容"};
+}
+
+function publicFullscreenHref(taskId: string, runId: string, tab: ContentTab): string {
+ const params = new URLSearchParams();
+ params.set("tab", tab.kind);
+ if (tab.kind === "file" && tab.fileIndex != null) {
+ params.set("file", String(tab.fileIndex));
+ }
+ return `#/public/scheduled-tasks/${encodeURIComponent(taskId)}/runs/${encodeURIComponent(runId)}/fullscreen?${params.toString()}`;
+}
+
/**
* Right-pane controller for a single run: builds the tab list (body + MDs),
* fetches each tab's source text, and hosts the view/edit toggle.
*/
function RunDetailPane({ run, taskId }: { run: PublicRun; taskId: string }) {
const queryClient = useQueryClient();
+ const [fullscreenPreview, setFullscreenPreview] = useState(null);
// HTML page run: render a sandboxed iframe preview instead of the MD tabs.
const isHtmlRun =
@@ -837,9 +892,67 @@ function RunDetailPane({ run, taskId }: { run: PublicRun; taskId: string }) {
const fileIndex = activeTabMeta?.fileIndex;
const fileIsLoading = isFileTab && fileIndex != null ? !!fileLoading[fileIndex] : false;
const showEmptyHint = !activeSource && !fileIsLoading;
+ const fullscreenHref = activeTabMeta ? publicFullscreenHref(taskId, run.id, activeTabMeta) : "";
+ const fullscreenDisabled =
+ !activeTabMeta ||
+ (activeTabMeta.kind === "html" ? !htmlContent : activeTabMeta.kind === "file" ? fileIsLoading : false);
+
+ const handleOpenFullscreen = useCallback(() => {
+ if (!activeTabMeta) return;
+ if (activeTabMeta.kind === "html") {
+ if (!htmlContent) return;
+ setFullscreenPreview({
+ mode: "html",
+ title: run.result?.title || "HTML 预览",
+ html: htmlContent,
+ });
+ return;
+ }
+ if (activeTabMeta.kind === "body") {
+ setFullscreenPreview({
+ mode: run.result?.format === "markdown" || run.result?.format === "mixed" ? "markdown" : "text",
+ title: run.result?.title || "正文预览",
+ content: activeSource || "无正文内容",
+ });
+ return;
+ }
+ const sourceFile = markdownFiles.find((file) => file.index === activeTabMeta.fileIndex);
+ setFullscreenPreview({
+ mode: "file",
+ title: activeTabMeta.fileName || sourceFile?.name || "文件预览",
+ file: {
+ index: activeTabMeta.fileIndex ?? 0,
+ name: activeTabMeta.fileName || sourceFile?.name || "result.md",
+ mime_type: sourceFile?.mime_type ?? "text/markdown",
+ kind: sourceFile?.kind ?? "markdown",
+ size: sourceFile?.size ?? activeSource.length,
+ previewable: true,
+ content: activeSource,
+ },
+ });
+ }, [activeTabMeta, activeSource, htmlContent, markdownFiles, run.result?.format, run.result?.title]);
return (
<>
+
@@ -859,6 +972,30 @@ function RunDetailPane({ run, taskId }: { run: PublicRun; taskId: string }) {
+ {!isEditing && activeTabMeta && (
+ <>
+
+
+ >
+ )}
{isFileTab && fileIndex != null && (
);
}
+
+export function PublicScheduledTaskFullscreenPage() {
+ const { taskId, runId } = useParams<{ taskId: string; runId: string }>();
+ const [searchParams] = useSearchParams();
+ const requestedTab = searchParams.get("tab");
+ const tabKind: ContentTab["kind"] =
+ requestedTab === "body" || requestedTab === "file" || requestedTab === "html"
+ ? requestedTab
+ : "html";
+ const fileIndexValue = Number(searchParams.get("file"));
+ const fileIndexValid = Number.isInteger(fileIndexValue) && fileIndexValue >= 0;
+
+ const { data: run, isLoading, isError, error } = useQuery({
+ queryKey: ["public-scheduled-task", "run", runId],
+ queryFn: () => fetchPublicRun(runId!),
+ enabled: !!runId,
+ refetchOnWindowFocus: false,
+ retry: false,
+ });
+
+ const isHtmlRun =
+ run?.result?.format === "html" ||
+ (run?.result?.files ?? []).some(
+ (file) => file.kind === "html" || (file.name ?? "").toLowerCase().endsWith(".html"),
+ );
+
+ const { data: htmlData, isLoading: htmlLoading } = useQuery({
+ queryKey: ["public-scheduled-task", "html", runId],
+ queryFn: () => fetchPublicRunHtml(runId!),
+ enabled: !!runId && tabKind === "html" && isHtmlRun && !run?.result?.html,
+ refetchOnWindowFocus: false,
+ retry: false,
+ });
+
+ const {
+ data: fileData,
+ isLoading: fileLoading,
+ isError: fileIsError,
+ error: fileError,
+ } = useQuery({
+ queryKey: ["public-scheduled-task", "file", runId, fileIndexValue],
+ queryFn: () => fetchPublicRunFile(runId!, fileIndexValue),
+ enabled: !!runId && tabKind === "file" && fileIndexValid,
+ refetchOnWindowFocus: false,
+ retry: false,
+ });
+
+ const htmlContent = run?.result?.html ?? htmlData?.html ?? null;
+ const backHref = taskId ? `#/public/scheduled-tasks/${encodeURIComponent(taskId)}` : "#/public/scheduled-tasks";
+ const pageTitle =
+ tabKind === "file"
+ ? fileData?.name || "文件预览"
+ : tabKind === "body"
+ ? run?.result?.title || "正文预览"
+ : htmlData?.title || run?.result?.title || "HTML 预览";
+
+ return (
+
+
+
+
+
+ {pageTitle}
+
+ {tabKind === "file" && runId && fileIndexValid && (
+
+ )}
+
+
+
+ {!runId || !taskId ? (
+
+ 缺少任务或执行记录 ID。
+
+ ) : isLoading ? (
+
+
+ 正在加载预览...
+
+ ) : isError ? (
+
+ 加载失败:{error instanceof Error ? error.message : "未知错误"}
+
+ ) : tabKind === "html" ? (
+ htmlContent ? (
+
+ ) : (
+
+ {htmlLoading && }
+ HTML 页面尚未生成或正在生成中。
+
+ )
+ ) : tabKind === "file" ? (
+ fileLoading ? (
+
+
+ 正在加载文件...
+
+ ) : !fileIndexValid || fileIsError || !fileData ? (
+
+ 文件加载失败:{fileError instanceof Error ? fileError.message : "文件不存在或不可预览"}
+
+ ) : (
+
+
+
+ )
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+}
diff --git a/frontend-web/src/pages/RecommendedQuestionsPage.tsx b/frontend-web/src/pages/RecommendedQuestionsPage.tsx
index 6a22170c..3877fd1f 100644
--- a/frontend-web/src/pages/RecommendedQuestionsPage.tsx
+++ b/frontend-web/src/pages/RecommendedQuestionsPage.tsx
@@ -59,7 +59,11 @@ const errorToastDarkOptions = {
icon:
,
} as const;
-export default function RecommendedQuestionsPage() {
+export default function RecommendedQuestionsPage({
+ embedded = false,
+}: {
+ embedded?: boolean;
+} = {}) {
const { theme } = useTheme();
const { questions, isLoading, error } = useRecommendedQuestions();
const createMutation = useCreateRecommendedQuestion();
@@ -176,10 +180,16 @@ export default function RecommendedQuestionsPage() {
: { success: undefined, error: undefined };
return (
-
+
-
推荐问题配置
+
推荐问题配置
管理首页展示的推荐问题,点击后直接发送到对话。拖拽可调整顺序。
diff --git a/frontend-web/src/pages/ScheduledTasksPage.tsx b/frontend-web/src/pages/ScheduledTasksPage.tsx
index 6a527443..ae93d9e7 100644
--- a/frontend-web/src/pages/ScheduledTasksPage.tsx
+++ b/frontend-web/src/pages/ScheduledTasksPage.tsx
@@ -21,6 +21,7 @@ import {
RefreshCw,
TagsIcon,
Trash2,
+ X,
} from "lucide-react";
import { useSearchParams } from "react-router-dom";
import { useRouteNavigate as useNavigate } from "@/core/route-base";
@@ -148,8 +149,16 @@ export default function ScheduledTasksPage() {
const initialTaskId = searchParams.get("task");
const [search, setSearch] = useState("");
const [tagIds, setTagIds] = useState
([]);
+ // Plaza (广场) name search is resolved client-side over the fetched published
+ // list, kept separate from the server-side 我的任务 search above.
+ const [plazaSearch, setPlazaSearch] = useState("");
+ // Publisher (创建者) filter for the 我的任务 tab — click a task's 创建者 to keep
+ // only that creator's tasks. {id} is creator_user_id; {label} the display name.
+ const [publisher, setPublisher] = useState<{ id: string; label: string } | null>(
+ null,
+ );
// Name + tag filtering is resolved server-side via the /api/scheduled-tasks query.
- const { data: tasks = [], isLoading } = useScheduledTasks({ search, tagIds });
+ const { data: tasks = [], isLoading, isFetching } = useScheduledTasks({ search, tagIds });
const { data: deliveryProfile } = useScheduledTaskDeliveryProfile();
const { agents } = useAgents();
const actions = useScheduledTaskActions();
@@ -199,6 +208,34 @@ export default function ScheduledTasksPage() {
}),
[tasks],
);
+ // The rendered 我的任务 list, optionally narrowed to one creator. Selection
+ // logic still runs against the full sortedTasks so the detail panel keeps
+ // working even when the picked task is from a filtered-out creator.
+ const visibleTasks = useMemo(
+ () =>
+ publisher
+ ? sortedTasks.filter((task) => task.creator_user_id === publisher.id)
+ : sortedTasks,
+ [sortedTasks, publisher],
+ );
+ // Plaza search: filter the fetched published list client-side. Match against
+ // both the raw and desensitized forms so typing what's shown on screen (the
+ // desensitized name) and the underlying text both hit.
+ const visiblePublished = useMemo(() => {
+ const q = plazaSearch.trim().toLowerCase();
+ if (!q) return published;
+ return published.filter((item) =>
+ [
+ item.name,
+ desensitize(item.name),
+ item.prompt_summary,
+ desensitize(item.prompt_summary ?? ""),
+ item.schedule_text,
+ displayUsername(item.creator_email),
+ ].some((field) => (field ?? "").toLowerCase().includes(q)),
+ );
+ }, [published, plazaSearch]);
+
const selectedTask = sortedTasks.find((task) => task.task_id === selectedTaskId) ?? null;
const selectedTaskCanEditRuns = !!selectedTask && selectedTask.role !== "subscriber";
const { data: runs = [], isLoading: runsLoading } = useScheduledTaskRuns(selectedTaskId);
@@ -449,8 +486,32 @@ export default function ScheduledTasksPage() {
onSearchChange={setSearch}
tagIds={tagIds}
onTagIdsChange={setTagIds}
+ searching={isFetching}
/>
)}
+ {activeTab === "plaza" && (
+
+ )}
+ {activeTab === "mine" && publisher && (
+
+ 筛选创建者:
+
+
+ )}
@@ -459,12 +520,14 @@ export default function ScheduledTasksPage() {
{isLoading && (
正在加载任务...
)}
- {!isLoading && sortedTasks.length === 0 && (
+ {!isLoading && visibleTasks.length === 0 && (
- 还没有定时任务。点击新建创建第一个任务,或去“广场”收藏他人发布的任务。
+ {publisher
+ ? `没有「${publisher.label}」创建的任务。`
+ : "还没有定时任务。点击新建创建第一个任务,或去“广场”收藏他人发布的任务。"}
)}
- {sortedTasks.map((task) => {
+ {visibleTasks.map((task) => {
const isOwner = task.role !== "subscriber";
const isSubscriber = task.role === "subscriber";
const notifyOn = !!task._subscription?.notify_element;
@@ -539,11 +602,36 @@ export default function ScheduledTasksPage() {
)}
{(task.creator_email || task.creator_user_id) && (
-
+ // The card itself is a
+
)}
{(task.tags?.length ?? 0) > 0 && (
@@ -694,14 +782,19 @@ export default function ScheduledTasksPage() {
) : (
{publishedLoading && (
-
正在加载广场任务...
- )}
- {!publishedLoading && published.length === 0 && (
-
- 广场上还没有发布的任务。
+
+
+ 正在加载广场任务...
)}
- {published.map((item) => {
+ {!publishedLoading && visiblePublished.length === 0 && (
+
+ {plazaSearch.trim()
+ ? `没有匹配「${plazaSearch.trim()}」的任务。`
+ : "广场上还没有发布的任务。"}
+
+ )}
+ {visiblePublished.map((item) => {
const subscribed = tasks.some((t) => t.task_id === item.task_id);
return (
@@ -781,7 +874,7 @@ export default function ScheduledTasksPage() {
{selectedTask ? desensitize(selectedTask.name) : "请选择左侧任务"}
{selectedTask && (selectedTask.creator_email || selectedTask.creator_user_id) && (
-
+
创建者:
{displayUsername(selectedTask.creator_email) ||
selectedTask.creator_user_id}
diff --git a/frontend-web/src/pages/WorkspaceRoutes.tsx b/frontend-web/src/pages/WorkspaceRoutes.tsx
index 57ff765d..bc5579e9 100644
--- a/frontend-web/src/pages/WorkspaceRoutes.tsx
+++ b/frontend-web/src/pages/WorkspaceRoutes.tsx
@@ -24,6 +24,8 @@ import AdminScheduledTasksPage from './AdminScheduledTasksPage';
import AdminSkillsPage from './AdminSkillsPage';
import ScheduledTaskRunDetailPage from './ScheduledTaskRunDetailPage';
import ScheduledTasksPage from './ScheduledTasksPage';
+import KnowledgeBasePage from './KnowledgeBasePage';
+import KnowledgeGraphPage from './KnowledgeGraphPage';
import LlmMetricsPage from './LlmMetricsPage';
import CheckpointMigrationPage from './CheckpointMigrationPage';
import MemoryPage from './MemoryPage';
@@ -344,6 +346,30 @@ export default function WorkspaceRoutes({
}
/>
+
+
+
+ }
+ />
+
+
+
+ }
+ />
+
+
+
+ }
+ />
}
diff --git a/frontend-web/src/strategy-components/MarkdownRenderer.tsx b/frontend-web/src/strategy-components/MarkdownRenderer.tsx
index 51821967..42e5879a 100644
--- a/frontend-web/src/strategy-components/MarkdownRenderer.tsx
+++ b/frontend-web/src/strategy-components/MarkdownRenderer.tsx
@@ -16,12 +16,33 @@ import tsx from 'react-syntax-highlighter/dist/esm/languages/prism/tsx';
import typescript from 'react-syntax-highlighter/dist/esm/languages/prism/typescript';
import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism';
+export interface WikiLinkTarget {
+ target: string;
+ kind: 'note' | 'vault' | 'missing';
+ exists: boolean;
+ note_id?: string;
+ vault_path?: string;
+ title?: string;
+}
+
+export type WikiLinkMap = Record;
+
interface MarkdownRendererProps {
content: string;
className?: string;
onCitationClick?: (index: number) => void;
+ /** Resolved `[[wikilink]]` targets; when provided, non-numeric `[[...]]` render as links. */
+ wikiLinks?: WikiLinkMap;
+ onWikiLinkClick?: (link: WikiLinkTarget) => void;
}
+const escapeHtmlAttr = (value: string): string =>
+ value
+ .replace(/&/g, '&')
+ .replace(//g, '>')
+ .replace(/"/g, '"');
+
SyntaxHighlighter.registerLanguage('bash', bash);
SyntaxHighlighter.registerLanguage('css', css);
SyntaxHighlighter.registerLanguage('html', markup);
@@ -126,8 +147,48 @@ const CodeBlock: React.FC<{ children: React.ReactNode; className?: string }> = (
);
};
-const MarkdownRenderer: React.FC = ({ content, className = '', onCitationClick }) => {
- const contentWithCitationBadge = content.replace(
+const MarkdownRenderer: React.FC = ({
+ content,
+ className = '',
+ onCitationClick,
+ wikiLinks,
+ onWikiLinkClick,
+}) => {
+ // 1) Non-numeric [[wikilinks]] → clickable (or greyed-out) links, driven by the
+ // resolver map. Numeric [[123]] are left untouched here for the citation pass.
+ let processed = content;
+ if (wikiLinks) {
+ processed = processed.replace(
+ /\[\[([^\]\n|]+)(?:\|([^\]\n]+))?\]\]/g,
+ (match, rawTarget: string, alias?: string) => {
+ const target = String(rawTarget).trim();
+ if (/^\d+$/.test(target)) return match; // citation badge, handled below
+ const resolved = wikiLinks[target];
+ const label = (alias?.trim() || resolved?.title || target);
+ const safeLabel = escapeHtmlAttr(label);
+ const safeTarget = escapeHtmlAttr(target);
+ if (resolved && resolved.exists) {
+ return ``;
+ }
+ return `${safeLabel}`;
+ },
+ );
+ }
+ // 2) wiki-capture provenance markers ^[inferred] / ^[ambiguous] → small badges
+ // (otherwise they render as raw text). They mark model-inferred / uncertain
+ // statements per the wiki-capture spec.
+ processed = processed
+ .replace(
+ /\s*\^\[inferred\]/g,
+ ' 推测',
+ )
+ .replace(
+ /\s*\^\[ambiguous\]/g,
+ ' 存疑',
+ );
+
+ // 3) Numeric [[123]] citation badges.
+ const contentWithCitationBadge = processed.replace(
/\[\[(\d+)\]\]/g,
'',
);
@@ -136,8 +197,17 @@ const MarkdownRenderer: React.FC = ({ content, className
{
- if (!onCitationClick) return;
const target = event.target as HTMLElement;
+ const wikiButton = target.closest('[data-wikilink-target]') as HTMLElement | null;
+ if (wikiButton && onWikiLinkClick && wikiLinks) {
+ const key = wikiButton.dataset.wikilinkTarget || '';
+ const link = wikiLinks[key];
+ if (link) {
+ onWikiLinkClick(link);
+ return;
+ }
+ }
+ if (!onCitationClick) return;
const button = target.closest('[data-citation-index]') as HTMLElement | null;
if (!button) return;
const index = Number(button.dataset.citationIndex || '0');
diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md
index f6bfc4c9..e8944de1 100644
--- a/offline-backend-20260512/backend/CLAUDE.md
+++ b/offline-backend-20260512/backend/CLAUDE.md
@@ -250,9 +250,18 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA
| **Scheduled Tasks** (`/api/scheduled-tasks`) | Per-user cron-like agent tasks. `GET/POST /` - list/create owned tasks; `PATCH/DELETE /{id}` - update/delete (owner); `POST /{id}/{pause,resume,trigger}`; `GET /{id}/runs`, `GET /runs/{rid}`, run-file preview/edit/download; `GET /runs/{rid}/messages` - agent messages for the run's underlying agent run (used to inspect a stuck/running run; authorized via `_load_run`); `GET /runs/{rid}/html` - generated HTML document for an `html_page` run; `DELETE /runs/{rid}` - owner-scoped, with an admin fallback so admins can delete any user's run; publish/subscribe under `/{id}/{publish,unpublish,subscribe}` and `GET /published`. **Admin (cross-user):** `GET /admin/all` - every task; `PATCH/DELETE /admin/{id}` - edit/delete any task; `POST /admin/{id}/trigger`; `GET /admin/{id}/runs`. Admin routes require `system_role == "admin"`. |
| **HTML Page Favorites** (`/api/html-page-favorites`) | Bookmarked generated HTML pages, owner-scoped. `GET /` - list; `POST /` `{page_id, title?, description?, tags?}` - bookmark a `scheduled_html_pages` row (extracts a reusable style/layout summary via GLM5 at favorite time); a page can only be favorited once per user (`DuplicateFavoriteError` → HTTP 409). `GET/DELETE /{id}` - read/delete (the create-task reference picker deletes favorites here). A favorite can be selected as a style reference when creating a new `html_page` task. |
| **Public Scheduled Tasks** (`/api/public/scheduled-tasks`) | **Unauthenticated** mirror for embedding. Adds `GET /runs/{rid}/html` and `GET /{task_id}/latest-html` for the `html_page` preview; the run result keeps `result.html` (only file paths are redacted). |
+| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` → `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "已改用规则抽取". The "保存到知识库" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). |
Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → Gateway.
+### Admin Leaderboard Snapshots (`/api/admin/users/leaderboard*`)
+
+The admin 用户排行榜 is served from **precomputed per-day snapshots** in `admin_leaderboard_daily_stats` (keyed by `(stat_date, settings_hash)`), not by aggregating raw rows on every request. A background scheduler inside the Gateway (`_leaderboard_snapshot_scheduler_loop` in `app/gateway/app.py`, singleton file-locked) ticks every `ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS` (default 300s): it prewarms recent days, then builds any `queued`/`error`/stale-`running` day. `GET /leaderboard` merges the per-day snapshots for the requested range. **Today** is rebuilt whenever its snapshot is older than `ADMIN_LEADERBOARD_TODAY_TTL_SECONDS` (default 300s); a past day, once `ready`, is treated as fresh forever.
+
+Two on-demand/automatic refresh paths layer on top:
+- **Manual "刷新当天"** — `POST /leaderboard/refresh-today` force-rebuilds *today's* snapshot **synchronously** (bounded by the same semaphore/timeout the scheduler uses), then returns the merged range. Lets an admin pull the latest real-time records into the stats without waiting for the TTL. Frontend: a "刷新当天" button on `AdminUserLeaderboardPage` → `useRefreshTodayLeaderboard` (`core/admin/hooks.ts`), which primes + invalidates the `["admin-activity","leaderboard"]` query.
+- **Daily finalize of the previous day** — once per day, on/after `ADMIN_LEADERBOARD_FINALIZE_HOUR` (local Beijing, default `1` = 凌晨 1 点), the scheduler tick force-recomputes *yesterday* a single time so runs/tool-calls that only landed near midnight are fully captured. The decision is the pure `deerflow.persistence.admin_stats.finalize.should_finalize_previous_day` (status/generated_at/now/hour) — the rebuild stamps `generated_at` to today, which self-debounces it for the rest of the day (no extra state file). Tests: `tests/test_leaderboard_finalize.py`.
+
### Scheduled Task Runtime
`ScheduledTaskService` (`deerflow/runtime/scheduler/service.py`) polls due tasks every `poll_interval_seconds` and executes each in its own throwaway thread, serialized by a shared `_agent_run_lock`.
@@ -273,11 +282,32 @@ Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` →
**HTML skills (tools the render step uses).** Two offline skills under `skills/public/` shape the generated HTML and are toggled in the skills settings page (enabled by default): `html-page-builder` (a self-contained inline-CSS design system — colors/cards/tables/KPI/responsive) and `html-charts` (inline-SVG charts that render in the no-`allow-scripts` preview iframe; optional local ECharts documented in its `assets/README.md`). A skill opts into HTML injection via `metadata.kind: html_page` in its SKILL.md frontmatter; `_collect_html_skill_instructions()` reads the bodies of **enabled** such skills and `build_generation_prompt(skill_instructions=…)` injects them, so the (raw-model) render step honors the user's skill selection. The data-gathering step and the render/repair/style passes all use the task's selected `model_name` (no silent fallback to the default model — and there is **no hardcoded GLM-5**; an unset selection falls back to config `models[0]`). To stop auxiliary LLM calls from leaking onto the default model, **`TitleMiddleware` is skipped when `is_scheduled_run` is set** (`_build_middlewares` in `lead_agent/agent.py`) — scheduled/HTML runs use throwaway threads whose titles are never shown, and `config.title.model_name` is usually unset (→ would call `models[0]`, not the chosen model). The completeness gate (`validate_html`) also requires a non-empty `
` and forbids external stylesheets, on top of the doctype/html/head/body + non-empty-body + no-script checks.
+### Knowledge Base (`packages/harness/deerflow/knowledge/`)
+
+Self-contained product knowledge base. **All knowledge capability lives in this one module** (harness layer, `deerflow.*` only — no `app.*` imports). Wired as `app.state.knowledge_service` in `deps.py` via `make_knowledge_service(sf, config.knowledge)` (returns `None` on the memory backend or when `knowledge.enabled=false` → router 503). Surfaced by `app/gateway/routers/knowledge.py` (`/api/knowledge`). Config: `config.yaml` → `knowledge` (`deerflow/config/knowledge_config.py`).
+
+**Two stores, kept in sync on every write:**
+- **DB mirror** — tables `knowledge_notes` / `knowledge_sources` / `knowledge_tags` / `knowledge_note_tags` (`knowledge/models.py`, registered in `persistence/models/__init__.py`, auto-created by `Base.metadata.create_all`; portable types `PortableLongText`/`BeijingDateTime`). Powers listing/filter/search/audit. Global/shared — no `user_id` column; `created_by`/`updated_by` are audit-only (resolved via `get_effective_user_id()`).
+- **Markdown vault** — obsidian-wiki compatible, default `{$DEER_FLOW_HOME}/data/knowledge/obsidian-vault`. `ObsidianWikiAdapter` (`obsidian_adapter.py`) writes wiki-capture frontmatter+body, and after **every** write syncs `index.md` / `log.md` (CAPTURE/INGEST/UPDATE/ARCHIVE/EXPORT) / `hot.md` / `.manifest.json` (source-hash dedup for `skip_existing`). `markdown_vault.py` handles frontmatter build/parse, slugging, wikilink extraction and **path-traversal-proof** resolution.
+
+**Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`.
+
+**Wikilinks & multi-note capture** (`knowledge.{related_notes_enabled,related_notes_limit,multi_note_enabled}`, all default-on; `project_name` defaults to `zncm`):
+- **Navigable `[[wikilinks]]`** — `service.resolve_wikilink(s)` maps a `[[target]]` to a knowledge note (by id → `vault_path` → title) or a vault meta page (`read_vault_page`, path-traversal-proof), else `missing`. `markdown_vault.normalize_wikilink_target` canonicalizes targets (drops `|alias`, slashes, `.md`). The frontend `MarkdownRenderer` renders non-numeric `[[...]]` as clickable links from the resolver map (numeric `[[123]]` stay citation badges); note links navigate, vault links open a `VaultPageDialog`, missing links are greyed.
+- **Real related notes only (no structural meta-links)** — `extractor.related_links()` now returns `[]`: notes no longer auto-link `index` / `_meta/taxonomy` / a project hub (they were noise in every `## Related`). On capture, `service._find_related_notes` cross-links **only by shared *distinctive* tags** (project/`knowledge-base` defaults excluded; free-text title/summary matching was removed because CJK tokenization linked unrelated notes via generic words like 助手/知识图谱) and `_inject_related_notes` writes real `[[Title]]` links into `## Related` (plus note→note `related` graph edges); no distinctive tags → no related notes. The renderers omit the `## Related` section entirely when there are no related notes and no sources. The vault's own `index.md`/`_meta/taxonomy.md` still exist (obsidian scaffold) and remain resolvable for manually-written links. The router enriches `GET /notes/{id}` with `created_by_name`/`updated_by_name` (auth-provider email lookup) so the UI shows a person, not a UUID. The frontend `MarkdownRenderer` renders wiki-capture provenance markers `^[inferred]`/`^[ambiguous]` as small "推测"/"存疑" badges.
+- **LLM-extraction outcomes** — capture distinguishes three LLM outcomes: (1) **success** → declarative note(s); (2) **deliberate rejection** — the model returns `should_ingest:false` with no notes (greeting / meta chatter) → `extract_thread_llm_multi` raises `LLMRejectedIngest` and `capture_thread` raises `EmptyThreadError` (skip — nothing persisted, so worthless chats never enter the KB); (3) **genuine failure** — API error / non-JSON → returns `[]`, capture falls back to the rule-based verbatim extractor and lands the note as a **`draft`** (not `approved`), logs a WARNING, and returns `llm_fallback:true` (surfaced by the router). `llm_extractor` logs an actionable WARNING/INFO for each non-success path so extraction issues are diagnosable.
+- **Multi-note split** — `llm_extractor.extract_thread_llm_multi` lets one conversation distill into 1–4 cross-linked notes (`notes:[…]` JSON array; legacy single-object output still accepted). `capture_thread` persists each, sibling-cross-links them by title, and returns `{note, created, notes:[…]}`. Gated by `multi_note_enabled` (false → first note only). Tests: `tests/test_knowledge_wikilinks.py`.
+
+**Phases 2–4 (all config-gated, defaults safe):**
+- **Phase 2 — vector RAG**: `knowledge.embedding.{enabled,base_url,api_key,model,...}` enables an OpenAI-compatible embedding endpoint; notes are chunked + embedded on write (`knowledge_embeddings`), `POST /search` supports `mode=keyword|vector|hybrid` (vector degrades to keyword when unconfigured). `KnowledgeRagMiddleware` (lead-agent chain) injects a transient `` recall before the model call, gated **per-conversation** by the `knowledge_rag_enabled` runtime flag (frontend chat toggle) falling back to global `knowledge.rag_enabled`. `POST /reindex` rebuilds the index.
+- **Phase 3 — governance**: `sensitive.py` redacts secrets/PII before persistence (`sensitive_redaction_enabled`); every update snapshots `knowledge_note_versions` (`GET /notes/{id}/versions`); `GET /duplicates` + `POST /merge` for dedup; background `AutoIngestQueue` (wired in `deps.py`, started/stopped in the gateway lifespan) auto-sediments finished conversations **off the chat path** when `auto_ingest_enabled` — enqueued from `KnowledgeRagMiddleware.after_agent`, value-judged by min-chars + `skip_existing`, landing as `draft`.
+- **Phase 4 — graph**: LLM extraction also yields entities/relations → `knowledge_entities`/`knowledge_relations`; `exporter.build_graph` adds entity nodes + note→entity/entity→entity edges (reflected in `graph.html`/`graph.json`); `GET /graph` returns product graph data.
+
### Custom Agent Persistence
Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.agents`). `agents.id` is the stable runtime id and the filesystem directory name under `.deer-flow/agents/{id}/`; `agents.name` is display-only, can contain Chinese characters, and is not unique. `user_id IS NULL` marks built-in agents, user-owned agents are visible only to their owner unless `published = true`, and update/delete operations require ownership. The runtime accepts `agent_id` in context/configurable, and non-default `assistant_id` values are treated as agent ids after visibility checks.
-**Listing cache + pagination** — to keep `GET /api/agents` fast, the gallery's per-card extras (skills + tool_groups) are denormalized into two cache tables instead of reading every agent's `config.yaml` on each request: `agent_skills` (relational, one row per (agent, skill) with `position`) and `agent_extras` (one row per agent: `tool_groups` JSON + `model` + `skills_synced_at`). `config.yaml` remains authoritative for the agent runtime; these tables are a read cache, write-through on create/update and backfilled from disk by `_sync_legacy_agents` (mtime-vs-`skills_synced_at` drift check). The list endpoint enriches via `AgentStore.get_extras_for([ids])` (two batched `IN` queries). Both tables are auto-created by `Base.metadata.create_all` (no migration needed). `GET /api/agents` supports **opt-in server-side pagination**: `scope` (`mine` = owned + favorited / `square` = built-in/published public), `page`, `page_size`, returning `{agents, total, page, page_size}`; omitting `scope`/`page` preserves the legacy "return everything" behavior. Name `search` matches the raw OR desensitized name (the frontend's static term→abbrev map is mirrored server-side in `deerflow/config/agent_name_desensitize.py`); `tag_ids` is an SQL OR-filter. `POST /api/agents/private-square/list` takes the same pagination params. Repository entry point: `AgentStore.list_paginated(...)`.
+**Listing cache + pagination** — to keep `GET /api/agents` fast, the gallery's per-card extras (skills + tool_groups) are denormalized into two cache tables instead of reading every agent's `config.yaml` on each request: `agent_skills` (relational, one row per (agent, skill) with `position`) and `agent_extras` (one row per agent: `tool_groups` JSON + `model` + `skills_synced_at`). `config.yaml` remains authoritative for the agent runtime; these tables are a read cache, write-through on create/update and backfilled from disk by `_sync_legacy_agents` (mtime-vs-`skills_synced_at` drift check). The list endpoint enriches via `AgentStore.get_extras_for([ids])` (two batched `IN` queries). Both tables are auto-created by `Base.metadata.create_all` (no migration needed). `GET /api/agents` supports **opt-in server-side pagination**: `scope` (`mine` = owned + favorited / `square` = built-in/published public), `page`, `page_size`, returning `{agents, total, page, page_size}`; omitting `scope`/`page` preserves the legacy "return everything" behavior. Name `search` matches the raw OR desensitized name (the frontend's static term→abbrev map is mirrored server-side in `deerflow/config/agent_name_desensitize.py`); `tag_ids` is an SQL OR-filter; `owner` keeps only agents published by that user id (the card's 发布者 — built-in agents never match), powering the gallery's click-发布者-to-filter. `POST /api/agents/private-square/list` takes the same pagination params (incl. `owner`). Repository entry point: `AgentStore.list_paginated(...)`. Tests: `tests/test_agent_listing_cache.py::test_list_paginated_owner_filter_keeps_one_publisher`.
**Homepage selector (agent picker on the new-chat page)** — single admin-curated list backed by `agents.featured_order` (nullable BIGINT). NULL = not featured. Only `published = true` agents can be featured (router enforces on writes); any published agent is eligible regardless of ownership. Cleared with `order: null`. The list rendered to end users is `WHERE featured_order IS NOT NULL` + visibility-filter, sorted by `featured_order ASC`. Admins reorder by sending the desired ids to `PUT /api/agents/featured/order` (assigns indices 0..N-1). BIGINT is used rather than INT so the management UI can seed initial values from `Date.now()` (overflows INT on MySQL).
@@ -289,6 +319,16 @@ Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` /
- **`PortableLongText`** — a large text column: maps to MySQL `LONGTEXT` (4 GB) and plain `TEXT` on SQLite/Postgres (already unbounded). Use for big JSON/markdown blobs that would overflow MySQL's 64 KB `TEXT` cap — e.g. `ai_writing_sessions.transcript`.
- **`BeijingDateTime`** — stores timestamps as Beijing (UTC+8) wall-clock time. The app records time with `datetime.now(UTC)`, but MySQL `DATETIME` / SQLite have no timezone and persist whatever wall clock the driver serializes — so a raw `DateTime` column would store UTC and read 8 hours behind Beijing. `BeijingDateTime` converts UTC→Beijing on bind and re-attaches `+08:00` on result, so the database holds Beijing time while Python still gets timezone-aware datetimes (instant comparisons stay correct, including in WHERE clauses). PostgreSQL `timestamptz` stores the instant unambiguously, so values pass through unchanged there. Uses a fixed UTC+8 offset (China has no DST), so no `tzdata` dependency.
+### Schema Migrations / Manual Column-Add SQL (`scripts/sql/`)
+
+Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup only `Base.metadata.create_all` runs, which creates **new tables** but never patches **existing** ones (no new columns, no new indexes). So a column added to an ORM model appears automatically on a brand-new deployment, but is **missing on any existing MySQL database** — and code reading/writing that column will error there until the column is added by hand.
+
+**Convention (required):** whenever code adds a field to an existing table (a new `mapped_column` on an ORM model), you must, in the same change:
+1. Keep the matching alembic version file under `persistence/migrations/versions/` (guarded with `_has_table`/`_has_column`), and
+2. Append the idempotent `ADD COLUMN` statement to **`scripts/sql/column_additions_mysql.sql`**, with a definition matching the ORM column exactly (type, length, nullability).
+
+Operators upgrading an existing DB run the whole file (`USE ;` then execute) — it is idempotent (existing columns are skipped via `information_schema.columns`), so it is safe to re-run. Index-only changes go in the sibling `scripts/sql/leaderboard_indexes_mysql.sql` (same idempotent pattern).
+
### Sandbox System (`packages/harness/deerflow/sandbox/`)
**Interface**: Abstract `Sandbox` with `execute_command`, `read_file`, `write_file`, `list_dir`
@@ -358,6 +398,7 @@ Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` /
- **Loading**: `load_skills()` recursively scans `skills/{public,custom}` for `SKILL.md`, parses metadata, and reads enabled state from extensions_config.json
- **Injection**: Enabled skills listed in agent system prompt with container paths
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
+- **Chinese name (`name_zh`)**: a Chinese display name stored in the **`skills` DB table** (`SkillRow.name_zh`, `String(128)`, migration `20260609_01`) — never written to the on-disk SKILL.md, so it works for built-in skills too. Threaded through `SkillStore` create/ensure_legacy/update/update_admin exactly like `detail`; surfaced on `SkillResponse.name_zh` from the ownership record in `_skill_to_response`. Authorization mirrors `detail`: custom skill → owner or admin; built-in/public → admin only (row materialized on demand via `ensure_legacy`). Endpoints (in `app/gateway/routers/skills.py`): `PUT /api/skills/{name}/name-zh` (set/clear), `POST /api/skills/{name}/name-zh/generate` (LLM generates one), `POST /api/skills/name-zh/generate-missing` (一键补充: **admin-only** bulk-generate for every skill that lacks one, bounded-concurrency `asyncio.gather`). **Optional model selection**: both generate endpoints accept an optional JSON body `{model_name}` — an explicit name wins, else the curator model setting, else the main model (`_resolve_skill_name_zh_model_name`). **Model-error surfacing**: `_generate_skill_name_zh` raises `SkillNameZhModelError` when the model itself fails (build/timeout/API error) — distinct from the model returning an unusable name (→ `None`). The single endpoint maps that to `502 大模型报错,无法补全技能中文名:…`; the bulk endpoint fails fast on a model-build error (502) and otherwise carries a representative `model_error` string in `SkillNameZhBulkResponse` so the UI can say "N 个因大模型报错失败". The frontend (`skill-settings-page.tsx` list + `skill-detail-dialog.tsx` editor, plus the agent skill pickers/badges in `NewAgentPage.tsx` + `agent-card.tsx`) shows `name_zh || name`, offers per-skill "AI 生成" plus an **admin-only** header "一键补充中文名" button, and a compact model dropdown (`components/workspace/model-name-select.tsx`, default = configured model) beside each action.
#### Skill Curator (`packages/harness/deerflow/skills/curator.py`)
diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py
index 5eb1accc..78bcde73 100644
--- a/offline-backend-20260512/backend/app/gateway/app.py
+++ b/offline-backend-20260512/backend/app/gateway/app.py
@@ -6,6 +6,8 @@ import sys
import threading
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
+from datetime import datetime
+from time import perf_counter
from dotenv import load_dotenv
from fastapi import FastAPI
@@ -30,12 +32,14 @@ from app.gateway.routers import (
artifacts,
assistants_compat,
auth,
+ browser_context,
business_mapping,
channels,
checkpoint_migration,
feedback,
html_page_favorites,
intent,
+ knowledge,
llm_metrics,
mcp,
memory,
@@ -45,6 +49,7 @@ from app.gateway.routers import (
parallel_agent_tasks,
parallel_agents,
public_embed,
+ public_leaderboard,
public_scheduled_tasks,
recommend,
recommended_questions,
@@ -469,6 +474,91 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
60.0,
)
+# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
+# *previous* day's snapshot once, so records that landed near midnight are fully
+# captured. Default 1 → "每天凌晨 1 点重新统计前一天".
+_LEADERBOARD_FINALIZE_HOUR = min(
+ 23,
+ _env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
+)
+
+
+def _leaderboard_scheduler_sql_for_log(stmt) -> str:
+ try:
+ return str(stmt.compile(compile_kwargs={"literal_binds": True}))
+ except Exception:
+ return str(stmt)
+
+
+async def _execute_leaderboard_scheduler_query(session, step: str, stmt, **context):
+ sql = _leaderboard_scheduler_sql_for_log(stmt)
+ logger.info(
+ "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ sql,
+ )
+ start = perf_counter()
+ try:
+ result = await session.execute(stmt)
+ except Exception:
+ logger.exception(
+ "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ (perf_counter() - start) * 1000,
+ )
+ raise
+ logger.info(
+ "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ (perf_counter() - start) * 1000,
+ )
+ return result
+
+
+async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql: str = "", **context):
+ logger.info(
+ "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ sql,
+ )
+ start = perf_counter()
+ try:
+ result = await operation()
+ except Exception:
+ logger.exception(
+ "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ (perf_counter() - start) * 1000,
+ )
+ raise
+ logger.info(
+ "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ (perf_counter() - start) * 1000,
+ )
+ return result
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
@@ -496,21 +586,27 @@ async def _leaderboard_snapshot_due_dates(
cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
async with session_factory() as session:
- rows = (
- await session.execute(
- select(AdminLeaderboardDailyStatRow.stat_date)
- .where(
- AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
- or_(
- AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
- and_(
- AdminLeaderboardDailyStatRow.status == "running",
- AdminLeaderboardDailyStatRow.updated_at < cutoff,
- ),
+ stmt = (
+ select(AdminLeaderboardDailyStatRow.stat_date)
+ .where(
+ AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
+ or_(
+ AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
+ and_(
+ AdminLeaderboardDailyStatRow.status == "running",
+ AdminLeaderboardDailyStatRow.updated_at < cutoff,
),
- )
- .order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
- .limit(limit)
+ ),
+ )
+ .order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
+ .limit(limit)
+ )
+ rows = (
+ await _execute_leaderboard_scheduler_query(
+ session,
+ "scheduler.due_dates",
+ stmt,
+ settings_hash=settings_hash,
)
).all()
return [str(row[0]) for row in rows]
@@ -530,9 +626,15 @@ async def _claim_leaderboard_snapshot(
from deerflow.persistence.types import BEIJING_TZ
async with session_factory() as session:
- row = await session.get(
- AdminLeaderboardDailyStatRow,
- {"stat_date": stat_date, "settings_hash": settings_hash},
+ row = await _time_leaderboard_scheduler_db_operation(
+ "scheduler.claim.get",
+ lambda: session.get(
+ AdminLeaderboardDailyStatRow,
+ {"stat_date": stat_date, "settings_hash": settings_hash},
+ ),
+ sql="SELECT admin_leaderboard_daily_stats by primary key",
+ stat_date=stat_date,
+ settings_hash=settings_hash,
)
if row is None or _snapshot_is_fresh(row, stat_date):
return False
@@ -548,10 +650,68 @@ async def _claim_leaderboard_snapshot(
row.message = "排行榜后台统计任务已认领。"
row.error = None
row.updated_at = datetime.now(UTC)
- await session.commit()
+ await _time_leaderboard_scheduler_db_operation(
+ "scheduler.claim.commit",
+ lambda: session.commit(),
+ sql="COMMIT",
+ stat_date=stat_date,
+ settings_hash=settings_hash,
+ )
return True
+async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash: str) -> bool:
+ """Once per day after ``_LEADERBOARD_FINALIZE_HOUR``, force-recompute yesterday.
+
+ A ``ready`` past-day snapshot is otherwise treated as fresh forever and never
+ rebuilt. But the previous day's snapshot is usually generated before midnight,
+ so it can miss runs/tool-calls that only land in the DB around the day
+ boundary. This re-queues yesterday (``force=True``) a single time; the rebuild
+ stamps ``generated_at`` to today, which stops it from re-firing for the rest of
+ the day.
+ """
+ from datetime import timedelta
+
+ from app.gateway.routers.admin_users import (
+ _as_beijing,
+ _queue_daily_leaderboard_snapshots,
+ )
+ from deerflow.persistence.admin_stats.finalize import should_finalize_previous_day
+ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
+ from deerflow.persistence.types import BEIJING_TZ
+
+ now = datetime.now(BEIJING_TZ)
+ yesterday = (now.date() - timedelta(days=1)).isoformat()
+ async with session_factory() as session:
+ row = await session.get(
+ AdminLeaderboardDailyStatRow,
+ {"stat_date": yesterday, "settings_hash": settings_hash},
+ )
+ generated_at = _as_beijing(row.generated_at) if (row is not None and row.generated_at is not None) else None
+ if not should_finalize_previous_day(
+ status=(row.status if row is not None else None),
+ generated_at=generated_at,
+ now=now,
+ finalize_hour=_LEADERBOARD_FINALIZE_HOUR,
+ ):
+ return False
+ queued = await _queue_daily_leaderboard_snapshots(
+ session_factory,
+ [yesterday],
+ settings_hash,
+ force=True,
+ message="每日凌晨重新统计前一天完整数据,确保全部记录纳入统计。",
+ )
+ if queued:
+ logger.info(
+ "Leaderboard snapshot finalize: queued previous day %s for full recompute (hash=%s)",
+ yesterday,
+ settings_hash[:10],
+ )
+ return True
+ return False
+
+
async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
from app.gateway.routers.admin_users import (
_daily_job_key,
@@ -576,6 +736,12 @@ async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
message="排行榜后台统计任务定期预热/补算队列。",
)
+ # 每天凌晨(默认 1 点)后强制重算"前一天"的完整快照一次,覆盖跨午夜才落库的记录。
+ try:
+ await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
+ except Exception:
+ logger.exception("Leaderboard snapshot finalize check failed (non-fatal)")
+
due_dates = await _leaderboard_snapshot_due_dates(
session_factory,
settings_hash=settings_hash,
@@ -1115,6 +1281,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Uploads API is mounted at /api/threads/{thread_id}/uploads
app.include_router(uploads.router)
+ # Optional Chrome-extension page capture API is mounted at /api/browser-context
+ app.include_router(browser_context.router)
+
# Thread cleanup API is mounted at /api/threads/{thread_id}
app.include_router(threads.router)
@@ -1162,10 +1331,16 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# surface results without a DeerFlow login.
app.include_router(public_scheduled_tasks.router)
+ # Public read-only user leaderboard publish page (/api/public/leaderboard)
+ app.include_router(public_leaderboard.router)
+
# Public embed Q&A status polling (/api/public/embed/*)
app.include_router(public_embed.router)
app.include_router(html_page_favorites.router)
+ # Product knowledge base API is mounted at /api/knowledge
+ app.include_router(knowledge.router)
+
# System settings API is mounted at /api/system-settings
app.include_router(system_settings.router)
@@ -1174,6 +1349,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Admin window into other users' threads / messages / memory
app.include_router(admin_users.router)
+ app.include_router(admin_users.self_router)
# Admin tool / skill call metrics audit + export
app.include_router(tool_metrics.router)
diff --git a/offline-backend-20260512/backend/app/gateway/auth_middleware.py b/offline-backend-20260512/backend/app/gateway/auth_middleware.py
index ff3bb86c..ae124dfb 100644
--- a/offline-backend-20260512/backend/app/gateway/auth_middleware.py
+++ b/offline-backend-20260512/backend/app/gateway/auth_middleware.py
@@ -20,6 +20,7 @@ from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, i
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
from app.gateway.authz import _ALL_PERMISSIONS, AuthContext
from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token
+from app.gateway.proxy_path import app_relative_path
from deerflow.runtime.user_context import reset_current_user, set_current_user
# Paths that never require authentication.
@@ -84,7 +85,9 @@ class AuthMiddleware(BaseHTTPMiddleware):
super().__init__(app)
async def dispatch(self, request: Request, call_next: Callable) -> Response:
- if _is_public(request.url.path):
+ # Match the application-relative path so a reverse-proxy prefix
+ # (e.g. /xxx/api/public/...) does not defeat the public whitelist.
+ if _is_public(app_relative_path(request)):
return await call_next(request)
if is_auth_disabled():
diff --git a/offline-backend-20260512/backend/app/gateway/csrf_middleware.py b/offline-backend-20260512/backend/app/gateway/csrf_middleware.py
index 07b46047..c5494277 100644
--- a/offline-backend-20260512/backend/app/gateway/csrf_middleware.py
+++ b/offline-backend-20260512/backend/app/gateway/csrf_middleware.py
@@ -8,6 +8,7 @@ import secrets
from collections.abc import Callable
from app.gateway.auth.disabled_mode import is_auth_disabled
+from app.gateway.proxy_path import app_relative_path
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.responses import JSONResponse
@@ -37,7 +38,9 @@ def should_check_csrf(request: Request) -> bool:
if request.method not in ("POST", "PUT", "DELETE", "PATCH"):
return False
- path = request.url.path
+ # Match the application-relative path so a reverse-proxy prefix
+ # (e.g. /xxx/api/v1/auth/me) does not defeat these whitelists.
+ path = app_relative_path(request)
stripped = path.rstrip("/")
# Exempt /api/v1/auth/me endpoint
if stripped == "/api/v1/auth/me":
@@ -56,6 +59,10 @@ _AUTH_EXEMPT_PATHS: frozenset[str] = frozenset(
{
"/api/v1/auth/login/local",
"/api/v1/auth/login/username",
+ # Token-exchange login is sessionless on first call (no CSRF cookie yet),
+ # exactly like the other login endpoints — mirror auth_middleware's
+ # _PUBLIC_EXACT_PATHS, which already whitelists it.
+ "/api/v1/auth/login/token",
"/api/v1/auth/logout",
"/api/v1/auth/register",
"/api/v1/auth/initialize",
@@ -69,8 +76,10 @@ def is_auth_endpoint(request: Request) -> bool:
"""Check if the request is to an auth endpoint.
Auth endpoints don't need CSRF validation on first call (no token).
+ Matches the application-relative path so a reverse-proxy prefix does not
+ defeat the whitelist.
"""
- return request.url.path.rstrip("/") in _AUTH_EXEMPT_PATHS
+ return app_relative_path(request).rstrip("/") in _AUTH_EXEMPT_PATHS
def _has_bearer_auth(request: Request) -> bool:
@@ -82,6 +91,22 @@ def _has_bearer_auth(request: Request) -> bool:
return scheme.lower() == "bearer" and bool(token.strip())
+def _has_cookie_session(request: Request) -> bool:
+ """Return True when the request carries a cookie-based session.
+
+ Double-Submit-Cookie CSRF only protects requests the browser authenticates
+ **automatically via cookies** — that is the exact surface a cross-site
+ forgery can ride. A request with no ``access_token`` session cookie has no
+ session for an attacker to exploit, so enforcing CSRF on it is both
+ pointless and harmful: a freshly-loaded browser (e.g. one that just cleared
+ its cache/cookies) hitting the login endpoint would otherwise be rejected
+ with "CSRF token missing" before it can even sign in. Such an
+ unauthenticated request, if forged, is rejected by the auth layer (401)
+ anyway.
+ """
+ return bool(request.cookies.get("access_token"))
+
+
class CSRFMiddleware(BaseHTTPMiddleware):
"""Middleware that implements CSRF protection using Double Submit Cookie pattern."""
@@ -95,8 +120,11 @@ class CSRFMiddleware(BaseHTTPMiddleware):
return await call_next(request)
# Double-submit-cookie CSRF applies to cookie-based browser sessions.
- # API clients using explicit Bearer tokens are not CSRF-prone.
- if should_check_csrf(request) and not _is_auth and not _has_bearer_auth(request):
+ # API clients using explicit Bearer tokens are not CSRF-prone, and a
+ # request with no session cookie at all (e.g. a fresh browser signing
+ # in) has no session to forge — only enforce when an ``access_token``
+ # cookie is actually present.
+ if should_check_csrf(request) and not _is_auth and not _has_bearer_auth(request) and _has_cookie_session(request):
cookie_token = request.cookies.get(CSRF_COOKIE_NAME)
header_token = request.headers.get(CSRF_HEADER_NAME)
diff --git a/offline-backend-20260512/backend/app/gateway/deps.py b/offline-backend-20260512/backend/app/gateway/deps.py
index cfc86437..62ae4daa 100644
--- a/offline-backend-20260512/backend/app/gateway/deps.py
+++ b/offline-backend-20260512/backend/app/gateway/deps.py
@@ -143,6 +143,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.persistence.ai_writing_sessions import (
make_ai_writing_session_store,
+ )
+ from deerflow.persistence.ai_writing_sessions import (
register_default_store as _register_ai_writing_default_store,
)
@@ -159,6 +161,25 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.memory_config_store = make_user_memory_config_store(sf)
+ from deerflow.knowledge import make_knowledge_service
+
+ app.state.knowledge_service = make_knowledge_service(sf, config.knowledge)
+
+ # Background auto-sedimentation queue (phase 3) — drains finished
+ # conversations off the chat path. Worker only does work when
+ # ``knowledge.auto_ingest_enabled`` is set; the queue is always wired so
+ # the middleware can enqueue without a hard dependency.
+ from deerflow.knowledge.auto_ingest import AutoIngestQueue, set_auto_ingest_queue
+
+ if app.state.knowledge_service is not None:
+ _auto_ingest_queue = AutoIngestQueue(
+ lambda: getattr(app.state, "knowledge_service", None),
+ min_chars=config.knowledge.auto_ingest_min_chars,
+ )
+ _auto_ingest_queue.start()
+ set_auto_ingest_queue(_auto_ingest_queue)
+ app.state.knowledge_auto_ingest_queue = _auto_ingest_queue
+
from deerflow.persistence.user_prompts import make_user_prompt_store
app.state.user_prompt_store = make_user_prompt_store(sf)
@@ -189,6 +210,8 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.persistence.roundtable_jobs import (
make_roundtable_job_store,
+ )
+ from deerflow.persistence.roundtable_jobs import (
register_default_store as _register_roundtable_job_default_store,
)
@@ -246,6 +269,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.persistence.notifications import set_notification_store
set_notification_store(None)
+ queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
+ if queue is not None:
+ from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
+
+ await queue.stop()
+ set_auto_ingest_queue(None)
await close_engine()
@@ -380,6 +409,14 @@ def get_memory_config_store(request: Request):
return val
+def get_knowledge_service(request: Request):
+ """Return the knowledge base service (503 when disabled / memory backend)."""
+ val = getattr(request.app.state, "knowledge_service", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Knowledge base not available")
+ return val
+
+
def get_run_context(request: Request) -> RunContext:
"""Build a :class:`RunContext` from ``app.state`` singletons.
diff --git a/offline-backend-20260512/backend/app/gateway/langgraph_auth.py b/offline-backend-20260512/backend/app/gateway/langgraph_auth.py
index 0929b193..581a8020 100644
--- a/offline-backend-20260512/backend/app/gateway/langgraph_auth.py
+++ b/offline-backend-20260512/backend/app/gateway/langgraph_auth.py
@@ -40,6 +40,13 @@ def _check_csrf(request) -> None:
# CSRF protection is for cookie-auth browser flows.
return
+ # No cookie-based session → nothing for a cross-site forgery to ride, so
+ # CSRF is moot. Enforcing it here would reject a freshly-loaded browser
+ # (just cleared its cookies) before it can authenticate; such a request is
+ # rejected as unauthenticated (401) below anyway.
+ if not request.cookies.get("access_token"):
+ return
+
cookie_token = request.cookies.get("csrf_token")
header_token = request.headers.get("x-csrf-token")
diff --git a/offline-backend-20260512/backend/app/gateway/proxy_path.py b/offline-backend-20260512/backend/app/gateway/proxy_path.py
new file mode 100644
index 00000000..22942978
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/proxy_path.py
@@ -0,0 +1,39 @@
+"""Resolve the application-relative request path behind a reverse proxy.
+
+A reverse proxy may forward requests under a path prefix, e.g.
+``/xxx/api/v1/auth/login/token``. FastAPI strips that prefix for **routing**
+when it is told about it via the ASGI ``root_path`` (uvicorn ``--root-path`` /
+``--proxy-headers``, or ``X-Forwarded-Prefix`` from the proxy) — which is why
+the route still matches and the request reaches a handler.
+
+Starlette middleware, however, runs *before* routing and sees the full,
+prefixed ``request.url.path``. Middlewares that match that raw path against
+fixed whitelists — the CSRF-exempt auth paths and the AuthMiddleware public
+paths — then mis-match, so login and ``/api/public/*`` endpoints get wrongly
+rejected (``CSRF token missing`` / ``401``) on a prefixed deployment.
+
+``app_relative_path`` returns the path with the proxy prefix removed so that
+whitelist matching is prefix-agnostic. It is a no-op when no prefix is present
+(direct deployment), and tolerates the prefix already being stripped from the
+path (it only strips when the path actually starts with the prefix).
+"""
+
+from fastapi import Request
+
+
+def app_relative_path(request: Request) -> str:
+ """Return ``request.url.path`` with any reverse-proxy prefix removed.
+
+ Prefix source order: ASGI ``root_path`` (set by the server when the proxy
+ forwards it), then the ``X-Forwarded-Prefix`` header (proxies that add a
+ prefix without informing the ASGI server). Returns ``"/"`` for an empty
+ result so downstream ``rstrip("/")`` / ``startswith`` checks stay sane.
+ """
+ path = request.url.path or "/"
+ prefix = (request.scope.get("root_path") or request.headers.get("x-forwarded-prefix") or "").rstrip("/")
+ if prefix and path.startswith(prefix):
+ stripped = path[len(prefix):]
+ if not stripped:
+ return "/"
+ return stripped if stripped.startswith("/") else "/" + stripped
+ return path
diff --git a/offline-backend-20260512/backend/app/gateway/routers/admin_users.py b/offline-backend-20260512/backend/app/gateway/routers/admin_users.py
index 753471ea..ead24625 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/admin_users.py
+++ b/offline-backend-20260512/backend/app/gateway/routers/admin_users.py
@@ -19,8 +19,11 @@ import logging
import os
import re
from collections.abc import AsyncIterator
+from contextvars import ContextVar
from difflib import SequenceMatcher
from hashlib import sha256
+from time import perf_counter
+from typing import Any
from datetime import UTC, datetime, timedelta, time as datetime_time
@@ -56,6 +59,7 @@ from deerflow.persistence.user.model import UserRow
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
+self_router = APIRouter(prefix="/api/users/me", tags=["user-qa"])
_LEADERBOARD_MAX_CONCURRENT = max(
1,
@@ -99,6 +103,119 @@ _LEADERBOARD_QUEUE_ON_REQUEST_ENABLED = os.getenv(
"off",
}
_leaderboard_semaphore = asyncio.Semaphore(_LEADERBOARD_MAX_CONCURRENT)
+_LEADERBOARD_TRACE_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar(
+ "leaderboard_trace_context",
+ default=None,
+)
+
+
+def _leaderboard_log_context(**context: object) -> dict[str, object] | None:
+ current = _LEADERBOARD_TRACE_CONTEXT.get()
+ merged: dict[str, object] = dict(current or {})
+ for key, value in context.items():
+ if value is None or value == "":
+ continue
+ merged[key] = value
+ return merged or None
+
+
+def _leaderboard_sql_for_log(stmt) -> str:
+ try:
+ return str(stmt.compile(compile_kwargs={"literal_binds": True}))
+ except Exception:
+ return str(stmt)
+
+
+async def _execute_leaderboard_query(session, step: str, stmt, **context: object):
+ log_context = _leaderboard_log_context(**context)
+ if not log_context:
+ return await session.execute(stmt)
+
+ started_at = datetime.now(BEIJING_TZ).isoformat()
+ sql = _leaderboard_sql_for_log(stmt)
+ logger.info(
+ "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
+ step,
+ log_context.get("stat_date") or log_context.get("stat_dates") or "",
+ log_context.get("job_key") or "",
+ log_context.get("settings_hash") or "",
+ started_at,
+ sql,
+ )
+ start = perf_counter()
+ try:
+ result = await session.execute(stmt)
+ except Exception:
+ elapsed_ms = (perf_counter() - start) * 1000
+ logger.exception(
+ "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
+ step,
+ log_context.get("stat_date") or log_context.get("stat_dates") or "",
+ log_context.get("job_key") or "",
+ log_context.get("settings_hash") or "",
+ elapsed_ms,
+ )
+ raise
+
+ elapsed_ms = (perf_counter() - start) * 1000
+ logger.info(
+ "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ step,
+ log_context.get("stat_date") or log_context.get("stat_dates") or "",
+ log_context.get("job_key") or "",
+ log_context.get("settings_hash") or "",
+ datetime.now(BEIJING_TZ).isoformat(),
+ elapsed_ms,
+ )
+ return result
+
+
+async def _time_leaderboard_db_operation(
+ step: str,
+ operation,
+ *,
+ sql: str = "",
+ **context: object,
+):
+ log_context = _leaderboard_log_context(**context)
+ if not log_context:
+ return await operation()
+
+ logger.info(
+ "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
+ step,
+ log_context.get("stat_date") or log_context.get("stat_dates") or "",
+ log_context.get("job_key") or "",
+ log_context.get("settings_hash") or "",
+ datetime.now(BEIJING_TZ).isoformat(),
+ sql,
+ )
+ start = perf_counter()
+ try:
+ result = await operation()
+ except Exception:
+ elapsed_ms = (perf_counter() - start) * 1000
+ logger.exception(
+ "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
+ step,
+ log_context.get("stat_date") or log_context.get("stat_dates") or "",
+ log_context.get("job_key") or "",
+ log_context.get("settings_hash") or "",
+ elapsed_ms,
+ )
+ raise
+
+ elapsed_ms = (perf_counter() - start) * 1000
+ logger.info(
+ "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ step,
+ log_context.get("stat_date") or log_context.get("stat_dates") or "",
+ log_context.get("job_key") or "",
+ log_context.get("settings_hash") or "",
+ datetime.now(BEIJING_TZ).isoformat(),
+ elapsed_ms,
+ )
+ return result
async def _require_admin(request: Request) -> None:
@@ -285,6 +402,13 @@ def _parse_datetime(value: str | None, fallback: datetime) -> datetime:
raise HTTPException(status_code=400, detail=f"Invalid datetime: {value}") from exc
+def _parse_optional_datetime(value: str | None) -> datetime | None:
+ if not value:
+ return None
+ parsed = _parse_datetime(value, _now())
+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=BEIJING_TZ)
+
+
def _resolve_range(days: int, since: str | None, until: str | None) -> tuple[datetime, datetime, int]:
end = _parse_datetime(until, _now())
start = _parse_datetime(since, end - timedelta(days=max(1, min(days, 365))))
@@ -325,7 +449,7 @@ def _bounds_for_stat_date(stat_date: str) -> tuple[datetime, datetime]:
async def _try_set_session_value(session, sql: str) -> None:
try:
- await session.execute(text(sql))
+ await _execute_leaderboard_query(session, "session_setting", text(sql))
except Exception:
logger.debug("Failed to apply analytics session setting: %s", sql, exc_info=True)
try:
@@ -334,13 +458,36 @@ async def _try_set_session_value(session, sql: str) -> None:
logger.debug("Failed to rollback after analytics session setting failure", exc_info=True)
+def _is_mariadb(dialect) -> bool:
+ """SQLAlchemy reports ``dialect.name == 'mysql'`` for MariaDB too."""
+ if getattr(dialect, "_is_mariadb", False) or getattr(dialect, "is_mariadb", False):
+ return True
+ version_info = getattr(dialect, "server_version_info", None) or ()
+ return any("mariadb" in str(part).lower() for part in version_info)
+
+
async def _set_analytics_read_timeout(session) -> None:
+ """Best-effort per-statement read timeout for the analytics queries.
+
+ The right knob differs by server, and some servers have none at all:
+ - MySQL >= 5.7.4: ``max_execution_time`` (milliseconds)
+ - MariaDB: ``max_statement_time`` (seconds)
+ - MySQL <= 5.6: no per-statement timeout variable — skip it entirely
+ (5.6 raises "Unknown system variable" otherwise)
+ - PostgreSQL: ``statement_timeout`` (milliseconds)
+ """
conn = await session.connection()
- dialect = conn.dialect.name
- if dialect == "mysql":
- await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
+ dialect = conn.dialect
+ name = dialect.name
+ if name == "mysql":
+ if _is_mariadb(dialect):
+ # MariaDB: max_statement_time is in SECONDS (fractional ok).
+ await _try_set_session_value(session, f"SET SESSION max_statement_time = {_LEADERBOARD_QUERY_TIMEOUT_MS / 1000}")
+ elif (getattr(dialect, "server_version_info", None) or ()) >= (5, 7, 4):
+ await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
+ # else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
- elif dialect == "postgresql":
+ elif name == "postgresql":
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
@@ -661,11 +808,15 @@ async def _query_user_leaderboard(
if user_id:
stmt = stmt.where(UserRow.id == user_id)
stmt = _apply_user_scope(stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
+ # 排行榜只统计真正产生过问答的用户;从未问答的账号(仅注册、零活跃)不计入榜单与人数。
+ # 个人分析(指定 user_id)不受此限制,仍可查看任意单个用户。
+ if user_id is None:
+ stmt = stmt.where(func.coalesce(run_agg.c.qa_count, 0) > 0)
stmt = stmt.order_by(
desc(func.coalesce(run_agg.c.run_count, 0) + func.coalesce(tool_agg.c.tool_call_count, 0)),
desc(func.coalesce(run_agg.c.total_tokens, 0)),
)
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, "users", stmt)).all()
return [
AdminLeaderboardUser(
user_id=r[0],
@@ -725,7 +876,7 @@ async def _query_call_rankings(
include_admins=include_admins,
include_scheduled=include_scheduled,
)
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, f"{field}_rankings", stmt)).all()
return [
AdminLeaderboardSkill(
name=r[0] or "",
@@ -791,7 +942,7 @@ async def _query_run_dimensions(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, f"{field}_dimensions", stmt)).all()
return [
AdminLeaderboardDimension(
name=r[0] or "",
@@ -848,7 +999,7 @@ async def _query_model_rankings(
stmt = stmt.where(LlmCallMetricRow.status != "error")
if not include_scheduled:
stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt())))
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, "model_rankings", stmt)).all()
return [
AdminLeaderboardDimension(
name=r[0] or "",
@@ -895,7 +1046,7 @@ async def _query_recent_questions(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, "recent_questions", stmt)).all()
questions: list[AdminLeaderboardQuestion] = []
for r in rows:
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
@@ -947,7 +1098,7 @@ async def _query_frequent_questions(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, "frequent_questions", stmt)).all()
groups: list[dict] = []
for r in rows:
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
@@ -1022,7 +1173,7 @@ async def _query_trends(
include_failed=include_failed,
include_scheduled=include_scheduled,
)
- rows = (await session.execute(stmt)).all()
+ rows = (await _execute_leaderboard_query(session, "trends", stmt)).all()
return [
AdminLeaderboardTrendPoint(
date=str(r[0]),
@@ -1071,8 +1222,8 @@ async def _query_overview(
total_users_stmt = select(func.count(UserRow.id))
total_users_stmt = _apply_user_scope(total_users_stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins)
- total_users = 1 if user_id else (await session.execute(total_users_stmt)).scalar_one()
- run_row = (await session.execute(
+ total_users = 1 if user_id else (await _execute_leaderboard_query(session, "overview.total_users", total_users_stmt)).scalar_one()
+ run_summary_stmt = (
select(
func.count(RunRow.run_id),
_question_count_expr(),
@@ -1080,16 +1231,19 @@ async def _query_overview(
func.count(func.distinct(RunRow.user_id)),
func.sum(case((RunRow.status == "error", 1), else_=0)),
).where(*run_where)
- )).one()
- thread_count = (await session.execute(select(func.count(ThreadMetaRow.thread_id)).where(*thread_where))).scalar_one()
- tool_row = (await session.execute(
+ )
+ run_row = (await _execute_leaderboard_query(session, "overview.run_summary", run_summary_stmt)).one()
+ thread_count_stmt = select(func.count(ThreadMetaRow.thread_id)).where(*thread_where)
+ thread_count = (await _execute_leaderboard_query(session, "overview.thread_count", thread_count_stmt)).scalar_one()
+ tool_summary_stmt = (
select(
func.count(ToolCallMetricRow.id),
func.sum(case((func.nullif(ToolCallMetricRow.skill_name, "").is_not(None), 1), else_=0)),
)
.outerjoin(RunRow, RunRow.run_id == ToolCallMetricRow.run_id)
.where(*tool_where)
- )).one()
+ )
+ tool_row = (await _execute_leaderboard_query(session, "overview.tool_summary", tool_summary_stmt)).one()
return AdminLeaderboardOverview(
total_users=int(total_users or 0),
@@ -1579,13 +1733,12 @@ async def _load_daily_snapshot_rows(
if not stat_dates:
return {}
async with session_factory() as session:
+ stmt = select(AdminLeaderboardDailyStatRow).where(
+ AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
+ AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
+ )
rows = (
- await session.execute(
- select(AdminLeaderboardDailyStatRow).where(
- AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
- AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
- )
- )
+ await _execute_leaderboard_query(session, "snapshot_rows.load", stmt)
).scalars().all()
return {row.stat_date: row for row in rows}
@@ -1603,9 +1756,15 @@ async def _store_daily_snapshot(
) -> None:
async with session_factory() as session:
now = datetime.now(UTC)
- row = await session.get(
- AdminLeaderboardDailyStatRow,
- {"stat_date": stat_date, "settings_hash": settings_hash},
+ row = await _time_leaderboard_db_operation(
+ "snapshot.store.get",
+ lambda: session.get(
+ AdminLeaderboardDailyStatRow,
+ {"stat_date": stat_date, "settings_hash": settings_hash},
+ ),
+ sql="SELECT admin_leaderboard_daily_stats by primary key",
+ stat_date=stat_date,
+ settings_hash=settings_hash,
)
if row is None:
row = AdminLeaderboardDailyStatRow(
@@ -1630,7 +1789,13 @@ async def _store_daily_snapshot(
if generated_at is not None:
row.generated_at = generated_at
row.updated_at = now
- await session.commit()
+ await _time_leaderboard_db_operation(
+ "snapshot.store.commit",
+ lambda: session.commit(),
+ sql="COMMIT",
+ stat_date=stat_date,
+ settings_hash=settings_hash,
+ )
async def _queue_daily_leaderboard_snapshots(
@@ -1646,12 +1811,17 @@ async def _queue_daily_leaderboard_snapshots(
return queued
async with session_factory() as session:
now = datetime.now(UTC)
+ stmt = select(AdminLeaderboardDailyStatRow).where(
+ AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
+ AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
+ )
rows = (
- await session.execute(
- select(AdminLeaderboardDailyStatRow).where(
- AdminLeaderboardDailyStatRow.stat_date.in_(stat_dates),
- AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
- )
+ await _execute_leaderboard_query(
+ session,
+ "snapshot_queue.load_existing",
+ stmt,
+ stat_dates=stat_dates,
+ settings_hash=settings_hash,
)
).scalars().all()
rows_by_date = {row.stat_date: row for row in rows}
@@ -1680,7 +1850,13 @@ async def _queue_daily_leaderboard_snapshots(
row.updated_at = now
queued.append(stat_date)
if queued:
- await session.commit()
+ await _time_leaderboard_db_operation(
+ "snapshot_queue.commit",
+ lambda: session.commit(),
+ sql="COMMIT",
+ stat_dates=queued,
+ settings_hash=settings_hash,
+ )
return queued
@@ -1722,7 +1898,22 @@ async def _refresh_daily_leaderboard_snapshot(
include_failed: bool,
) -> None:
acquired = False
+ trace_token = _LEADERBOARD_TRACE_CONTEXT.set(
+ {
+ "job_key": job_key,
+ "stat_date": stat_date,
+ "settings_hash": settings_hash,
+ }
+ )
+ snapshot_start = perf_counter()
try:
+ logger.info(
+ "Leaderboard snapshot build start: job=%s stat_date=%s settings_hash=%s started_at=%s",
+ job_key,
+ stat_date,
+ settings_hash,
+ datetime.now(BEIJING_TZ).isoformat(),
+ )
if _LEADERBOARD_START_DELAY_SECONDS > 0:
await asyncio.sleep(_LEADERBOARD_START_DELAY_SECONDS)
await _store_daily_snapshot(
@@ -1758,6 +1949,14 @@ async def _refresh_daily_leaderboard_snapshot(
error=None,
generated_at=generated_at,
)
+ logger.info(
+ "Leaderboard snapshot build done: job=%s stat_date=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ job_key,
+ stat_date,
+ settings_hash,
+ datetime.now(BEIJING_TZ).isoformat(),
+ (perf_counter() - snapshot_start) * 1000,
+ )
except asyncio.TimeoutError:
logger.warning("Daily leaderboard snapshot timed out: %s", job_key)
await _store_daily_snapshot(
@@ -1781,6 +1980,7 @@ async def _refresh_daily_leaderboard_snapshot(
finally:
if acquired:
_leaderboard_semaphore.release()
+ _LEADERBOARD_TRACE_CONTEXT.reset(trace_token)
async def _load_or_schedule_daily_leaderboard(
@@ -1978,6 +2178,72 @@ async def queue_leaderboard_snapshot_backfill(
)
+@router.post("/leaderboard/refresh-today", response_model=AdminLeaderboardResponse)
+async def refresh_today_leaderboard(
+ request: Request,
+ days: int = Query(default=30, ge=1, le=365),
+ since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
+ until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
+ exclude_user_ids: list[str] | None = Query(default=None, description="User ids excluded from this leaderboard aggregation"),
+ include_scheduled: bool | None = Query(default=None, description="Include runs created by scheduled tasks"),
+ include_admins: bool | None = Query(default=None, description="Include admin accounts in leaderboard aggregation"),
+ include_failed: bool | None = Query(default=None, description="Include failed runs and LLM calls"),
+ limit: int = Query(default=20, ge=5, le=100),
+) -> AdminLeaderboardResponse:
+ """Force a synchronous rebuild of *today's* snapshot, then return the range.
+
+ Today's snapshot otherwise refreshes only every ``_LEADERBOARD_TODAY_TTL_SECONDS``
+ (~5 min) via the background scheduler. This endpoint lets an admin pull the
+ latest real-time records into the stats on demand: it rebuilds today's daily
+ snapshot immediately (bounded by the same semaphore/timeout the scheduler uses)
+ and then returns the merged leaderboard for the requested range.
+ """
+ await _require_admin(request)
+ session_factory = get_session_factory()
+ if session_factory is None:
+ raise HTTPException(status_code=503, detail="Analytics requires database persistence")
+
+ settings = load_system_settings().leaderboard
+ (
+ excluded,
+ resolved_include_scheduled,
+ resolved_include_admins,
+ resolved_include_failed,
+ settings_hash,
+ ) = _resolve_leaderboard_snapshot_scope(
+ settings,
+ exclude_user_ids=exclude_user_ids,
+ include_scheduled=include_scheduled,
+ include_admins=include_admins,
+ include_failed=include_failed,
+ )
+ today = _today_stat_date()
+ await _refresh_daily_leaderboard_snapshot(
+ _daily_job_key(today, settings_hash),
+ session_factory=session_factory,
+ settings=settings,
+ stat_date=today,
+ settings_hash=settings_hash,
+ excluded=excluded,
+ include_scheduled=resolved_include_scheduled,
+ include_admins=resolved_include_admins,
+ include_failed=resolved_include_failed,
+ )
+
+ since_dt, until_dt, _range_days = _resolve_range(days, since, until)
+ return await _load_or_schedule_daily_leaderboard(
+ session_factory,
+ settings,
+ since_dt,
+ until_dt,
+ limit=limit,
+ exclude_user_ids=exclude_user_ids,
+ include_scheduled=include_scheduled,
+ include_admins=include_admins,
+ include_failed=include_failed,
+ )
+
+
async def _build_leaderboard_report_prompt(
session_factory,
settings,
@@ -2248,22 +2514,182 @@ class AdminUserQaPageResponse(BaseModel):
items: list[AdminUserQaItem] = Field(default_factory=list)
-@router.get("/{user_id}/qa", response_model=AdminUserQaPageResponse)
-async def list_user_qa(
- user_id: str,
- request: Request,
- page: int = Query(default=1, ge=1),
- page_size: int = Query(default=20, ge=1, le=100),
- q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
- since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
- until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
-) -> AdminUserQaPageResponse:
- """Return one user's Q&A runs from lightweight run summary fields."""
- await _require_admin(request)
- session_factory = get_session_factory()
- if session_factory is None:
- raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
+def _qa_sort_datetime(value: str | None) -> datetime:
+ if not value:
+ return datetime.min.replace(tzinfo=BEIJING_TZ)
+ text_value = str(value).strip()
+ if text_value.endswith("Z"):
+ text_value = text_value[:-1] + "+00:00"
+ try:
+ parsed = datetime.fromisoformat(text_value)
+ except ValueError:
+ return datetime.min.replace(tzinfo=BEIJING_TZ)
+ if parsed.tzinfo is None:
+ return parsed.replace(tzinfo=BEIJING_TZ)
+ return parsed.astimezone(BEIJING_TZ)
+
+def _qa_dedupe_key(item: AdminUserQaItem) -> tuple[str, str, str]:
+ question_key = _question_similarity_key(item.question)[:160]
+ answer_key = _question_similarity_key(item.answer)[:160]
+ return (item.thread_id, question_key, answer_key)
+
+
+def _message_value(message: Any, key: str, default: Any = None) -> Any:
+ if isinstance(message, dict):
+ return message.get(key, default)
+ return getattr(message, key, default)
+
+
+def _qa_message_type(message: Any) -> str:
+ value = _message_value(message, "type") or _message_value(message, "role") or ""
+ if value == "user":
+ return "human"
+ if value == "assistant":
+ return "ai"
+ return str(value)
+
+
+def _qa_message_id(message: Any, fallback: str) -> str:
+ return str(_message_value(message, "id") or fallback)
+
+
+def _qa_message_text(message: Any) -> str:
+ return _message_content_to_text(_message_value(message, "content")).strip()
+
+
+async def _read_thread_checkpoint_messages(checkpointer, thread_id: str) -> list[dict]:
+ config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
+ try:
+ checkpoint_tuple = await checkpointer.aget_tuple(config)
+ except Exception:
+ logger.warning("Failed to read checkpoint while listing Q&A: thread=%s", thread_id, exc_info=True)
+ return []
+ if checkpoint_tuple is None:
+ return []
+ checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
+ channel_values = checkpoint.get("channel_values", {}) if isinstance(checkpoint, dict) else {}
+ values = serialize_channel_values(channel_values)
+ messages = values.get("messages", [])
+ return messages if isinstance(messages, list) else []
+
+
+def _qa_items_from_checkpoint_messages(
+ *,
+ user_id: str,
+ thread_id: str,
+ thread_title: str,
+ assistant_id: str,
+ thread_time: str,
+ messages: list[dict],
+) -> list[AdminUserQaItem]:
+ items: list[AdminUserQaItem] = []
+ current_question = ""
+ current_answer = ""
+ current_id = ""
+ current_index = 0
+
+ def flush() -> None:
+ nonlocal current_question, current_answer, current_id, current_index
+ question = _clean_question_text(current_question)
+ if not _is_displayable_question(question):
+ current_question = ""
+ current_answer = ""
+ current_id = ""
+ return
+ items.append(
+ AdminUserQaItem(
+ run_id=current_id or f"{thread_id}:checkpoint:{current_index}",
+ thread_id=thread_id,
+ thread_title=thread_title,
+ assistant_id=assistant_id,
+ status="checkpoint",
+ question=question[:2000],
+ answer=current_answer.strip()[:6000],
+ created_at=thread_time,
+ updated_at=thread_time,
+ )
+ )
+ current_question = ""
+ current_answer = ""
+ current_id = ""
+
+ for index, message in enumerate(messages):
+ msg_type = _qa_message_type(message)
+ if msg_type == "human":
+ if current_question:
+ flush()
+ current_question = _qa_message_text(message)
+ current_answer = ""
+ current_id = _qa_message_id(message, f"{thread_id}:checkpoint:{index}")
+ current_index = index
+ continue
+ if msg_type == "ai" and current_question:
+ text = _qa_message_text(message)
+ if text:
+ current_answer = text
+
+ if current_question:
+ flush()
+
+ return items
+
+
+async def _list_thread_checkpoint_qa_items(
+ *,
+ request: Request,
+ user_id: str,
+) -> list[AdminUserQaItem]:
+ repo = get_thread_store(request)
+ checkpointer = get_checkpointer(request)
+ items: list[AdminUserQaItem] = []
+ offset = 0
+ page_size = 200
+ while True:
+ rows = await repo.search(limit=page_size, offset=offset, user_id=user_id)
+ if not rows:
+ break
+ for row in rows:
+ thread_id = str(row.get("thread_id") or "")
+ if not thread_id:
+ continue
+ messages = await _read_thread_checkpoint_messages(checkpointer, thread_id)
+ if not messages:
+ continue
+ metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+ context = row.get("context") if isinstance(row.get("context"), dict) else {}
+ assistant_id = str(
+ row.get("assistant_id")
+ or (context or {}).get("agent_id")
+ or (metadata or {}).get("agent_id")
+ or (context or {}).get("agent_name")
+ or (metadata or {}).get("agent_name")
+ or ""
+ )
+ thread_time = str(row.get("updated_at") or row.get("created_at") or "")
+ items.extend(
+ _qa_items_from_checkpoint_messages(
+ user_id=user_id,
+ thread_id=thread_id,
+ thread_title=str(row.get("display_name") or ""),
+ assistant_id=assistant_id,
+ thread_time=thread_time,
+ messages=messages,
+ )
+ )
+ if len(rows) < page_size:
+ break
+ offset += len(rows)
+ return items
+
+
+async def _list_run_summary_qa_items(
+ *,
+ session_factory,
+ user_id: str,
+ since: str | None,
+ until: str | None,
+) -> list[AdminUserQaItem]:
filters = [
RunRow.user_id == user_id,
RunRow.first_human_message.is_not(None),
@@ -2272,26 +2698,9 @@ async def list_user_qa(
if since or until:
since_dt, until_dt, _range_days = _resolve_range(365, since, until)
filters.append(_run_time_filter(since_dt, until_dt))
- needle = (q or "").strip()
- if needle:
- like = f"%{needle}%"
- filters.append(
- or_(
- RunRow.first_human_message.like(like),
- RunRow.last_ai_message.like(like),
- RunRow.run_id.like(like),
- RunRow.thread_id.like(like),
- )
- )
- offset = (page - 1) * page_size
async with session_factory() as session:
await _set_analytics_read_timeout(session)
- total = (
- await session.execute(
- select(func.count(RunRow.run_id)).where(*filters)
- )
- ).scalar_one()
rows = (
await session.execute(
select(
@@ -2316,32 +2725,152 @@ async def list_user_qa(
)
.where(*filters)
.order_by(RunRow.created_at.desc(), RunRow.run_id.desc())
- .limit(page_size)
- .offset(offset)
)
).all()
+ return [
+ AdminUserQaItem(
+ run_id=r[0] or "",
+ thread_id=r[1] or "",
+ thread_title=r[2] or "",
+ assistant_id=r[3] or "",
+ status=r[4] or "",
+ model_name=r[5] or "",
+ question=_clean_question_text(r[6])[:2000],
+ answer=str(r[7] or "").strip()[:6000],
+ total_tokens=int(r[8] or 0),
+ created_at=_iso(r[9]),
+ updated_at=_iso(r[10]),
+ )
+ for r in rows
+ if _is_displayable_question(r[6])
+ ]
+
+
+def _filter_qa_items(
+ items: list[AdminUserQaItem],
+ *,
+ q: str | None,
+ since: str | None,
+ until: str | None,
+) -> list[AdminUserQaItem]:
+ needle = (q or "").strip().lower()
+ since_dt = _parse_optional_datetime(since) if since else None
+ until_dt = _parse_optional_datetime(until) if until else None
+ filtered: list[AdminUserQaItem] = []
+ for item in items:
+ item_dt = _qa_sort_datetime(item.created_at or item.updated_at)
+ if since_dt and item_dt < since_dt:
+ continue
+ if until_dt and item_dt > until_dt:
+ continue
+ if needle:
+ haystack = "\n".join(
+ [
+ item.question,
+ item.answer,
+ item.run_id,
+ item.thread_id,
+ item.thread_title,
+ item.assistant_id,
+ ]
+ ).lower()
+ if needle not in haystack:
+ continue
+ filtered.append(item)
+ return filtered
+
+
+async def _list_user_qa_page(
+ *,
+ request: Request,
+ user_id: str,
+ page: int,
+ page_size: int,
+ q: str | None,
+ since: str | None,
+ until: str | None,
+ include_checkpoints: bool = True,
+) -> AdminUserQaPageResponse:
+ session_factory = get_session_factory()
+ if session_factory is None:
+ raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
+
+ items = await _list_run_summary_qa_items(
+ session_factory=session_factory,
+ user_id=user_id,
+ since=since,
+ until=until,
+ )
+ if include_checkpoints:
+ items.extend(await _list_thread_checkpoint_qa_items(request=request, user_id=user_id))
+
+ by_key: dict[tuple[str, str, str], AdminUserQaItem] = {}
+ for item in items:
+ key = _qa_dedupe_key(item)
+ existing = by_key.get(key)
+ if existing is None or (existing.status == "checkpoint" and item.status != "checkpoint"):
+ by_key[key] = item
+
+ filtered = _filter_qa_items(list(by_key.values()), q=q, since=since, until=until)
+ filtered.sort(
+ key=lambda item: (_qa_sort_datetime(item.created_at or item.updated_at), item.thread_id, item.run_id),
+ reverse=True,
+ )
+ offset = (page - 1) * page_size
return AdminUserQaPageResponse(
user_id=user_id,
page=page,
page_size=page_size,
- total=int(total or 0),
- items=[
- AdminUserQaItem(
- run_id=r[0] or "",
- thread_id=r[1] or "",
- thread_title=r[2] or "",
- assistant_id=r[3] or "",
- status=r[4] or "",
- model_name=r[5] or "",
- question=_clean_question_text(r[6])[:2000],
- answer=str(r[7] or "").strip()[:6000],
- total_tokens=int(r[8] or 0),
- created_at=_iso(r[9]),
- updated_at=_iso(r[10]),
- )
- for r in rows
- ],
+ total=len(filtered),
+ items=filtered[offset : offset + page_size],
+ )
+
+
+@router.get("/{user_id}/qa", response_model=AdminUserQaPageResponse)
+async def list_user_qa(
+ user_id: str,
+ request: Request,
+ page: int = Query(default=1, ge=1),
+ page_size: int = Query(default=20, ge=1, le=100),
+ q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
+ since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
+ until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
+) -> AdminUserQaPageResponse:
+ """Return one user's Q&A runs, including older checkpoint-only threads."""
+ await _require_admin(request)
+ return await _list_user_qa_page(
+ request=request,
+ user_id=user_id,
+ page=page,
+ page_size=page_size,
+ q=q,
+ since=since,
+ until=until,
+ )
+
+
+@self_router.get("/qa", response_model=AdminUserQaPageResponse)
+async def list_my_qa(
+ request: Request,
+ page: int = Query(default=1, ge=1),
+ page_size: int = Query(default=20, ge=1, le=100),
+ q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
+ since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
+ until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
+) -> AdminUserQaPageResponse:
+ """Return the current user's own Q&A history."""
+ user = getattr(request.state, "user", None)
+ if user is None:
+ user = await get_current_user_from_request(request)
+ return await _list_user_qa_page(
+ request=request,
+ user_id=str(user.id),
+ page=page,
+ page_size=page_size,
+ q=q,
+ since=since,
+ until=until,
)
diff --git a/offline-backend-20260512/backend/app/gateway/routers/agents.py b/offline-backend-20260512/backend/app/gateway/routers/agents.py
index a75df4c8..7848dc8c 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/agents.py
+++ b/offline-backend-20260512/backend/app/gateway/routers/agents.py
@@ -15,7 +15,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.auth.password import verify_password_async
-from app.gateway.deps import get_agent_store, get_config, get_tag_store
+from app.gateway.deps import get_agent_store, get_config, get_tag_store
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.config.agents_api_config import get_agents_api_config
from deerflow.config.agents_config import list_custom_agents, load_agent_config, load_agent_soul, validate_agent_id
@@ -39,6 +39,7 @@ class AgentResponse(BaseModel):
name: str = Field(..., description="Display name. Chinese and duplicate names are allowed.")
description: str = Field(default="", description="Agent description")
user_id: str | None = Field(default=None, description="Owner user id. Null means built-in.")
+ owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in agents.")
published: bool = Field(default=False, description="Whether a user-created agent is public")
is_private: bool = Field(default=False, description="Whether the agent lives in the password-gated private square instead of the public one")
builtin: bool = Field(default=False, description="True when user_id is null")
@@ -62,15 +63,15 @@ class AgentResponse(BaseModel):
is_favorite: bool = Field(
default=False,
description="Whether the requesting user has favorited (收藏) this agent.",
- )
-
- is_pinned: bool = Field(
- default=False,
- description="Whether the requesting user has pinned this agent.",
- )
-
-
-class AgentsListResponse(BaseModel):
+ )
+
+ is_pinned: bool = Field(
+ default=False,
+ description="Whether the requesting user has pinned this agent.",
+ )
+
+
+class AgentsListResponse(BaseModel):
"""Response model for listing agents.
``total`` / ``page`` / ``page_size`` describe server-side pagination. When
@@ -115,7 +116,7 @@ class AgentUpdateRequest(BaseModel):
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
-class AgentSelectorResponse(BaseModel):
+class AgentSelectorResponse(BaseModel):
"""Response for the homepage agent selector dropdown.
The list is fully curated by admins (``featured_order`` on each row).
@@ -125,10 +126,10 @@ class AgentSelectorResponse(BaseModel):
agents: list[AgentResponse] = Field(
default_factory=list,
description="Admin-curated agents, sorted by featured_order ascending.",
- )
-
-
-class AgentFeaturedRequest(BaseModel):
+ )
+
+
+class AgentFeaturedRequest(BaseModel):
"""Request body for setting/clearing an agent's homepage featured rank."""
order: int | None = Field(
@@ -159,6 +160,41 @@ def _is_admin_user(request: Request) -> bool:
return getattr(user, "system_role", None) == "admin"
+def _email_local_part(email: str | None) -> str | None:
+ """Return the username portion (before ``@``) of an email address."""
+ if not email:
+ return None
+ at = email.find("@")
+ return email[:at] if at > 0 else email
+
+
+async def _resolve_usernames(user_ids: set[str]) -> dict[str, str]:
+ """Batch-resolve owner user ids to friendly usernames (email local part).
+
+ One lookup per distinct id; tolerates missing accounts (deleted users) and
+ an uninitialised auth provider by simply omitting those ids from the map.
+ """
+ user_ids = {uid for uid in user_ids if uid}
+ if not user_ids:
+ return {}
+ try:
+ from app.gateway.deps import get_local_provider
+
+ provider = get_local_provider()
+ except Exception:
+ return {}
+ names: dict[str, str] = {}
+ for uid in user_ids:
+ try:
+ user = await provider.get_user(uid)
+ except Exception:
+ user = None
+ local = _email_local_part(getattr(user, "email", None)) if user is not None else None
+ if local:
+ names[uid] = local
+ return names
+
+
def _validate_display_name(name: str) -> str:
normalized = name.strip()
if not normalized:
@@ -348,7 +384,7 @@ def _ensure_agent_dir_from_record(record: dict[str, Any]) -> None:
logger.exception("Failed to self-heal agent dir for '%s'", agent_id)
-def _record_to_response(
+def _record_to_response(
record: dict[str, Any],
*,
include_soul: bool = False,
@@ -390,21 +426,21 @@ def _record_to_response(
tool_groups=extras["tool_groups"],
skills=extras["skills"],
featured_order=record.get("featured_order"),
- )
-
-
-def _sort_agent_responses(responses: list[AgentResponse], sort: str) -> list[AgentResponse]:
- reverse = sort not in {"time_asc", "updated_asc", "name_asc"}
- if sort in {"name_asc", "name_desc"}:
- ordered = sorted(responses, key=lambda r: (r.name or "").lower(), reverse=reverse)
- elif sort in {"updated_desc", "updated_asc"}:
- ordered = sorted(responses, key=lambda r: str(r.updated_at or ""), reverse=reverse)
- else:
- ordered = sorted(responses, key=lambda r: str(r.created_at or ""), reverse=reverse)
- return sorted(ordered, key=lambda r: 0 if r.is_pinned else 1)
-
-
-async def _record_to_response_async(
+ )
+
+
+def _sort_agent_responses(responses: list[AgentResponse], sort: str) -> list[AgentResponse]:
+ reverse = sort not in {"time_asc", "updated_asc", "name_asc"}
+ if sort in {"name_asc", "name_desc"}:
+ ordered = sorted(responses, key=lambda r: (r.name or "").lower(), reverse=reverse)
+ elif sort in {"updated_desc", "updated_asc"}:
+ ordered = sorted(responses, key=lambda r: str(r.updated_at or ""), reverse=reverse)
+ else:
+ ordered = sorted(responses, key=lambda r: str(r.created_at or ""), reverse=reverse)
+ return sorted(ordered, key=lambda r: 0 if r.is_pinned else 1)
+
+
+async def _record_to_response_async(
record: dict[str, Any],
*,
include_soul: bool = False,
@@ -478,20 +514,23 @@ async def _build_list_responses(
_record_to_response(a, include_soul=False, include_extras=False) for a in records
]
ids = [r.id for r in responses]
- extras = await store.get_extras_for(ids)
- assignments = await tag_store.list_assignments("agent", ids)
- favorite_ids = await store.list_favorite_ids(user_id)
- pin_ids = await store.list_pin_ids(user_id)
- for resp in responses:
+ extras = await store.get_extras_for(ids)
+ assignments = await tag_store.list_assignments("agent", ids)
+ favorite_ids = await store.list_favorite_ids(user_id)
+ pin_ids = await store.list_pin_ids(user_id)
+ usernames = await _resolve_usernames({r.user_id for r in responses if r.user_id})
+ for resp in responses:
cached = extras.get(resp.id)
if cached is not None:
resp.skills = cached.get("skills")
resp.tool_groups = cached.get("tool_groups")
resp.model = cached.get("model")
- resp.tags = assignments.get(resp.id, [])
- resp.is_favorite = resp.id in favorite_ids
- resp.is_pinned = resp.id in pin_ids
- return responses
+ resp.tags = assignments.get(resp.id, [])
+ resp.is_favorite = resp.id in favorite_ids
+ resp.is_pinned = resp.id in pin_ids
+ if resp.user_id:
+ resp.owner_username = usernames.get(resp.user_id)
+ return responses
@router.get(
@@ -509,14 +548,18 @@ async def list_agents(
pattern="^(mine|square)$",
description="Pagination scope. 'mine' = owned + favorited; 'square' = built-in/published public. Omit for the default combined visibility list.",
),
- page: int | None = Query(default=None, ge=1, description="1-based page index. Providing this (or 'scope') enables server-side pagination."),
- page_size: int = Query(default=20, ge=1, le=200, description="Items per page when paginating."),
- sort: str = Query(
- default="time_desc",
- pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
- description="Sort order. Default is newest created first.",
- ),
- admin: bool = Query(
+ page: int | None = Query(default=None, ge=1, description="1-based page index. Providing this (or 'scope') enables server-side pagination."),
+ page_size: int = Query(default=20, ge=1, le=200, description="Items per page when paginating."),
+ sort: str = Query(
+ default="time_desc",
+ pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
+ description="Sort order. Default is newest created first.",
+ ),
+ owner: str | None = Query(
+ default=None,
+ description="Keep only agents published by this user id (the card's publisher). Built-in agents never match.",
+ ),
+ admin: bool = Query(
default=False,
description=(
"Admin-only override. When ``true`` and the caller is a system admin, "
@@ -543,11 +586,12 @@ async def list_agents(
scope=scope,
search=search,
tag_ids=tag_ids,
- page=eff_page,
- page_size=page_size,
- admin=admin,
- sort=sort,
- )
+ page=eff_page,
+ page_size=page_size,
+ admin=admin,
+ sort=sort,
+ owner=owner,
+ )
responses = await _build_list_responses(store, tag_store, user_id, rows)
return AgentsListResponse(agents=responses, total=total, page=eff_page, page_size=page_size)
@@ -559,11 +603,13 @@ async def list_agents(
if search:
needle = search.strip().lower()
responses = [r for r in responses if needle in r.name.lower()]
- if tag_ids:
- wanted = set(tag_ids)
- responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
- responses = _sort_agent_responses(responses, sort)
- return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
+ if tag_ids:
+ wanted = set(tag_ids)
+ responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
+ if owner and owner.strip():
+ responses = [r for r in responses if r.user_id == owner.strip()]
+ responses = _sort_agent_responses(responses, sort)
+ return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
except HTTPException:
raise
except Exception as e:
@@ -576,14 +622,14 @@ async def list_agents(
summary="Check Agent Name",
description="Validate an agent display name. Duplicate names are allowed.",
)
-async def check_agent_name(name: str) -> dict[str, Any]:
- _require_agents_api_enabled()
- normalized = _validate_display_name(name)
- return {"available": True, "name": normalized}
-
-
-@router.get(
- "/agents/selector",
+async def check_agent_name(name: str) -> dict[str, Any]:
+ _require_agents_api_enabled()
+ normalized = _validate_display_name(name)
+ return {"available": True, "name": normalized}
+
+
+@router.get(
+ "/agents/selector",
response_model=AgentSelectorResponse,
summary="Agent Homepage Selector",
description=(
@@ -635,12 +681,12 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
# worker thread so the asyncio loop stays responsive under concurrent
# detail fetches.
- response = await _record_to_response_async(record, include_soul=True, include_extras=True)
- favorite_ids = await store.list_favorite_ids(user_id)
- pin_ids = await store.list_pin_ids(user_id)
- response.is_favorite = agent_id in favorite_ids
- response.is_pinned = agent_id in pin_ids
- return response
+ response = await _record_to_response_async(record, include_soul=True, include_extras=True)
+ favorite_ids = await store.list_favorite_ids(user_id)
+ pin_ids = await store.list_pin_ids(user_id)
+ response.is_favorite = agent_id in favorite_ids
+ response.is_pinned = agent_id in pin_ids
+ return response
except HTTPException:
raise
except Exception as e:
@@ -874,16 +920,16 @@ async def delete_agent(agent_id: str, request: Request) -> None:
await get_tag_store(request).unassign_all("agent", agent_id)
except Exception: # noqa: BLE001 — tag cleanup must not block deletion
pass
- try:
- await store.delete_favorites_for_agent(agent_id)
- except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
- pass
- try:
- await store.delete_pins_for_agent(agent_id)
- except Exception: # noqa: BLE001 - pin cleanup must not block deletion
- pass
- try:
- await store.delete_extras_for_agent(agent_id)
+ try:
+ await store.delete_favorites_for_agent(agent_id)
+ except Exception: # noqa: BLE001 — favorite cleanup must not block deletion
+ pass
+ try:
+ await store.delete_pins_for_agent(agent_id)
+ except Exception: # noqa: BLE001 - pin cleanup must not block deletion
+ pass
+ try:
+ await store.delete_extras_for_agent(agent_id)
except Exception: # noqa: BLE001 — cache cleanup must not block deletion
pass
logger.info("Deleted agent '%s' from %s", agent_id, agent_dir)
@@ -925,7 +971,7 @@ async def favorite_agent(agent_id: str, request: Request) -> None:
summary="Unfavorite Agent",
description="Remove an agent from the current user's favorites. No-op when not favorited.",
)
-async def unfavorite_agent(agent_id: str, request: Request) -> None:
+async def unfavorite_agent(agent_id: str, request: Request) -> None:
_require_agents_api_enabled()
agent_id = _validate_storage_id(agent_id)
@@ -936,60 +982,60 @@ async def unfavorite_agent(agent_id: str, request: Request) -> None:
logger.info("User '%s' unfavorited agent '%s'", user_id, agent_id)
except HTTPException:
raise
- except Exception as e:
- logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
- raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
-
-
-@router.post(
- "/agents/{agent_id}/pin",
- status_code=204,
- summary="Pin Agent",
- description="Pin a visible agent to the top of the current user's lists. Idempotent.",
-)
-async def pin_agent(agent_id: str, request: Request) -> None:
- _require_agents_api_enabled()
- agent_id = _validate_storage_id(agent_id)
-
- try:
- store = get_agent_store(request)
- user_id = _current_user_id(request)
- record = await store.get_visible(agent_id, user_id)
- if record is None:
- raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
- await store.add_pin(user_id, agent_id)
- logger.info("User '%s' pinned agent '%s'", user_id, agent_id)
- except HTTPException:
- raise
- except Exception as e:
- logger.error("Failed to pin agent '%s': %s", agent_id, e, exc_info=True)
- raise HTTPException(status_code=500, detail=f"Failed to pin agent: {str(e)}") from e
-
-
-@router.delete(
- "/agents/{agent_id}/pin",
- status_code=204,
- summary="Unpin Agent",
- description="Remove the current user's pin from an agent. No-op when not pinned.",
-)
-async def unpin_agent(agent_id: str, request: Request) -> None:
- _require_agents_api_enabled()
- agent_id = _validate_storage_id(agent_id)
-
- try:
- store = get_agent_store(request)
- user_id = _current_user_id(request)
- await store.remove_pin(user_id, agent_id)
- logger.info("User '%s' unpinned agent '%s'", user_id, agent_id)
- except HTTPException:
- raise
- except Exception as e:
- logger.error("Failed to unpin agent '%s': %s", agent_id, e, exc_info=True)
- raise HTTPException(status_code=500, detail=f"Failed to unpin agent: {str(e)}") from e
-
-
-@router.put(
- "/agents/{agent_id}/featured",
+ except Exception as e:
+ logger.error("Failed to unfavorite agent '%s': %s", agent_id, e, exc_info=True)
+ raise HTTPException(status_code=500, detail=f"Failed to unfavorite agent: {str(e)}") from e
+
+
+@router.post(
+ "/agents/{agent_id}/pin",
+ status_code=204,
+ summary="Pin Agent",
+ description="Pin a visible agent to the top of the current user's lists. Idempotent.",
+)
+async def pin_agent(agent_id: str, request: Request) -> None:
+ _require_agents_api_enabled()
+ agent_id = _validate_storage_id(agent_id)
+
+ try:
+ store = get_agent_store(request)
+ user_id = _current_user_id(request)
+ record = await store.get_visible(agent_id, user_id)
+ if record is None:
+ raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
+ await store.add_pin(user_id, agent_id)
+ logger.info("User '%s' pinned agent '%s'", user_id, agent_id)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error("Failed to pin agent '%s': %s", agent_id, e, exc_info=True)
+ raise HTTPException(status_code=500, detail=f"Failed to pin agent: {str(e)}") from e
+
+
+@router.delete(
+ "/agents/{agent_id}/pin",
+ status_code=204,
+ summary="Unpin Agent",
+ description="Remove the current user's pin from an agent. No-op when not pinned.",
+)
+async def unpin_agent(agent_id: str, request: Request) -> None:
+ _require_agents_api_enabled()
+ agent_id = _validate_storage_id(agent_id)
+
+ try:
+ store = get_agent_store(request)
+ user_id = _current_user_id(request)
+ await store.remove_pin(user_id, agent_id)
+ logger.info("User '%s' unpinned agent '%s'", user_id, agent_id)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error("Failed to unpin agent '%s': %s", agent_id, e, exc_info=True)
+ raise HTTPException(status_code=500, detail=f"Failed to unpin agent: {str(e)}") from e
+
+
+@router.put(
+ "/agents/{agent_id}/featured",
response_model=AgentResponse,
summary="Set Agent Featured Rank (admin)",
description=(
@@ -1031,19 +1077,20 @@ async def set_agent_featured(
raise HTTPException(status_code=500, detail=f"Failed to set featured: {str(e)}") from e
-class PrivateSquareListRequest(BaseModel):
+class PrivateSquareListRequest(BaseModel):
"""Request body for listing all private-square agents."""
password: str = Field(..., description="Plain-text private-square password. Verified against the admin-configured hash before the listing is returned.")
search: str | None = Field(default=None, description="Case-insensitive fuzzy match on agent name (raw or desensitized).")
tag_ids: list[str] | None = Field(default=None, description="Keep agents carrying any of these tags (OR).")
- page: int | None = Field(default=None, ge=1, description="1-based page index. Providing this enables server-side pagination.")
- page_size: int = Field(default=20, ge=1, le=200, description="Items per page when paginating.")
- sort: str = Field(
- default="time_desc",
- pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
- description="Sort order. Default is newest created first.",
- )
+ page: int | None = Field(default=None, ge=1, description="1-based page index. Providing this enables server-side pagination.")
+ page_size: int = Field(default=20, ge=1, le=200, description="Items per page when paginating.")
+ sort: str = Field(
+ default="time_desc",
+ pattern="^(time_desc|time_asc|updated_desc|updated_asc|name_asc|name_desc)$",
+ description="Sort order. Default is newest created first.",
+ )
+ owner: str | None = Field(default=None, description="Keep only agents published by this user id (the card's publisher).")
class AgentPrivateUpdateRequest(BaseModel):
@@ -1086,17 +1133,18 @@ async def list_private_square(body: PrivateSquareListRequest, request: Request)
scope="private",
search=body.search,
tag_ids=body.tag_ids,
- page=body.page,
- page_size=body.page_size,
- sort=body.sort,
- )
+ page=body.page,
+ page_size=body.page_size,
+ sort=body.sort,
+ owner=body.owner,
+ )
responses = await _build_list_responses(store, tag_store, user_id, rows)
return AgentsListResponse(agents=responses, total=total, page=body.page, page_size=body.page_size)
records = await store.list_private()
- responses = await _build_list_responses(store, tag_store, user_id, records)
- responses = _sort_agent_responses(responses, body.sort)
- return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
+ responses = await _build_list_responses(store, tag_store, user_id, records)
+ responses = _sort_agent_responses(responses, body.sort)
+ return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
except HTTPException:
raise
except Exception as e:
diff --git a/offline-backend-20260512/backend/app/gateway/routers/browser_context.py b/offline-backend-20260512/backend/app/gateway/routers/browser_context.py
new file mode 100644
index 00000000..3a2237c0
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/routers/browser_context.py
@@ -0,0 +1,590 @@
+"""Browser page context capture endpoints.
+
+This router is intentionally isolated behind ``config.yaml``:
+
+ browser_context:
+ enabled: true
+
+When disabled, mutating endpoints behave as if the integration is unavailable.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from datetime import UTC, datetime
+from typing import Any, Literal
+from urllib.parse import urlencode
+from uuid import uuid4
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from app.gateway.authz import require_permission
+from app.gateway.deps import get_checkpointer, get_config, get_thread_store
+from app.gateway.routers.thread_runs import RunCreateRequest
+from app.gateway.routers.threads import ThreadCreateRequest, create_thread
+from app.gateway.routers.uploads import _make_file_sandbox_writable
+from app.gateway.services import start_run
+from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
+from deerflow.config.app_config import AppConfig
+from deerflow.config.browser_context_config import BrowserContextConfig
+from deerflow.runtime import serialize_channel_values
+from deerflow.runtime.user_context import get_effective_user_id
+from deerflow.sandbox.sandbox_provider import get_sandbox_provider
+from deerflow.uploads.manager import (
+ UnsafeUploadPathError,
+ ensure_uploads_dir,
+ normalize_filename,
+ open_upload_file_no_symlink,
+ upload_artifact_url,
+ upload_virtual_path,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/browser-context", tags=["browser-context"])
+DEFAULT_BROWSER_CONTEXT_PROMPT = "\u8bf7\u57fa\u4e8e\u521a\u521a\u53d1\u9001\u7684\u7f51\u9875\u5185\u5bb9\uff0c\u5148\u7528\u4e2d\u6587\u603b\u7ed3\u8981\u70b9\uff0c\u5e76\u7b49\u5f85\u6211\u7ee7\u7eed\u63d0\u95ee\u3002"
+
+
+class BrowserContextStatusResponse(BaseModel):
+ enabled: bool
+ max_content_chars: int
+ max_title_chars: int
+ max_url_chars: int
+ task_result_timeout_seconds: int
+ task_poll_interval_seconds: float
+ task_lease_seconds: int
+ tool_result_max_chars: int
+
+
+class BrowserContextCaptureRequest(BaseModel):
+ thread_id: str | None = Field(default=None, description="Existing or caller-generated zncm thread id")
+ title: str | None = Field(default=None, description="Current browser page title")
+ url: str | None = Field(default=None, description="Current browser page URL")
+ content: str = Field(description="Captured page content, preferably Markdown")
+ content_type: Literal["markdown", "text"] = Field(default="markdown")
+ prompt: str | None = Field(default=None, description="Optional first question to ask in zncm after capture")
+
+
+class BrowserContextFileInfo(BaseModel):
+ filename: str
+ size: int
+ path: str
+ virtual_path: str
+ artifact_url: str
+
+
+class BrowserContextCaptureResponse(BaseModel):
+ success: bool
+ thread_id: str
+ file: BrowserContextFileInfo
+ chat_path: str
+ chat_url: str
+ browser_context_token: str
+ prompt: str
+ truncated: bool = False
+ message: str
+
+
+class BrowserContextAskRequest(BaseModel):
+ thread_id: str | None = Field(default=None, description="Existing zncm thread id for follow-up questions")
+ title: str | None = Field(default=None, description="Current browser page title")
+ url: str | None = Field(default=None, description="Current browser page URL")
+ content: str = Field(description="Captured page content, preferably Markdown")
+ content_type: Literal["markdown", "text"] = Field(default="markdown")
+ question: str = Field(description="Question asked from the in-page assistant")
+
+
+class BrowserContextAskResponse(BaseModel):
+ success: bool
+ thread_id: str
+ run_id: str
+ answer: str
+ truncated: bool = False
+ message: str
+
+
+class BrowserContextTaskPayload(BaseModel):
+ id: str
+ kind: str = "fetch"
+ url: str
+ instruction: str = ""
+ session_id: str = ""
+ action: dict[str, Any] | None = None
+ created_at: str
+ updated_at: str
+ max_content_chars: int
+
+
+class BrowserContextTaskNextResponse(BaseModel):
+ task: BrowserContextTaskPayload | None = None
+
+
+class BrowserContextTaskResultRequest(BaseModel):
+ kind: str = Field(default="fetch", description="Task kind the extension processed: fetch or act")
+ title: str | None = Field(default=None, description="Captured page title")
+ url: str | None = Field(default=None, description="Final page URL after redirects")
+ content: str | None = Field(default=None, description="Captured page content")
+ content_type: Literal["markdown", "text"] = Field(default="markdown")
+ observation: dict[str, Any] | None = Field(default=None, description="Structured observation for an act step (elements, text, status)")
+ error: str | None = Field(default=None, description="Extension-side task error")
+ metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class BrowserContextTaskResultResponse(BaseModel):
+ success: bool
+ task_id: str
+ status: str
+ truncated: bool = False
+
+
+def _settings(config: AppConfig) -> BrowserContextConfig:
+ value = getattr(config, "browser_context", None)
+ if isinstance(value, BrowserContextConfig):
+ return value
+ if isinstance(value, dict):
+ return BrowserContextConfig.model_validate(value)
+ return BrowserContextConfig()
+
+
+def _require_enabled(config: AppConfig) -> BrowserContextConfig:
+ settings = _settings(config)
+ if not settings.enabled:
+ raise HTTPException(status_code=404, detail="Browser context integration is disabled")
+ return settings
+
+
+def _trim(value: str | None, limit: int) -> str:
+ if not value:
+ return ""
+ return value.strip()[:limit]
+
+
+def _limit_utf8_bytes(value: str, max_bytes: int) -> str:
+ text = value
+ while len(text.encode("utf-8")) > max_bytes:
+ text = text[:-1]
+ return text
+
+
+def _filename_part(value: str, fallback: str, *, max_bytes: int) -> str:
+ text = value.strip() or fallback
+ # Keep unicode word chars so Chinese titles remain readable, while removing
+ # path separators and shell-hostile punctuation.
+ text = re.sub(r"[^\w.-]+", "-", text, flags=re.UNICODE).strip("-._")
+ text = _limit_utf8_bytes(text, max_bytes)
+ return text or fallback
+
+
+def _build_markdown(
+ *,
+ title: str,
+ url: str,
+ content: str,
+ content_type: str,
+ captured_at: str,
+ truncated: bool,
+) -> str:
+ lines = [
+ f"# {title or 'Captured Web Page'}",
+ "",
+ f"- Source URL: {url or '(unknown)'}",
+ f"- Captured at: {captured_at}",
+ ]
+ if truncated:
+ lines.append("- Note: Content was truncated by the browser_context.max_content_chars limit.")
+ lines.extend(["", "---", ""])
+
+ if content_type == "markdown":
+ lines.append(content.strip())
+ else:
+ lines.append("```text")
+ lines.append(content.strip())
+ lines.append("```")
+ lines.append("")
+ return "\n".join(lines)
+
+
+def _build_ask_prompt(
+ *,
+ title: str,
+ url: str,
+ content: str,
+ content_type: str,
+ captured_at: str,
+ truncated: bool,
+) -> str:
+ note = "页面内容已按配置截断,请只基于可见内容回答。" if truncated else "页面内容如下。"
+ body = content.strip()
+ if content_type == "text":
+ body = f"```text\n{body}\n```"
+ return "\n\n".join(
+ [
+ "你是 zncm 页面问答助手。请直接基于当前网页内容回答用户问题。",
+ "如果网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
+ f"页面标题:{title or '未知标题'}",
+ f"页面 URL:{url or '未知 URL'}",
+ f"读取时间:{captured_at}",
+ f"说明:{note}",
+ "当前网页内容:",
+ body,
+ ]
+ )
+
+
+def _message_type(message: Any) -> str:
+ if isinstance(message, dict):
+ return str(message.get("type") or message.get("role") or "").lower()
+ return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower()
+
+
+def _content_to_text(content: Any) -> str:
+ if isinstance(content, str):
+ return content.strip()
+ if isinstance(content, list):
+ parts: list[str] = []
+ for item in content:
+ if isinstance(item, str):
+ parts.append(item)
+ elif isinstance(item, dict):
+ if isinstance(item.get("text"), str):
+ parts.append(item["text"])
+ elif item.get("type") == "text" and isinstance(item.get("content"), str):
+ parts.append(item["content"])
+ return "\n".join(part.strip() for part in parts if part and part.strip()).strip()
+ return ""
+
+
+def _latest_ai_answer(values: dict[str, Any]) -> str:
+ messages = values.get("messages")
+ if not isinstance(messages, list):
+ return ""
+ for message in reversed(messages):
+ if _message_type(message) not in {"ai", "assistant"}:
+ continue
+ answer = _content_to_text(message.get("content") if isinstance(message, dict) else getattr(message, "content", None))
+ if answer:
+ return answer
+ return ""
+
+
+async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_path, data: bytes) -> None:
+ sandbox_provider = get_sandbox_provider()
+ if bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)):
+ return
+ sandbox_id = sandbox_provider.acquire(thread_id)
+ sandbox = sandbox_provider.get(sandbox_id)
+ if sandbox is None:
+ raise HTTPException(status_code=500, detail="Failed to acquire sandbox")
+ _make_file_sandbox_writable(file_path)
+ sandbox.update_file(virtual_path, data)
+
+
+@router.get("/status", response_model=BrowserContextStatusResponse)
+async def get_browser_context_status(
+ config: AppConfig = Depends(get_config),
+) -> BrowserContextStatusResponse:
+ settings = _settings(config)
+ return BrowserContextStatusResponse(
+ enabled=settings.enabled,
+ max_content_chars=settings.max_content_chars,
+ max_title_chars=settings.max_title_chars,
+ max_url_chars=settings.max_url_chars,
+ task_result_timeout_seconds=settings.task_result_timeout_seconds,
+ task_poll_interval_seconds=settings.task_poll_interval_seconds,
+ task_lease_seconds=settings.task_lease_seconds,
+ tool_result_max_chars=settings.tool_result_max_chars,
+ )
+
+
+@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse)
+@require_permission("runs", "create")
+async def claim_browser_context_task(
+ request: Request,
+ worker_id: str | None = None,
+ config: AppConfig = Depends(get_config),
+) -> BrowserContextTaskNextResponse:
+ settings = _require_enabled(config)
+ prune_browser_tasks()
+ task = claim_next_browser_task(
+ user_id=get_effective_user_id(),
+ worker_id=worker_id or "chrome-extension",
+ lease_seconds=settings.task_lease_seconds,
+ )
+ if task is None:
+ return BrowserContextTaskNextResponse(task=None)
+ return BrowserContextTaskNextResponse(task=BrowserContextTaskPayload.model_validate(task))
+
+
+@router.post("/tasks/{task_id}/result", response_model=BrowserContextTaskResultResponse)
+@require_permission("runs", "create")
+async def submit_browser_context_task_result(
+ task_id: str,
+ body: BrowserContextTaskResultRequest,
+ request: Request,
+ config: AppConfig = Depends(get_config),
+) -> BrowserContextTaskResultResponse:
+ settings = _require_enabled(config)
+ error = _trim(body.error, 4000)
+ content = body.content or ""
+ is_act = (body.kind or "fetch") == "act"
+ if not error and not is_act and not content.strip():
+ raise HTTPException(status_code=400, detail="Captured page content is empty")
+ if not error and is_act and not isinstance(body.observation, dict):
+ raise HTTPException(status_code=400, detail="Act task result requires an observation")
+
+ task = complete_browser_task(
+ task_id=task_id,
+ user_id=get_effective_user_id(),
+ title=_trim(body.title, settings.max_title_chars),
+ url=_trim(body.url, settings.max_url_chars),
+ content=content,
+ content_type=body.content_type,
+ error=error,
+ metadata=body.metadata,
+ observation=body.observation,
+ )
+ if task is None:
+ raise HTTPException(status_code=404, detail=f"Browser task {task_id} not found")
+
+ result = task.get("result") or {}
+ return BrowserContextTaskResultResponse(
+ success=True,
+ task_id=task_id,
+ status=str(task.get("status") or "unknown"),
+ truncated=bool(result.get("truncated")),
+ )
+
+
+@router.post("/capture", response_model=BrowserContextCaptureResponse)
+@require_permission("threads", "write")
+async def capture_browser_context(
+ body: BrowserContextCaptureRequest,
+ request: Request,
+ config: AppConfig = Depends(get_config),
+) -> BrowserContextCaptureResponse:
+ settings = _require_enabled(config)
+
+ raw_content = (body.content or "").strip()
+ if not raw_content:
+ raise HTTPException(status_code=400, detail="Captured page content is empty")
+
+ title = _trim(body.title, settings.max_title_chars)
+ url = _trim(body.url, settings.max_url_chars)
+ truncated = len(raw_content) > settings.max_content_chars
+ content = raw_content[: settings.max_content_chars] if truncated else raw_content
+
+ requested_thread_id = (body.thread_id or "").strip() or None
+ if requested_thread_id:
+ thread_store = get_thread_store(request)
+ allowed = await thread_store.check_access(
+ requested_thread_id,
+ get_effective_user_id(),
+ require_existing=False,
+ )
+ if not allowed:
+ raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
+
+ thread_response = await create_thread(
+ ThreadCreateRequest(
+ thread_id=requested_thread_id,
+ metadata={
+ "source": "browser_context",
+ "browser_context_url": url,
+ "browser_context_title": title,
+ },
+ ),
+ request,
+ )
+ thread_id = thread_response.thread_id
+
+ captured_at = datetime.now(UTC).isoformat()
+ markdown = _build_markdown(
+ title=title,
+ url=url,
+ content=content,
+ content_type=body.content_type,
+ captured_at=captured_at,
+ truncated=truncated,
+ )
+ data = markdown.encode("utf-8")
+
+ timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+ prefix = _filename_part(settings.filename_prefix, "webpage", max_bytes=40)
+ title_part = _filename_part(title, "page", max_bytes=120)
+ filename = normalize_filename(f"{prefix}-{timestamp}-{title_part}.md")
+
+ try:
+ uploads_dir = ensure_uploads_dir(thread_id)
+ file_path, fh = open_upload_file_no_symlink(uploads_dir, filename)
+ with fh:
+ fh.write(data)
+ except UnsafeUploadPathError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ except Exception as exc:
+ logger.exception("Failed to save browser context capture for thread %s", thread_id)
+ raise HTTPException(status_code=500, detail="Failed to save captured page") from exc
+
+ virtual_path = upload_virtual_path(filename)
+ await _sync_to_sandbox_if_needed(thread_id, virtual_path, file_path, data)
+
+ prompt = (body.prompt or "").strip() or DEFAULT_BROWSER_CONTEXT_PROMPT
+ token = uuid4().hex
+ route_base = settings.route_base.rstrip("/") or "/page/workspace"
+ chat_path = f"{route_base}/chats/{thread_id}"
+ query = urlencode(
+ {
+ "browser_context": token,
+ "bc_filename": filename,
+ "bc_size": str(len(data)),
+ "bc_path": virtual_path,
+ "bc_prompt": prompt,
+ }
+ )
+ chat_url = f"#{chat_path}?{query}"
+
+ return BrowserContextCaptureResponse(
+ success=True,
+ thread_id=thread_id,
+ file=BrowserContextFileInfo(
+ filename=filename,
+ size=len(data),
+ path=virtual_path,
+ virtual_path=virtual_path,
+ artifact_url=upload_artifact_url(thread_id, filename),
+ ),
+ chat_path=chat_path,
+ chat_url=chat_url,
+ browser_context_token=token,
+ prompt=prompt,
+ truncated=truncated,
+ message="Captured page saved as a zncm upload",
+ )
+
+
+@router.post("/ask", response_model=BrowserContextAskResponse)
+@require_permission("runs", "create")
+async def ask_browser_context(
+ body: BrowserContextAskRequest,
+ request: Request,
+ config: AppConfig = Depends(get_config),
+) -> BrowserContextAskResponse:
+ settings = _require_enabled(config)
+
+ raw_content = (body.content or "").strip()
+ if not raw_content:
+ raise HTTPException(status_code=400, detail="Captured page content is empty")
+
+ question = (body.question or "").strip()
+ if not question:
+ raise HTTPException(status_code=400, detail="Question is empty")
+
+ title = _trim(body.title, settings.max_title_chars)
+ url = _trim(body.url, settings.max_url_chars)
+ content_limit = max(1_000, min(settings.max_content_chars, settings.tool_result_max_chars))
+ truncated = len(raw_content) > content_limit
+ content = raw_content[:content_limit] if truncated else raw_content
+
+ requested_thread_id = (body.thread_id or "").strip() or None
+ if requested_thread_id:
+ thread_store = get_thread_store(request)
+ allowed = await thread_store.check_access(
+ requested_thread_id,
+ get_effective_user_id(),
+ require_existing=False,
+ )
+ if not allowed:
+ raise HTTPException(status_code=404, detail=f"Thread {requested_thread_id} not found")
+
+ thread_response = await create_thread(
+ ThreadCreateRequest(
+ thread_id=requested_thread_id,
+ assistant_id="lead_agent",
+ metadata={
+ "source": "browser_context_page_assistant",
+ "browser_context_url": url,
+ "browser_context_title": title,
+ },
+ ),
+ request,
+ )
+ thread_id = thread_response.thread_id
+ captured_at = datetime.now(UTC).isoformat()
+ is_follow_up = requested_thread_id is not None
+ if is_follow_up:
+ prompt_prefix = "\n\n".join(
+ [
+ "你是 zncm 页面问答助手。请继续基于本会话里已经读取的网页内容回答用户问题。",
+ "如果本会话里的网页内容不足以回答,请说明缺少哪些信息;不要编造,也不要再尝试抓取其它网页。",
+ f"当前页面标题:{title or '未知标题'}",
+ f"当前页面 URL:{url or '未知 URL'}",
+ f"本轮提问时间:{captured_at}",
+ ]
+ )
+ else:
+ prompt_prefix = _build_ask_prompt(
+ title=title,
+ url=url,
+ content=content,
+ content_type=body.content_type,
+ captured_at=captured_at,
+ truncated=truncated,
+ )
+
+ run_body = RunCreateRequest(
+ assistant_id="lead_agent",
+ input={
+ "messages": [
+ {
+ "type": "human",
+ "content": question,
+ "additional_kwargs": {
+ "prompt_prefix": prompt_prefix,
+ },
+ }
+ ]
+ },
+ metadata={
+ "source": "browser_context_page_assistant",
+ "browser_context_url": url,
+ "browser_context_title": title,
+ "browser_context_truncated": truncated,
+ },
+ stream_mode=["values"],
+ on_disconnect="continue",
+ multitask_strategy="reject",
+ )
+
+ record = await start_run(run_body, thread_id, request)
+ if record.task is not None:
+ try:
+ await record.task
+ except asyncio.CancelledError:
+ pass
+
+ checkpointer = get_checkpointer(request)
+ values: dict[str, Any] = {}
+ try:
+ checkpoint_tuple = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
+ if checkpoint_tuple is not None:
+ checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
+ values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
+ except Exception:
+ logger.exception("Failed to fetch browser context answer for run %s", record.run_id)
+
+ answer = _latest_ai_answer(values)
+ if not answer:
+ answer = record.error or "zncm did not return an answer."
+
+ return BrowserContextAskResponse(
+ success=bool(answer),
+ thread_id=thread_id,
+ run_id=record.run_id,
+ answer=answer,
+ truncated=truncated,
+ message="zncm page assistant answer completed",
+ )
diff --git a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py
new file mode 100644
index 00000000..9512f0f9
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py
@@ -0,0 +1,521 @@
+"""Knowledge base API (``/api/knowledge``).
+
+Sediments threads / searches into an obsidian-wiki compatible vault, mirrors
+metadata into the database, and serves listing, detail, edit, delete, search
+and graph export. The knowledge base is **global / shared** — no per-user
+isolation; writes record ``created_by`` / ``updated_by`` for audit.
+
+All obsidian-wiki specifics live in :mod:`deerflow.knowledge`; this router only
+adapts HTTP ↔ the :class:`~deerflow.knowledge.service.KnowledgeService`.
+"""
+
+from __future__ import annotations
+
+import logging
+import mimetypes
+from typing import Any
+
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
+from fastapi.responses import FileResponse
+from pydantic import BaseModel, Field
+
+from app.gateway.authz import require_auth, require_permission
+from app.gateway.deps import get_checkpointer, get_knowledge_service, get_local_provider, get_thread_store
+from app.gateway.utils import sanitize_log_param
+from deerflow.knowledge.service import EmptyThreadError
+from deerflow.runtime import serialize_channel_values
+from deerflow.runtime.references import build_reference_batches
+from deerflow.runtime.user_context import get_effective_user_id
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
+
+
+# --------------------------------------------------------------------------- #
+# Request / response models
+# --------------------------------------------------------------------------- #
+class IngestThreadRequest(BaseModel):
+ mode: str = Field(default="summary", description="summary | full")
+ title: str | None = None
+ status: str = Field(default="approved", description="draft | approved")
+ tags: list[str] = Field(default_factory=list)
+ include_sources: bool = True
+ background: bool = Field(
+ default=False,
+ description="When true, sediment the thread asynchronously in a background task and return immediately.",
+ )
+ model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置")
+
+
+class BatchIngestRequest(BaseModel):
+ limit: int = Field(default=100, ge=1, le=500)
+ offset: int = Field(default=0, ge=0)
+ status: str = "approved"
+ include_sources: bool = True
+ skip_existing: bool = True
+ mode: str = "summary"
+ model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置")
+
+
+class SearchResultItem(BaseModel):
+ title: str | None = None
+ url: str | None = None
+ snippet: str | None = None
+ tool_name: str | None = None
+ score: float | None = None
+
+
+class IngestSearchRequest(BaseModel):
+ query: str
+ results: list[SearchResultItem] = Field(default_factory=list)
+ thread_id: str | None = None
+ message_id: str | None = None
+ title: str | None = None
+ tags: list[str] = Field(default_factory=list)
+ status: str = "approved"
+
+
+class CreateNoteRequest(BaseModel):
+ title: str
+ content_md: str
+ summary: str | None = None
+ tags: list[str] = Field(default_factory=list)
+ status: str = "approved"
+
+
+class UpdateNoteRequest(BaseModel):
+ title: str | None = None
+ content_md: str | None = None
+ summary: str | None = None
+ tags: list[str] | None = None
+ status: str | None = None
+
+
+class SearchRequest(BaseModel):
+ query: str
+ mode: str = Field(default="keyword", description="keyword | vector | hybrid (vector/hybrid fall back to keyword in v1)")
+ limit: int = Field(default=8, ge=1, le=50)
+
+
+class ResolveWikilinksRequest(BaseModel):
+ targets: list[str] = Field(default_factory=list, description="`[[wikilink]]` targets to resolve to note ids / vault pages.")
+
+
+# --------------------------------------------------------------------------- #
+# Helpers
+# --------------------------------------------------------------------------- #
+async def _read_thread_values(checkpointer, thread_id: str) -> dict[str, Any]:
+ """Return serialized channel values for a thread, or raise 404/500."""
+ config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
+ try:
+ checkpoint_tuple = await checkpointer.aget_tuple(config)
+ except Exception:
+ logger.exception("Failed to read thread %s for knowledge ingest", sanitize_log_param(thread_id))
+ raise HTTPException(status_code=500, detail="Failed to read thread state")
+ if checkpoint_tuple is None:
+ raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
+ checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
+ return serialize_channel_values(checkpoint.get("channel_values", {}) or {})
+
+
+async def _enrich_actor_names(note: dict[str, Any]) -> None:
+ """Resolve created_by/updated_by user ids to a human label (email).
+
+ Adds ``created_by_name`` / ``updated_by_name`` to the note dict so the UI can
+ show a person instead of a raw UUID. Best-effort: unresolved ids fall back to
+ ``None`` and the frontend then shows a shortened id.
+ """
+ provider = get_local_provider()
+ cache: dict[str, str | None] = {}
+
+ async def name_of(uid: str | None) -> str | None:
+ if not uid:
+ return None
+ if uid not in cache:
+ try:
+ user = await provider.get_user(uid)
+ cache[uid] = user.email if user else None
+ except Exception:
+ cache[uid] = None
+ return cache[uid]
+
+ note["created_by_name"] = await name_of(note.get("created_by"))
+ note["updated_by_name"] = await name_of(note.get("updated_by"))
+
+
+def _messages_and_meta(values: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None, list[dict[str, Any]]]:
+ messages = values.get("messages")
+ if not isinstance(messages, list):
+ messages = []
+ title = values.get("title") if isinstance(values.get("title"), str) else None
+ batches = build_reference_batches(messages) if messages else []
+ return messages, title, batches
+
+
+async def _ingest_thread_background(service, checkpointer, thread_id: str, body: IngestThreadRequest, created_by: str | None) -> None:
+ """Sediment a thread into a note without an HTTP context.
+
+ Runs after the response is sent (FastAPI ``BackgroundTasks``); ``created_by``
+ must be captured by the request handler since the user contextvar is gone
+ by the time this executes. All failures are swallowed (logged) — the user
+ has already been told the save was queued.
+ """
+ try:
+ values = await _read_thread_values(checkpointer, thread_id)
+ messages, title, batches = _messages_and_meta(values)
+ if not messages:
+ logger.warning("Background knowledge ingest skipped: thread %s has no messages", sanitize_log_param(thread_id))
+ return
+ await service.capture_thread(
+ thread_id=thread_id,
+ messages=messages,
+ thread_title=title,
+ mode=body.mode,
+ title=body.title,
+ status=body.status,
+ tags=body.tags,
+ include_sources=body.include_sources,
+ reference_batches=batches,
+ created_by=created_by,
+ model_name=body.model_name,
+ )
+ except EmptyThreadError:
+ logger.warning("Background knowledge ingest skipped: thread %s has no sediment-able content", sanitize_log_param(thread_id))
+ except Exception:
+ logger.exception("Background knowledge ingest failed for thread %s", sanitize_log_param(thread_id))
+
+
+# --------------------------------------------------------------------------- #
+# Ingest
+# --------------------------------------------------------------------------- #
+@router.post("/ingest/thread/{thread_id}")
+@require_auth
+@require_permission("threads", "read", owner_check=True)
+async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Request, background_tasks: BackgroundTasks) -> dict[str, Any]:
+ """Sediment a single thread into a knowledge note.
+
+ With ``background=true`` the heavy capture (read checkpoint → extract →
+ write vault + DB) is deferred to a background task and the endpoint returns
+ ``{"queued": true}`` immediately, so the user's "保存" click never blocks.
+ """
+ service = get_knowledge_service(request)
+ checkpointer = get_checkpointer(request)
+ if body.background:
+ # Capture the user id now — the contextvar is gone once the task runs.
+ background_tasks.add_task(
+ _ingest_thread_background,
+ service,
+ checkpointer,
+ thread_id,
+ body,
+ get_effective_user_id(),
+ )
+ return {"queued": True}
+ values = await _read_thread_values(checkpointer, thread_id)
+ messages, title, batches = _messages_and_meta(values)
+ if not messages:
+ raise HTTPException(status_code=400, detail="Thread has no messages to sediment")
+ try:
+ result = await service.capture_thread(
+ thread_id=thread_id,
+ messages=messages,
+ thread_title=title,
+ mode=body.mode,
+ title=body.title,
+ status=body.status,
+ tags=body.tags,
+ include_sources=body.include_sources,
+ reference_batches=batches,
+ created_by=get_effective_user_id(),
+ model_name=body.model_name,
+ )
+ except EmptyThreadError:
+ raise HTTPException(status_code=400, detail="Thread has no sediment-able content")
+ return {
+ "note": result["note"],
+ "created": result["created"],
+ "notes": result.get("notes", [result["note"]]),
+ "llm_fallback": result.get("llm_fallback", False),
+ }
+
+
+@router.post("/ingest/threads/batch")
+@require_auth
+async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> dict[str, Any]:
+ """Batch-sediment the caller's recent threads into knowledge notes."""
+ service = get_knowledge_service(request)
+ checkpointer = get_checkpointer(request)
+ thread_store = get_thread_store(request)
+
+ threads = await thread_store.search(limit=body.limit, offset=body.offset)
+ created = skipped = failed = 0
+ # Notes that landed via the rule-based fallback because the chosen model
+ # produced nothing usable (timeout / API error / non-JSON). Surfaced so the
+ # caller can tell the user "X 条因大模型异常改用了规则抽取".
+ llm_fallback = 0
+ items: list[dict[str, Any]] = []
+ for meta in threads:
+ thread_id = meta.get("thread_id")
+ if not thread_id:
+ continue
+ try:
+ values = await _read_thread_values(checkpointer, thread_id)
+ messages, title, batches = _messages_and_meta(values)
+ if not messages:
+ skipped += 1
+ items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
+ continue
+ result = await service.capture_thread(
+ thread_id=thread_id,
+ messages=messages,
+ thread_title=title or meta.get("display_name"),
+ mode=body.mode,
+ status=body.status,
+ include_sources=body.include_sources,
+ reference_batches=batches,
+ created_by=get_effective_user_id(),
+ skip_existing=body.skip_existing,
+ model_name=body.model_name,
+ )
+ if result.get("llm_fallback"):
+ llm_fallback += 1
+ if result["created"]:
+ created += 1
+ items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "created", "llm_fallback": bool(result.get("llm_fallback"))})
+ else:
+ skipped += 1
+ items.append({"thread_id": thread_id, "note_id": result["note"]["id"], "status": "skipped"})
+ except EmptyThreadError:
+ skipped += 1
+ items.append({"thread_id": thread_id, "note_id": None, "status": "skipped"})
+ except HTTPException:
+ failed += 1
+ items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
+ except Exception:
+ logger.exception("Batch ingest failed for thread %s", sanitize_log_param(str(thread_id)))
+ failed += 1
+ items.append({"thread_id": thread_id, "note_id": None, "status": "failed"})
+
+ return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "llm_fallback": llm_fallback, "items": items}
+
+
+@router.post("/ingest/search")
+@require_auth
+async def ingest_search(body: IngestSearchRequest, request: Request) -> dict[str, Any]:
+ """Sediment search/tool results as a reference note."""
+ service = get_knowledge_service(request)
+ note = await service.capture_search(
+ query=body.query,
+ results=[r.model_dump() for r in body.results],
+ thread_id=body.thread_id,
+ message_id=body.message_id,
+ title=body.title,
+ tags=body.tags,
+ status=body.status,
+ created_by=get_effective_user_id(),
+ )
+ return {"note": note}
+
+
+# --------------------------------------------------------------------------- #
+# CRUD
+# --------------------------------------------------------------------------- #
+@router.post("/notes")
+@require_auth
+async def create_note(body: CreateNoteRequest, request: Request) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ note = await service.create_manual_note(
+ title=body.title,
+ content_md=body.content_md,
+ summary=body.summary,
+ tags=body.tags,
+ status=body.status,
+ created_by=get_effective_user_id(),
+ )
+ return {"note": note}
+
+
+@router.get("/notes")
+@require_auth
+async def list_notes(
+ request: Request,
+ keyword: str | None = None,
+ tag: str | None = None,
+ source_type: str | None = None,
+ status: str | None = None,
+ limit: int = 20,
+ offset: int = 0,
+) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ limit = max(1, min(limit, 100))
+ offset = max(0, offset)
+ return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
+
+
+@router.get("/notes/{note_id}")
+@require_auth
+async def get_note(note_id: str, request: Request) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ note = await service.get_note(note_id)
+ if note is None:
+ raise HTTPException(status_code=404, detail="Knowledge note not found")
+ await _enrich_actor_names(note)
+ return note
+
+
+@router.put("/notes/{note_id}")
+@require_auth
+async def update_note(note_id: str, body: UpdateNoteRequest, request: Request) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ note = await service.update_note(
+ note_id,
+ title=body.title,
+ content_md=body.content_md,
+ summary=body.summary,
+ tags=body.tags,
+ status=body.status,
+ updated_by=get_effective_user_id(),
+ )
+ if note is None:
+ raise HTTPException(status_code=404, detail="Knowledge note not found")
+ return note
+
+
+@router.delete("/notes/{note_id}")
+@require_auth
+async def delete_note(note_id: str, request: Request, hard: bool = False) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ ok = await service.delete_note(note_id, updated_by=get_effective_user_id(), hard=hard)
+ if not ok:
+ raise HTTPException(status_code=404, detail="Knowledge note not found")
+ return {"ok": True, "note_id": note_id}
+
+
+# --------------------------------------------------------------------------- #
+# Search & graph
+# --------------------------------------------------------------------------- #
+@router.post("/search")
+@require_auth
+async def search(body: SearchRequest, request: Request) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ items = await service.search(body.query, mode=body.mode, limit=body.limit)
+ return {"items": items}
+
+
+# --------------------------------------------------------------------------- #
+# Wikilinks ([[...]] resolution + vault meta-page viewer)
+# --------------------------------------------------------------------------- #
+@router.post("/resolve")
+@require_auth
+async def resolve_wikilinks(body: ResolveWikilinksRequest, request: Request) -> dict[str, Any]:
+ """Resolve ``[[wikilink]]`` targets to navigable destinations.
+
+ Each target resolves to a knowledge note (``kind=note``), a vault meta/hub
+ page (``kind=vault``), or nothing (``kind=missing``). Powers clickable
+ wikilinks in the knowledge UI.
+ """
+ service = get_knowledge_service(request)
+ return {"resolved": await service.resolve_wikilinks(body.targets)}
+
+
+@router.get("/notes/{note_id}/wikilinks")
+@require_auth
+async def resolve_note_wikilinks(note_id: str, request: Request) -> dict[str, Any]:
+ """Resolve every ``[[wikilink]]`` found in a note's body in one call."""
+ service = get_knowledge_service(request)
+ resolved = await service.resolve_note_wikilinks(note_id)
+ if resolved is None:
+ raise HTTPException(status_code=404, detail="Knowledge note not found")
+ return {"resolved": resolved}
+
+
+@router.get("/vault-page")
+@require_auth
+async def get_vault_page(request: Request, path: str) -> dict[str, Any]:
+ """Return a raw vault Markdown meta/hub page (no DB note) by its path."""
+ service = get_knowledge_service(request)
+ page = service.read_vault_page(path)
+ if page is None:
+ raise HTTPException(status_code=404, detail="Vault page not found")
+ return page
+
+
+class MergeRequest(BaseModel):
+ primary_id: str
+ duplicate_ids: list[str] = Field(default_factory=list)
+
+
+@router.get("/duplicates")
+@require_auth
+async def list_duplicates(request: Request) -> dict[str, Any]:
+ """Return suggested duplicate groups (phase 3, heuristic)."""
+ service = get_knowledge_service(request)
+ return {"groups": await service.find_duplicates()}
+
+
+@router.post("/merge")
+@require_auth
+async def merge_notes(body: MergeRequest, request: Request) -> dict[str, Any]:
+ """Merge duplicate notes into a primary note (sources moved, dups archived)."""
+ service = get_knowledge_service(request)
+ note = await service.merge_notes(body.primary_id, body.duplicate_ids, updated_by=get_effective_user_id())
+ if note is None:
+ raise HTTPException(status_code=404, detail="Primary note not found")
+ return {"note": note}
+
+
+@router.post("/reindex")
+@require_auth
+async def reindex_embeddings(request: Request) -> dict[str, Any]:
+ """Rebuild the vector index for all notes (maintenance; no-op if embeddings off)."""
+ service = get_knowledge_service(request)
+ return await service.reindex_all()
+
+
+@router.get("/notes/{note_id}/versions")
+@require_auth
+async def list_note_versions(note_id: str, request: Request) -> dict[str, Any]:
+ """Return the edit history (pre-update snapshots) for a note."""
+ service = get_knowledge_service(request)
+ return {"items": await service.list_versions(note_id)}
+
+
+@router.get("/graph")
+@require_auth
+async def get_graph(request: Request) -> dict[str, Any]:
+ """Return product knowledge-graph data (notes + entities + relations)."""
+ service = get_knowledge_service(request)
+ return await service.get_graph()
+
+
+@router.post("/export/graph")
+@require_auth
+async def export_graph(request: Request) -> dict[str, Any]:
+ service = get_knowledge_service(request)
+ result = await service.export_graph()
+ base = "/api/knowledge/export/files"
+ return {
+ "ok": True,
+ "files": {
+ "graph_json": f"{base}/graph.json",
+ "graph_html": f"{base}/graph.html",
+ "graph_graphml": f"{base}/graph.graphml",
+ "cypher": f"{base}/cypher.txt",
+ },
+ "stats": result["stats"],
+ }
+
+
+@router.get("/export/files/{filename}")
+@require_auth
+async def export_file(filename: str, request: Request) -> FileResponse:
+ service = get_knowledge_service(request)
+ path = service.export_file_path(filename)
+ if path is None:
+ # Lazily build the export on first access so the iframe never 404s.
+ await service.export_graph()
+ path = service.export_file_path(filename)
+ if path is None:
+ raise HTTPException(status_code=404, detail="Export file not found")
+ media_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
+ return FileResponse(path, media_type=media_type)
diff --git a/offline-backend-20260512/backend/app/gateway/routers/public_leaderboard.py b/offline-backend-20260512/backend/app/gateway/routers/public_leaderboard.py
new file mode 100644
index 00000000..b856d6e2
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/routers/public_leaderboard.py
@@ -0,0 +1,97 @@
+"""Public (unauthenticated) read-only user leaderboard.
+
+Mounted under the ``/api/public/`` prefix so the auth middleware lets this route
+through without a session cookie. Powers the public "publish" page for the user
+leaderboard — a view-only snapshot with no admin controls and no mutations.
+
+The snapshot honors the same system-wide leaderboard settings (excluded users,
+scheduled/admin/failed inclusion) as the authenticated admin endpoint, so any
+accounts an admin chose to hide stay hidden on the public page as well.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+
+from fastapi import APIRouter, HTTPException, Query
+
+from app.gateway.routers.admin_users import (
+ AdminLeaderboardResponse,
+ _load_or_schedule_daily_leaderboard,
+ _resolve_range,
+)
+from deerflow.config.system_settings import load_system_settings
+from deerflow.persistence.engine import get_session_factory
+
+logger = logging.getLogger(__name__)
+
+# Whitelisted under ``auth_middleware._PUBLIC_PATH_PREFIXES`` (``/api/public/``).
+router = APIRouter(prefix="/api/public/leaderboard", tags=["public-leaderboard"])
+
+
+def _mask_email(email: str | None) -> str:
+ """Reduce an email to its local part so the public page can show a
+ recognizable handle without leaking the full address (domain).
+
+ The leaderboard only makes sense if it names its top users, so the local
+ part is kept intentionally; the privacy gain is dropping the domain and the
+ raw account id from the unauthenticated payload.
+ """
+ if not email:
+ return ""
+ return email.split("@", 1)[0].strip() or ""
+
+
+def _anonymize(resp: AdminLeaderboardResponse) -> AdminLeaderboardResponse:
+ """Strip PII from a leaderboard snapshot before it leaves over the public,
+ unauthenticated endpoint: email → local part, account/thread ids → opaque
+ short hashes (kept stable so they still work as React keys)."""
+
+ def hashed(value: str | None) -> str:
+ if not value:
+ return ""
+ return hashlib.sha1(value.encode("utf-8")).hexdigest()[:12]
+
+ seen_users: set[int] = set()
+ for bucket in (resp.users_by_activity, resp.users_by_questions, resp.users_by_tokens):
+ for user in bucket:
+ if id(user) in seen_users:
+ continue
+ seen_users.add(id(user))
+ user.email = _mask_email(user.email)
+ user.user_id = hashed(user.user_id)
+
+ for bucket in (resp.recent_questions, resp.frequent_questions):
+ for question in bucket:
+ question.email = _mask_email(question.email)
+ question.user_id = hashed(question.user_id)
+ question.thread_id = ""
+ question.run_id = ""
+
+ return resp
+
+
+@router.get("", response_model=AdminLeaderboardResponse)
+@router.get("/", response_model=AdminLeaderboardResponse)
+async def get_public_leaderboard(
+ days: int = Query(default=30, ge=1, le=365),
+ since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
+ until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
+ limit: int = Query(default=20, ge=5, le=100),
+) -> AdminLeaderboardResponse:
+ """Return a read-only leaderboard snapshot for the public publish page."""
+ session_factory = get_session_factory()
+ if session_factory is None:
+ raise HTTPException(status_code=503, detail="Analytics requires database persistence")
+
+ settings = load_system_settings().leaderboard
+ since_dt, until_dt, _range_days = _resolve_range(days, since, until)
+ resp = await _load_or_schedule_daily_leaderboard(
+ session_factory,
+ settings,
+ since_dt,
+ until_dt,
+ limit=limit,
+ )
+ return _anonymize(resp)
diff --git a/offline-backend-20260512/backend/app/gateway/routers/skills.py b/offline-backend-20260512/backend/app/gateway/routers/skills.py
index b1c9ffe8..09f277da 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/skills.py
+++ b/offline-backend-20260512/backend/app/gateway/routers/skills.py
@@ -56,6 +56,44 @@ def _is_admin_user(request: Request) -> bool:
return getattr(user, "system_role", None) == "admin"
+def _email_local_part(email: str | None) -> str | None:
+ """Return the username portion (before ``@``) of an email address."""
+ if not email:
+ return None
+ at = email.find("@")
+ return email[:at] if at > 0 else email
+
+
+async def _attach_owner_usernames(responses: list["SkillResponse"]) -> list["SkillResponse"]:
+ """Resolve each custom skill's ``owner_user_id`` to a friendly username.
+
+ One lookup per distinct owner id; tolerates deleted accounts and an
+ uninitialised auth provider by leaving ``owner_username`` as None.
+ """
+ owner_ids = {r.owner_user_id for r in responses if r.owner_user_id}
+ if not owner_ids:
+ return responses
+ try:
+ from app.gateway.deps import get_local_provider
+
+ provider = get_local_provider()
+ except Exception:
+ return responses
+ names: dict[str, str] = {}
+ for uid in owner_ids:
+ try:
+ user = await provider.get_user(uid)
+ except Exception:
+ user = None
+ local = _email_local_part(getattr(user, "email", None)) if user is not None else None
+ if local:
+ names[uid] = local
+ for resp in responses:
+ if resp.owner_user_id:
+ resp.owner_username = names.get(resp.owner_user_id)
+ return responses
+
+
def _can_view_skill_source(
category: SkillCategory,
owner_user_id: str | None,
@@ -111,6 +149,7 @@ class SkillResponse(BaseModel):
"""Response model for skill information."""
name: str = Field(..., description="Name of the skill")
+ name_zh: str | None = Field(default=None, description="用户配置或大模型生成的中文名称")
description: str = Field(..., description="Description of what the skill does")
license: str | None = Field(None, description="License information")
category: SkillCategory = Field(..., description="Category of the skill (public or custom)")
@@ -125,6 +164,7 @@ class SkillResponse(BaseModel):
featured: bool = Field(default=False, description="Whether this skill is pinned to the top of the list (sort only)")
source: str = Field(default="upload", description="Origin: 'agent' for agent-created, 'upload' for user-uploaded")
owner_user_id: str | None = Field(default=None, description="Custom skill owner (null = built-in or legacy)")
+ owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in skills.")
published: bool = Field(default=False, description="Whether the owner has made this skill visible to other users")
builtin: bool = Field(default=False, description="True when owner_user_id is null (no human owner)")
detail: str | None = Field(default=None, description="Human-authored long-form detail text (Markdown), editable by owner/admin")
@@ -267,11 +307,13 @@ def _skill_to_response(
published = False
builtin = skill.category != SkillCategory.CUSTOM
detail: str | None = None
+ name_zh: str | None = None
if ownership is not None:
owner_user_id = ownership.get("owner_user_id")
published = bool(ownership.get("published", False))
builtin = bool(ownership.get("builtin", owner_user_id is None))
detail = ownership.get("detail")
+ name_zh = ownership.get("name_zh")
display = None if display_override is _UNSET else display_override
if display_override is _UNSET:
@@ -284,6 +326,7 @@ def _skill_to_response(
return SkillResponse(
name=skill.name,
+ name_zh=name_zh,
description=skill.description,
license=skill.license,
category=skill.category,
@@ -697,7 +740,8 @@ async def _attach_and_filter_tags(
resp.tags = assignments.get(resp.name, [])
if search:
needle = search.strip().lower()
- responses = [r for r in responses if needle in r.name.lower()]
+ # Match the English name OR the Chinese display name so 中文 search works.
+ responses = [r for r in responses if needle in r.name.lower() or (r.name_zh or "").lower().find(needle) >= 0]
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
@@ -726,7 +770,7 @@ async def list_skills(
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
tag_store: TagStore = Depends(get_tag_store),
- search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name"),
+ search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name or Chinese name (name_zh)"),
tag_ids: list[str] | None = Query(default=None, description="Keep skills carrying any of these tags (OR)"),
sort: str = Query(
default="time_desc",
@@ -786,6 +830,7 @@ async def list_skills(
)
)
responses = await _attach_and_filter_tags(tag_store, responses, search, tag_ids)
+ responses = await _attach_owner_usernames(responses)
responses = _sort_skill_responses(responses, sort)
return SkillsListResponse(skills=responses)
except HTTPException:
@@ -1176,6 +1221,7 @@ async def list_custom_skills(
display_sample=adapter.get("sample") if adapter else None,
)
)
+ responses = await _attach_owner_usernames(responses)
return SkillsListResponse(skills=responses)
except Exception as e:
logger.error("Failed to list custom skills: %s", e, exc_info=True)
@@ -2309,6 +2355,324 @@ async def update_skill_detail(
raise HTTPException(status_code=500, detail=f"Failed to update skill detail: {str(e)}")
+# ---------------------------------------------------------------------------
+# 技能中文名称(用户可配置 + 大模型自动生成 + 一键补充)
+# ---------------------------------------------------------------------------
+
+_SKILL_NAME_ZH_SYSTEM_PROMPT = (
+ "你是一名中文产品命名助手。根据给定技能的英文名与功能描述,生成一个简洁、自然、"
+ "易懂的中文名称。要求:2 到 10 个汉字;准确概括技能用途;不要包含标点、引号、英文、"
+ "解释或多余文字;只输出中文名称本身。"
+)
+
+
+class SkillNameZhModelError(Exception):
+ """The LLM call backing skill-name generation failed (a *model-side* problem).
+
+ Distinguishes a genuine model failure (timeout / API error / unbuildable
+ model) — which must be surfaced to the user as "大模型报错导致无法补全" — from
+ the model merely returning an unusable name (callers get ``None`` instead).
+ """
+
+
+def _clean_generated_name_zh(raw: str) -> str | None:
+ """Normalise an LLM-produced Chinese name: first line, strip quotes/punct, cap length."""
+ if not raw:
+ return None
+ text = str(raw).strip()
+ # Take the first non-empty line only.
+ for line in text.splitlines():
+ line = line.strip()
+ if line:
+ text = line
+ break
+ # Strip wrapping quotes / common trailing punctuation.
+ text = text.strip().strip("\"'“”‘’`「」《》【】()()[]").strip()
+ text = text.rstrip("。.,,、!!??;;::")
+ if not text:
+ return None
+ # Keep it short — guard against the model returning a sentence.
+ if len(text) > 20:
+ text = text[:20]
+ return text or None
+
+
+def _resolve_skill_name_zh_model_name(app_config: AppConfig, model_name: str | None) -> str | None:
+ """Pick the model name for skill-name generation.
+
+ An explicit ``model_name`` (user-selected in the UI) wins; otherwise fall
+ back to the curator/security-scan model setting (itself falling back to the
+ main model when unset).
+ """
+ if model_name and model_name.strip():
+ return model_name.strip()
+ if app_config.skill_evolution and app_config.skill_evolution.curator:
+ return app_config.skill_evolution.curator.model_name
+ return None
+
+
+async def _generate_skill_name_zh(skill: Skill, app_config: AppConfig, model: Any = None, *, model_name: str | None = None) -> str | None:
+ """Generate a Chinese display name for ``skill`` via the LLM.
+
+ An explicit ``model_name`` lets the caller pick the model; otherwise it
+ reuses the curator/security-scan model setting (falls back to the main
+ model). Returns ``None`` only when the model replies but the output cannot be
+ normalised into a short Chinese name. Raises :class:`SkillNameZhModelError`
+ when the model itself fails (cannot be built / times out / API error), so the
+ caller can tell the user it was a *model* problem.
+ """
+ user_prompt = (
+ f"技能英文名:{skill.name}\n"
+ f"功能描述:{skill.description or '(无描述)'}\n\n"
+ "请为该技能起一个中文名称。"
+ )
+ if model is None:
+ try:
+ from deerflow.models import create_chat_model
+
+ resolved = _resolve_skill_name_zh_model_name(app_config, model_name)
+ model = create_chat_model(name=resolved, thinking_enabled=False, app_config=app_config)
+ except Exception as e:
+ logger.warning("Skill name_zh model build failed for %s", skill.name, exc_info=True)
+ raise SkillNameZhModelError(str(e) or "无法初始化所选大模型") from e
+ try:
+ response = await asyncio.wait_for(
+ model.ainvoke(
+ [
+ {"role": "system", "content": _SKILL_NAME_ZH_SYSTEM_PROMPT},
+ {"role": "user", "content": user_prompt},
+ ],
+ config={"run_name": "skill_name_zh"},
+ ),
+ timeout=60,
+ )
+ except asyncio.TimeoutError as e:
+ logger.warning("Skill name_zh LLM call timed out for %s", skill.name)
+ raise SkillNameZhModelError("大模型响应超时(60 秒)") from e
+ except Exception as e:
+ logger.warning("Skill name_zh LLM call failed for %s", skill.name, exc_info=True)
+ raise SkillNameZhModelError(str(e) or "大模型调用失败") from e
+ return _clean_generated_name_zh(str(getattr(response, "content", "") or ""))
+
+
+async def _persist_skill_name_zh(
+ skill: Skill,
+ name_zh: str | None,
+ skill_store: SkillStore,
+ user_id: str,
+ *,
+ is_admin: bool,
+) -> str | None:
+ """Persist ``name_zh`` into the ``skills`` DB table (never the on-disk SKILL.md).
+
+ Authorization mirrors the ``detail`` annotation: a custom skill is editable by
+ its owner or an admin; a built-in / public skill only by an admin (its DB row
+ is materialized on demand). Returns the stored value.
+ """
+ if skill.category == SkillCategory.CUSTOM:
+ _record, mode = await _get_editable_skill_or_raise(skill_store, skill.name, user_id, is_admin=is_admin)
+ if mode == "admin":
+ updated = await skill_store.update_admin(skill.name, {"name_zh": name_zh})
+ else:
+ updated = await skill_store.update(skill.name, user_id, {"name_zh": name_zh})
+ else:
+ if not is_admin:
+ raise HTTPException(status_code=403, detail="Only admin users can edit built-in skill Chinese names")
+ if await skill_store.get_any(skill.name) is None:
+ await skill_store.ensure_legacy({"name": skill.name, "owner_user_id": None, "published": True})
+ updated = await skill_store.update_admin(skill.name, {"name_zh": name_zh})
+
+ if updated is None:
+ raise HTTPException(status_code=500, detail="Failed to persist skill Chinese name")
+ return updated.get("name_zh")
+
+
+class SkillNameZhRequest(BaseModel):
+ """Request body for setting a skill's Chinese display name."""
+
+ name_zh: str = Field(default="", description="中文名称;传空字符串表示清空")
+
+
+class SkillNameZhResponse(BaseModel):
+ """Response model for a single skill's Chinese-name update."""
+
+ name: str = Field(..., description="Skill name")
+ name_zh: str | None = Field(default=None, description="中文名称(为空表示已清空)")
+
+
+class SkillNameZhBulkResponse(BaseModel):
+ """Response model for the one-click bulk fill of missing Chinese names."""
+
+ generated: list[SkillNameZhResponse] = Field(default_factory=list, description="本次成功生成中文名的技能")
+ failed: list[str] = Field(default_factory=list, description="生成失败的技能英文名")
+ skipped: int = Field(default=0, description="已有中文名而被跳过的技能数")
+ model_error: str | None = Field(default=None, description="若有技能因大模型报错而失败,这里给出代表性的报错信息")
+
+
+class GenerateSkillNameZhRequest(BaseModel):
+ """Optional body for skill-name generation: pick the model to use."""
+
+ model_name: str | None = Field(default=None, description="可选:指定用于生成的大模型名称;留空使用默认配置")
+
+
+def _can_edit_skill_name_zh(skill: Skill, record: dict[str, Any] | None, user_id: str, *, is_admin: bool) -> bool:
+ """Whether ``user_id`` may set this skill's Chinese name (mirrors detail auth)."""
+ if is_admin:
+ return True
+ if skill.category != SkillCategory.CUSTOM:
+ return False
+ return record is not None and record.get("owner_user_id") == user_id
+
+
+@router.put(
+ "/skills/{skill_name}/name-zh",
+ response_model=SkillNameZhResponse,
+ summary="Set Skill Chinese Name",
+ description="设置技能的中文名称(存数据库,不改源文件)。自定义技能由其所有者或管理员编辑;内置/公开技能仅管理员可编辑。传空字符串表示清空,回退到使用英文名。",
+)
+async def set_skill_name_zh(
+ skill_name: str,
+ body: SkillNameZhRequest,
+ request: Request,
+ config: AppConfig = Depends(get_config),
+ skill_store: SkillStore = Depends(get_skill_store),
+) -> SkillNameZhResponse:
+ try:
+ skill_name = skill_name.replace("\r\n", "").replace("\n", "")
+ user_id = _current_user_id(request)
+ is_admin = _is_admin_user(request)
+ skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
+ skill = next((s for s in skills if s.name == skill_name), None)
+ if skill is None:
+ raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
+ name_zh = body.name_zh.strip() or None
+ stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
+ return SkillNameZhResponse(name=skill_name, name_zh=stored)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error("Failed to set name_zh for %s: %s", skill_name, e, exc_info=True)
+ raise HTTPException(status_code=500, detail=f"Failed to set skill Chinese name: {str(e)}")
+
+
+@router.post(
+ "/skills/{skill_name}/name-zh/generate",
+ response_model=SkillNameZhResponse,
+ summary="Generate Skill Chinese Name",
+ description="用大模型为该技能生成一个中文名称并保存到数据库。可在请求体里通过 model_name 指定使用的大模型。",
+)
+async def generate_skill_name_zh(
+ skill_name: str,
+ request: Request,
+ body: GenerateSkillNameZhRequest | None = None,
+ config: AppConfig = Depends(get_config),
+ skill_store: SkillStore = Depends(get_skill_store),
+) -> SkillNameZhResponse:
+ try:
+ body = body or GenerateSkillNameZhRequest()
+ skill_name = skill_name.replace("\r\n", "").replace("\n", "")
+ user_id = _current_user_id(request)
+ is_admin = _is_admin_user(request)
+ skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
+ skill = next((s for s in skills if s.name == skill_name), None)
+ if skill is None:
+ raise HTTPException(status_code=404, detail=f"Skill '{skill_name}' not found")
+ try:
+ name_zh = await _generate_skill_name_zh(skill, config, model_name=body.model_name)
+ except SkillNameZhModelError as e:
+ raise HTTPException(status_code=502, detail=f"大模型报错,无法补全技能中文名:{e}")
+ if not name_zh:
+ raise HTTPException(status_code=502, detail="大模型未能生成有效的中文名称,请稍后重试或手动填写")
+ stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
+ return SkillNameZhResponse(name=skill_name, name_zh=stored)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error("Failed to generate name_zh for %s: %s", skill_name, e, exc_info=True)
+ raise HTTPException(status_code=500, detail=f"Failed to generate skill Chinese name: {str(e)}")
+
+
+@router.post(
+ "/skills/name-zh/generate-missing",
+ response_model=SkillNameZhBulkResponse,
+ summary="Bulk Generate Missing Skill Chinese Names",
+ description="管理员一键为库中所有尚未配置中文名称的技能生成中文名(存数据库)。可在请求体里通过 model_name 指定使用的大模型。",
+)
+async def generate_missing_skill_names_zh(
+ request: Request,
+ body: GenerateSkillNameZhRequest | None = None,
+ config: AppConfig = Depends(get_config),
+ skill_store: SkillStore = Depends(get_skill_store),
+) -> SkillNameZhBulkResponse:
+ try:
+ from deerflow.models import create_chat_model
+
+ body = body or GenerateSkillNameZhRequest()
+ user_id = _current_user_id(request)
+ is_admin = _is_admin_user(request)
+ # 一键补充技能中文名仅对管理员开放。
+ if not is_admin:
+ raise HTTPException(status_code=403, detail="仅管理员可一键补充技能中文名")
+ skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
+
+ # Admin sees every row, so existing name_zh values come from the full set.
+ records = {r["name"]: r for r in await skill_store.list_all()}
+
+ pending: list[Skill] = []
+ skipped = 0
+ for skill in skills:
+ record = records.get(skill.name)
+ if record and record.get("name_zh"):
+ skipped += 1
+ continue
+ pending.append(skill)
+
+ generated: list[SkillNameZhResponse] = []
+ failed: list[str] = []
+ model_error: str | None = None
+ if pending:
+ # Build the model once and reuse it for every skill. A build failure
+ # is a model problem — fail fast with a clear message rather than
+ # silently producing zero results.
+ try:
+ resolved_name = _resolve_skill_name_zh_model_name(config, body.model_name)
+ model = create_chat_model(name=resolved_name, thinking_enabled=False, app_config=config)
+ except Exception as e:
+ logger.warning("Failed to build model for bulk name_zh generation", exc_info=True)
+ raise HTTPException(status_code=502, detail=f"大模型报错,无法补全技能中文名:{str(e) or '无法初始化所选大模型'}")
+
+ # Generate concurrently with a small bound to avoid overloading the model.
+ semaphore = asyncio.Semaphore(4)
+
+ async def _one(skill: Skill) -> tuple[Skill, str | None, str | None]:
+ async with semaphore:
+ try:
+ return skill, await _generate_skill_name_zh(skill, config, model=model), None
+ except SkillNameZhModelError as e:
+ return skill, None, str(e)
+
+ results = await asyncio.gather(*[_one(s) for s in pending])
+ for skill, name_zh, err in results:
+ if err and model_error is None:
+ model_error = err
+ if not name_zh:
+ failed.append(skill.name)
+ continue
+ try:
+ stored = await _persist_skill_name_zh(skill, name_zh, skill_store, user_id, is_admin=is_admin)
+ generated.append(SkillNameZhResponse(name=skill.name, name_zh=stored))
+ except HTTPException:
+ # Permission / persistence issue for this one skill — skip it.
+ failed.append(skill.name)
+
+ return SkillNameZhBulkResponse(generated=generated, failed=failed, skipped=skipped, model_error=model_error)
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error("Failed to bulk-generate skill Chinese names: %s", e, exc_info=True)
+ raise HTTPException(status_code=500, detail=f"Failed to bulk-generate skill Chinese names: {str(e)}")
+
+
class SkillPinRequest(BaseModel):
pinned: bool = Field(..., description="Whether to pin or unpin the skill")
diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml
index d95031b0..5ec643d9 100644
--- a/offline-backend-20260512/backend/config.yaml
+++ b/offline-backend-20260512/backend/config.yaml
@@ -10,90 +10,90 @@ gateway:
port: 8001
models:
- - name: deepseek-chat
- display_name: DeepSeek Chat
- use: langchain_openai:ChatOpenAI
- model: deepseek-chat
- api_key: sk-3fee562f834a4c08bb0d471ef44cb445
- base_url: https://api.deepseek.com
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: false
- supports_thinking: true
+ # - name: deepseek-chat
+ # display_name: DeepSeek Chat
+ # use: langchain_openai:ChatOpenAI
+ # model: deepseek-chat
+ # api_key: sk-1520028668cb4cc388ed8315ff1e856e
+ # base_url: https://api.deepseek.com
+ # request_timeout: 600.0
+ # max_retries: 2
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: false
+ # supports_thinking: true
- - name: deepseek-v4-flash
- display_name: DeepSeek V4 Flash
- use: langchain_openai:ChatOpenAI
- model: deepseek-v4-flash
- api_key: sk-3fee562f834a4c08bb0d471ef44cb445
- base_url: https://api.deepseek.com
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: false
- supports_thinking: true
+ # - name: deepseek-v4-flash
+ # display_name: DeepSeek V4 Flash
+ # use: langchain_openai:ChatOpenAI
+ # model: deepseek-v4-flash
+ # api_key: sk-e1c22b7931dc446f8b67e57878edd105
+ # base_url: https://api.deepseek.com
+ # request_timeout: 600.0
+ # max_retries: 2
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: false
+ # supports_thinking: true
- - name: deepseek-v4-pro
- display_name: DeepSeek V4 Pro
+ # - name: deepseek-v4-pro
+ # display_name: DeepSeek V4 Pro
+ # use: langchain_openai:ChatOpenAI
+ # model: deepseek-v4-pro
+ # api_key: sk-e1c22b7931dc446f8b67e57878edd105
+ # base_url: https://api.deepseek.com
+ # request_timeout: 600.0
+ # max_retries: 2
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: false
+ # supports_thinking: true
+ # - name: MiniMax-M2.7
+ # display_name: MiniMax-M2.7
+ # use: langchain_openai:ChatOpenAI
+ # model: MiniMax-M2.7
+ # api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
+ # base_url: https://api.minimaxi.com/v1
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: true
+ # supports_thinking: true
+ # - name: glm-5.1
+ # display_name: glm-5.1
+ # use: langchain_openai:ChatOpenAI
+ # model: glm-5.1
+ # api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR
+ # base_url: https://open.bigmodel.cn/api/paas/v4/
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: true
+ # supports_thinking: true
+ # - name: deepseek-ai/DeepSeek-OCR
+ # display_name: deepseek-ai/DeepSeek-OCR
+ # use: langchain_openai:ChatOpenAI
+ # model: deepseek-ai/DeepSeek-OCR
+ # api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
+ # base_url: https://api.siliconflow.cn/v1/
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: true
+ # supports_thinking: true
+ # - name: zai-org/GLM-5-FP8
+ # display_name: zai-org/GLM-5-FP8
+ # use: langchain_openai:ChatOpenAI
+ # model: zai-org/GLM-5-FP8
+ # api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
+ # base_url: https://api.us-west-2.modal.direct/v1
+ # request_timeout: 600.0
+ # max_retries: 2
+ # max_tokens: 4096
+ # temperature: 1.0
+ # supports_vision: true
+ # supports_thinking: true
+ - name: deepseek-ai/DeepSeek-V4-Flash
+ display_name: deepseek-ai/DeepSeek-V4-Flash
use: langchain_openai:ChatOpenAI
- model: deepseek-v4-pro
- api_key: sk-e1c22b7931dc446f8b67e57878edd105
- base_url: https://api.deepseek.com
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: false
- supports_thinking: true
- - name: MiniMax-M2.7
- display_name: MiniMax-M2.7
- use: langchain_openai:ChatOpenAI
- model: MiniMax-M2.7
- api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
- base_url: https://api.minimaxi.com/v1
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
- - name: glm-5.1
- display_name: glm-5.1
- use: langchain_openai:ChatOpenAI
- model: glm-5.1
- api_key: 75121e4bed7640d5b71ac95cf8b4310a.RA3GyRBAR3kv7VrR
- base_url: https://open.bigmodel.cn/api/paas/v4/
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
- - name: deepseek-ai/DeepSeek-OCR
- display_name: deepseek-ai/DeepSeek-OCR
- use: langchain_openai:ChatOpenAI
- model: deepseek-ai/DeepSeek-OCR
- api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
- base_url: https://api.siliconflow.cn/v1/
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
- - name: zai-org/GLM-5-FP8
- display_name: zai-org/GLM-5-FP8
- use: langchain_openai:ChatOpenAI
- model: zai-org/GLM-5-FP8
- api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
- base_url: https://api.us-west-2.modal.direct/v1
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
- - name: Qwen/Qwen3.6-27B
- display_name: Qwen/Qwen3.6-27B
- use: langchain_openai:ChatOpenAI
- model: Qwen/Qwen3.6-27B
+ model: deepseek-ai/DeepSeek-V4-Flash
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
base_url: https://api.siliconflow.cn/v1/
max_tokens: 4096
@@ -122,6 +122,26 @@ uploads:
auto_convert_documents: true
pdf_converter: auto
+# Chrome 插件「当前网页发送到 Deer」入口。默认关闭,打开后才启用
+# /api/browser-context/capture。它只保存当前页 Markdown 到线程 uploads,
+# 不会改变普通聊天、上传或工具调用流程。
+browser_context:
+ enabled: true
+ max_content_chars: 200000
+ max_title_chars: 300
+ max_url_chars: 2048
+ filename_prefix: webpage
+ route_base: /page/workspace
+ task_result_timeout_seconds: 60
+ task_poll_interval_seconds: 1
+ task_lease_seconds: 120
+ tool_result_max_chars: 60000
+ # 交互式页面操作(browser_act:点击/输入/滚动/等待/跳转)。关闭后只保留只读抓取。
+ actions_enabled: true
+ action_result_timeout_seconds: 45
+ max_observed_elements: 150
+ observation_text_max_chars: 12000
+
title:
enabled: true
max_words: 6
@@ -139,6 +159,7 @@ title:
tool_groups:
- name: web
+ - name: browser
- name: file:read
- name: file:write
- name: bash
@@ -154,6 +175,14 @@ tools:
use: deerflow.community.jina_ai.tools:web_fetch_tool
timeout: 15
+ - name: browser_fetch_page
+ group: browser
+ use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
+
+ - name: browser_act
+ group: browser
+ use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
+
- name: ls
group: file:read
use: deerflow.sandbox.tools:ls_tool
@@ -223,7 +252,7 @@ auth_login:
database:
backend: sqlite
- sqlite_dir: .deer-flow/data
+ sqlite_dir: /app/backend/.deer-flow/data
pool_size: 20 # 基础连接池(postgres/mysql 生效;sqlite 忽略)
max_overflow: 20 # 高峰期可临时超出 pool_size 的额外连接数,真实上限 = pool_size + max_overflow
pool_timeout: 30 # 池满时等待空闲连接的秒数,超时报错
@@ -231,7 +260,7 @@ database:
checkpointer:
type: sqlite
- connection_string: .deer-flow/data/deerflow.db
+ connection_string: /app/backend/.deer-flow/data/deerflow.db
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
# 三层覆盖优先级(高->低):
@@ -313,3 +342,39 @@ log_system:
enabled: true
url: "https://your-log-system.example.com"
label: "日志系统"
+
+# 产品内知识库 —— obsidian-wiki 兼容 vault。知识元数据写入数据库,正文
+# content_md 写入 Markdown vault(index.md/log.md/hot.md/.manifest.json/frontmatter/
+# wikilinks/wiki-export 图谱产物)。知识库为全局共享,不做 user_id 隔离,
+# 写入时记录 created_by / updated_by 作为审计。
+knowledge:
+ enabled: true
+ provider: obsidian_wiki
+ vault_dir: data/knowledge/obsidian-vault
+ project_name: zncm
+ auto_ingest_enabled: false
+ auto_ingest_min_chars: 300
+ # 用大模型按 wiki-capture 规范提炼陈述式知识(而非堆砌对话原文)。
+ # 关闭则回退到规则抽取;extract_model_name 留空时用 models[0]。
+ llm_extract_enabled: true
+ extract_model_name: ""
+ extract_max_input_chars: 12000
+ max_thread_messages: 80
+ max_source_items: 20
+ rag_enabled: false
+ rag_limit: 5
+ # 阶段三:写入前脱敏(token/密钥/手机号/身份证/邮箱);重复检测相似度阈值。
+ sensitive_redaction_enabled: true
+ dedup_similarity_threshold: 0.82
+ export:
+ enabled: true
+ graph_html_enabled: true
+ graph_json_enabled: true
+ query:
+ use_hot_md: true
+ use_index_md: true
+ use_frontmatter_scan: true
+ embedding:
+ enabled: false
+ provider: local
+ model: ""
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py
index ae1398fe..833091e2 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py
@@ -299,6 +299,16 @@ def _build_middlewares(
if not is_bootstrap and not is_writing_setup:
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
+ # KnowledgeRagMiddleware — inject knowledge-base recall (phase 2). Gated
+ # per-conversation by the ``knowledge_rag_enabled`` runtime flag, defaulting
+ # to the global ``knowledge.rag_enabled`` config. No-op when knowledge is
+ # disabled. Skipped in bootstrap/writing-setup like MemoryMiddleware.
+ knowledge_cfg = getattr(resolved_app_config, "knowledge", None)
+ if not is_bootstrap and not is_writing_setup and knowledge_cfg is not None and knowledge_cfg.enabled:
+ from deerflow.agents.middlewares.knowledge_rag_middleware import KnowledgeRagMiddleware
+
+ middlewares.append(KnowledgeRagMiddleware(limit=knowledge_cfg.rag_limit, default_enabled=knowledge_cfg.rag_enabled))
+
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
# ingests raw image data). In delegated mode the view_image tool already
# returns a text description and stores nothing in `viewed_images`, so the
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/knowledge_rag_middleware.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/knowledge_rag_middleware.py
new file mode 100644
index 00000000..9ddfd683
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/knowledge_rag_middleware.py
@@ -0,0 +1,190 @@
+"""KnowledgeRagMiddleware — inject knowledge-base recall before the model call.
+
+Phase 2 of the product knowledge base. When enabled (per-conversation runtime
+flag ``knowledge_rag_enabled``, falling back to the global
+``knowledge.rag_enabled`` config), the latest user question is used to retrieve
+the most relevant knowledge notes, which are injected as a transient
+```` system message — not persisted to state, so it never
+leaks to the frontend transcript.
+
+Retrieval is hybrid (vector + keyword) when an embedding endpoint is
+configured, else keyword/wiki-query layered search. Failures are swallowed: RAG
+must never break a conversation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, override
+
+from langchain.agents import AgentState
+from langchain.agents.middleware import AgentMiddleware
+from langchain_core.messages import HumanMessage
+from langgraph.runtime import Runtime
+
+logger = logging.getLogger(__name__)
+
+_TAG = "knowledge-context"
+
+
+def _coerce_text(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ parts = []
+ for block in content:
+ if isinstance(block, str):
+ parts.append(block)
+ elif isinstance(block, dict) and block.get("type") == "text":
+ parts.append(str(block.get("text", "")))
+ return " ".join(parts)
+ return str(content or "")
+
+
+def _latest_human_text(messages: list) -> str:
+ for msg in reversed(messages or []):
+ if isinstance(msg, HumanMessage):
+ return _coerce_text(msg.content).strip()
+ return ""
+
+
+class KnowledgeRagMiddleware(AgentMiddleware[AgentState]):
+ """Inject knowledge-base recall as transient system context."""
+
+ state_schema = AgentState
+
+ def __init__(self, *, limit: int = 5, default_enabled: bool = False) -> None:
+ super().__init__()
+ self._limit = limit
+ self._default_enabled = default_enabled
+ self._service = None # lazy
+
+ def _get_service(self):
+ if self._service is not None:
+ return self._service
+ try:
+ from deerflow.config import get_app_config
+ from deerflow.knowledge import make_knowledge_service
+ from deerflow.persistence.engine import get_session_factory
+
+ config = get_app_config()
+ self._service = make_knowledge_service(get_session_factory(), config.knowledge)
+ except Exception:
+ logger.exception("KnowledgeRagMiddleware: failed to build knowledge service")
+ self._service = None
+ return self._service
+
+ def _enabled_for(self, request: Any) -> bool:
+ runtime = getattr(request, "runtime", None)
+ if runtime is not None and getattr(runtime, "context", None):
+ flag = runtime.context.get("knowledge_rag_enabled")
+ if flag is not None:
+ return bool(flag)
+ return self._default_enabled
+
+ async def _build_context_message(self, request: Any) -> HumanMessage | None:
+ if not self._enabled_for(request):
+ return None
+ service = self._get_service()
+ if service is None:
+ return None
+ query = _latest_human_text(getattr(request, "messages", None) or [])
+ if not query:
+ return None
+ try:
+ result = await service.build_rag_context(query, limit=self._limit)
+ except Exception:
+ logger.exception("KnowledgeRagMiddleware: recall failed (non-fatal)")
+ return None
+ text = result.get("text") if isinstance(result, dict) else ""
+ if not text:
+ return None
+ fenced = (
+ f"<{_TAG}>\n"
+ "[系统提示:以下是知识库召回的背景资料,不是用户的新输入。回答时优先参考,"
+ "并在使用到时标注来源 note_id。]\n\n"
+ f"{text}\n"
+ f"{_TAG}>"
+ )
+ return HumanMessage(content=fenced)
+
+ @override
+ async def awrap_model_call(self, request: Any, handler: Any) -> Any:
+ msg = await self._build_context_message(request)
+ if msg is not None:
+ request = request.override(messages=[*request.messages, msg])
+ return await handler(request)
+
+ @override
+ def wrap_model_call(self, request: Any, handler: Any) -> Any:
+ # Sync path: best-effort. The gateway runtime is async, so this is rarely
+ # hit; run the async recall in a private loop when there is none.
+ if not self._enabled_for(request):
+ return handler(request)
+ try:
+ loop = asyncio.get_event_loop()
+ if loop.is_running():
+ # Cannot block a running loop from sync code; skip injection.
+ return handler(request)
+ msg = loop.run_until_complete(self._build_context_message(request))
+ except Exception:
+ msg = None
+ if msg is not None:
+ request = request.override(messages=[*request.messages, msg])
+ return handler(request)
+
+ # ---- after_agent: enqueue background auto-sedimentation -------------
+ def _resolve_thread_id(self, runtime: Runtime | None) -> str | None:
+ if runtime is not None and getattr(runtime, "context", None):
+ tid = runtime.context.get("thread_id")
+ if tid:
+ return str(tid)
+ try:
+ from langgraph.config import get_config
+
+ return get_config().get("configurable", {}).get("thread_id")
+ except Exception:
+ return None
+
+ @override
+ def after_agent(self, state: AgentState, runtime: Runtime) -> dict | None:
+ """When auto-ingest is on, hand the finished turn to the background queue."""
+ try:
+ from deerflow.config import get_app_config
+
+ knowledge_cfg = getattr(get_app_config(), "knowledge", None)
+ if knowledge_cfg is None or not knowledge_cfg.enabled or not knowledge_cfg.auto_ingest_enabled:
+ return None
+
+ from deerflow.knowledge.auto_ingest import AutoIngestItem, get_auto_ingest_queue
+
+ queue = get_auto_ingest_queue()
+ if queue is None:
+ return None
+
+ messages = state.get("messages") if isinstance(state, dict) else getattr(state, "messages", None)
+ if not messages:
+ return None
+
+ from deerflow.runtime.references import build_reference_batches
+ from deerflow.runtime.serialization import serialize_lc_object
+ from deerflow.runtime.user_context import get_effective_user_id
+
+ serialized = [serialize_lc_object(m) for m in messages]
+ thread_id = self._resolve_thread_id(runtime)
+ if not thread_id:
+ return None
+ title = state.get("title") if isinstance(state, dict) else getattr(state, "title", None)
+ queue.enqueue(
+ AutoIngestItem(
+ thread_id=thread_id,
+ user_id=get_effective_user_id(),
+ messages=serialized,
+ thread_title=title if isinstance(title, str) else None,
+ reference_batches=build_reference_batches(serialized),
+ )
+ )
+ except Exception:
+ logger.exception("KnowledgeRagMiddleware.after_agent enqueue failed (non-fatal)")
+ return None
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py
new file mode 100644
index 00000000..4fcd9227
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/__init__.py
@@ -0,0 +1,2 @@
+"""Browser-backed page context helpers."""
+
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py
new file mode 100644
index 00000000..9a8d2f51
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/browser_context/tasks.py
@@ -0,0 +1,303 @@
+"""Small file-backed task queue for browser page fetch requests."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import threading
+import time
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from deerflow.config.runtime_paths import runtime_home
+
+logger = logging.getLogger(__name__)
+
+_LOCK = threading.RLock()
+_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$")
+_TERMINAL_STATUSES = {"done", "error", "expired"}
+
+
+def tasks_dir() -> Path:
+ """Return the browser-context task directory, creating it if needed."""
+
+ path = runtime_home() / "browser-context" / "tasks"
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _iso(value: datetime | None = None) -> str:
+ return (value or _now()).isoformat()
+
+
+def _parse_iso(value: Any) -> datetime | None:
+ if not isinstance(value, str) or not value:
+ return None
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ if parsed.tzinfo is None:
+ return parsed.replace(tzinfo=UTC)
+ return parsed.astimezone(UTC)
+
+
+def _task_path(task_id: str) -> Path | None:
+ if not _TASK_ID_RE.fullmatch(task_id or ""):
+ return None
+ return tasks_dir() / f"{task_id}.json"
+
+
+def _read_task(path: Path) -> dict[str, Any] | None:
+ try:
+ with path.open("r", encoding="utf-8") as fh:
+ data = json.load(fh)
+ except FileNotFoundError:
+ return None
+ except Exception:
+ logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True)
+ return None
+ return data if isinstance(data, dict) else None
+
+
+def _write_task(path: Path, data: dict[str, Any]) -> None:
+ tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
+ with tmp.open("w", encoding="utf-8") as fh:
+ json.dump(data, fh, ensure_ascii=False, separators=(",", ":"))
+ tmp.replace(path)
+
+
+def _public_task(data: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "id": data.get("id"),
+ "kind": data.get("kind") or "fetch",
+ "url": data.get("url") or "",
+ "instruction": data.get("instruction") or "",
+ "session_id": data.get("session_id") or "",
+ "action": data.get("action") or None,
+ "created_at": data.get("created_at") or "",
+ "updated_at": data.get("updated_at") or "",
+ "max_content_chars": int(data.get("max_content_chars") or 200_000),
+ }
+
+
+def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]:
+ if data.get("status") in _TERMINAL_STATUSES:
+ return data
+ expires_at = _parse_iso(data.get("expires_at"))
+ if expires_at is not None and now >= expires_at:
+ data = dict(data)
+ data["status"] = "expired"
+ data["error"] = "Browser task expired before the extension returned a result."
+ data["updated_at"] = _iso(now)
+ _write_task(path, data)
+ return data
+
+
+def prune_browser_tasks(max_age_seconds: int = 86_400) -> int:
+ """Delete old terminal tasks and return the number removed."""
+
+ cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds)))
+ removed = 0
+ with _LOCK:
+ for path in tasks_dir().glob("*.json"):
+ data = _read_task(path)
+ if not data or data.get("status") not in _TERMINAL_STATUSES:
+ continue
+ updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at"))
+ if updated_at is None or updated_at < cutoff:
+ try:
+ path.unlink()
+ removed += 1
+ except OSError:
+ logger.debug("Failed to delete old browser task: %s", path, exc_info=True)
+ return removed
+
+
+def create_browser_task(
+ *,
+ user_id: str,
+ url: str,
+ instruction: str = "",
+ timeout_seconds: int = 60,
+ max_content_chars: int = 200_000,
+ metadata: dict[str, Any] | None = None,
+ kind: str = "fetch",
+ session_id: str = "",
+ action: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ """Create a browser task for the current user's extension to pick up.
+
+ ``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or
+ ``"act"`` for an interactive observe/act step driven by ``action`` within a
+ long-lived ``session_id`` (a tab kept open by the extension).
+ """
+
+ now = _now()
+ task_id = uuid4().hex
+ task = {
+ "id": task_id,
+ "user_id": str(user_id or "default"),
+ "status": "pending",
+ "kind": kind if kind in {"fetch", "act"} else "fetch",
+ "url": url,
+ "instruction": instruction or "",
+ "session_id": str(session_id or ""),
+ "action": action or None,
+ "created_at": _iso(now),
+ "updated_at": _iso(now),
+ "expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))),
+ "max_content_chars": max(1_000, int(max_content_chars)),
+ "worker_id": None,
+ "claimed_at": None,
+ "result": None,
+ "error": None,
+ "metadata": metadata or {},
+ }
+ path = tasks_dir() / f"{task_id}.json"
+ with _LOCK:
+ _write_task(path, task)
+ return task
+
+
+def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None:
+ """Read a task if it belongs to the requested user."""
+
+ path = _task_path(task_id)
+ if path is None:
+ return None
+ with _LOCK:
+ data = _read_task(path)
+ if not data or str(data.get("user_id") or "") != str(user_id or "default"):
+ return None
+ return _expire_if_needed(path, data, _now())
+
+
+def claim_next_browser_task(
+ *,
+ user_id: str,
+ worker_id: str,
+ lease_seconds: int = 120,
+) -> dict[str, Any] | None:
+ """Claim the oldest pending task for a user."""
+
+ now = _now()
+ user_id = str(user_id or "default")
+ worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
+ lease_seconds = max(10, int(lease_seconds))
+
+ with _LOCK:
+ paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
+ for path in paths:
+ data = _read_task(path)
+ if not data or str(data.get("user_id") or "") != user_id:
+ continue
+
+ data = _expire_if_needed(path, data, now)
+ status = data.get("status")
+ if status == "running":
+ updated_at = _parse_iso(data.get("updated_at")) or now
+ if (now - updated_at).total_seconds() <= lease_seconds:
+ continue
+ status = "pending"
+ data = dict(data)
+ data["status"] = status
+
+ if status != "pending":
+ continue
+
+ data = dict(data)
+ data["status"] = "running"
+ data["worker_id"] = worker_id
+ data["claimed_at"] = _iso(now)
+ data["updated_at"] = _iso(now)
+ _write_task(path, data)
+ return _public_task(data)
+
+ return None
+
+
+def complete_browser_task(
+ *,
+ task_id: str,
+ user_id: str,
+ title: str = "",
+ url: str = "",
+ content: str = "",
+ content_type: str = "markdown",
+ error: str = "",
+ metadata: dict[str, Any] | None = None,
+ observation: dict[str, Any] | None = None,
+) -> dict[str, Any] | None:
+ """Mark a browser task complete or failed.
+
+ For ``act`` tasks the extension returns a structured ``observation`` (final
+ URL/title, interactive element list, page text) instead of a captured
+ markdown document; it is stored alongside the result verbatim.
+ """
+
+ path = _task_path(task_id)
+ if path is None:
+ return None
+
+ now = _now()
+ with _LOCK:
+ data = _read_task(path)
+ if not data or str(data.get("user_id") or "") != str(user_id or "default"):
+ return None
+
+ data = dict(data)
+ if error:
+ data["status"] = "error"
+ data["error"] = str(error).strip()[:4000]
+ data["result"] = None
+ else:
+ raw_content = str(content or "").strip()
+ max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000))
+ truncated = len(raw_content) > max_content_chars
+ data["status"] = "done"
+ data["error"] = None
+ data["result"] = {
+ "title": str(title or "").strip()[:500],
+ "url": str(url or data.get("url") or "").strip()[:4096],
+ "content": raw_content[:max_content_chars] if truncated else raw_content,
+ "content_type": content_type if content_type in {"markdown", "text"} else "markdown",
+ "truncated": truncated,
+ "observation": observation if isinstance(observation, dict) else None,
+ "captured_at": _iso(now),
+ "metadata": metadata or {},
+ }
+ data["updated_at"] = _iso(now)
+ _write_task(path, data)
+ return data
+
+
+async def wait_for_browser_task_result(
+ *,
+ task_id: str,
+ user_id: str,
+ timeout_seconds: int,
+ poll_interval_seconds: float = 1.0,
+) -> dict[str, Any] | None:
+ """Wait for a browser task to reach a terminal state."""
+
+ deadline = time.monotonic() + max(1, int(timeout_seconds))
+ interval = max(0.25, float(poll_interval_seconds))
+ while True:
+ task = get_browser_task(task_id=task_id, user_id=user_id)
+ if task is None:
+ return None
+ if task.get("status") in _TERMINAL_STATUSES:
+ return task
+ if time.monotonic() >= deadline:
+ return task
+ await asyncio.sleep(interval)
+
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py
index f1494bb6..543e546d 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/app_config.py
@@ -10,11 +10,13 @@ from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
+from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
+from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
@@ -133,6 +135,8 @@ class AppConfig(BaseModel):
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
+ browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
+ knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
@classmethod
def resolve_config_path(cls, config_path: str | None = None) -> Path:
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py
new file mode 100644
index 00000000..fdbfed8b
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/browser_context_config.py
@@ -0,0 +1,74 @@
+"""Configuration for browser page context capture."""
+
+from pydantic import BaseModel, Field
+
+
+class BrowserContextConfig(BaseModel):
+ """Controls the optional Chrome-extension page capture integration."""
+
+ enabled: bool = Field(
+ default=False,
+ description="Enable /api/browser-context endpoints for Chrome extension page capture.",
+ )
+ max_content_chars: int = Field(
+ default=200_000,
+ ge=1_000,
+ description="Maximum captured page text stored per request. Longer content is truncated.",
+ )
+ max_title_chars: int = Field(
+ default=300,
+ ge=20,
+ description="Maximum title length accepted from the browser extension.",
+ )
+ max_url_chars: int = Field(
+ default=2_048,
+ ge=100,
+ description="Maximum source URL length accepted from the browser extension.",
+ )
+ filename_prefix: str = Field(
+ default="webpage",
+ description="Prefix used for Markdown files created from captured pages.",
+ )
+ route_base: str = Field(
+ default="/page/workspace",
+ description="Frontend route base used when returning chat paths to the extension.",
+ )
+ task_result_timeout_seconds: int = Field(
+ default=60,
+ ge=5,
+ description="Maximum time the browser_fetch_page tool waits for the extension result.",
+ )
+ task_poll_interval_seconds: float = Field(
+ default=1.0,
+ ge=0.25,
+ description="How often the backend tool checks for a returned browser task result.",
+ )
+ task_lease_seconds: int = Field(
+ default=120,
+ ge=10,
+ description="How long a claimed browser task is reserved before another extension may retry it.",
+ )
+ tool_result_max_chars: int = Field(
+ default=60_000,
+ ge=1_000,
+ description="Maximum page characters returned from browser_fetch_page into the model context.",
+ )
+ actions_enabled: bool = Field(
+ default=True,
+ description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).",
+ )
+ action_result_timeout_seconds: int = Field(
+ default=45,
+ ge=5,
+ description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.",
+ )
+ max_observed_elements: int = Field(
+ default=150,
+ ge=10,
+ description="Maximum number of interactive elements returned in a browser_act observation.",
+ )
+ observation_text_max_chars: int = Field(
+ default=12_000,
+ ge=500,
+ description="Maximum page text characters returned alongside a browser_act observation.",
+ )
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/extensions_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/extensions_config.py
index 78ebbf6c..0a0c9f1f 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/config/extensions_config.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/extensions_config.py
@@ -3,7 +3,7 @@
import json
import os
from pathlib import Path
-from typing import Any, Literal
+from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -53,115 +53,115 @@ class McpServerConfig(BaseModel):
model_config = ConfigDict(extra="allow", populate_by_name=True)
-class SkillDisplayTableColumn(BaseModel):
- """A configured table column for structured skill result display."""
-
- label: str = Field(description="Column label shown in the frontend table")
- field: str = Field(description="Field name or dot path read from each result item")
- width: str | None = Field(default=None, description="Optional CSS width for this column")
- model_config = ConfigDict(extra="allow", populate_by_name=True)
-
-
-class SkillDisplayResultSetConfig(BaseModel):
- """Display adapter for one result array inside a skill response."""
-
- key: str = Field(description="Stable key for this result set")
- label: str | None = Field(default=None, description="Human-readable result set label")
- enabled: bool = Field(default=True, description="Whether this result set is enabled")
- mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
- default="citation",
- description="How the frontend should render this result set",
- )
- result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
- title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
- summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
- content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
- source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
- author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
- type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
- time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
- score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
- url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
- id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
- link_strategy: Literal["auto", "url", "id"] = Field(
- default="auto",
- description="Whether links use a URL field, an ID template, or automatic fallback",
- alias="linkStrategy",
- )
- link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
- citation_token: Literal["doubleBracket", "singleBracket"] = Field(
- default="singleBracket",
- description="Citation token style expected in model output",
- alias="citationToken",
- )
- max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
- custom_instruction: str = Field(
- default="",
- description="User-authored instruction for custom rendering schema generation",
- alias="customInstruction",
- )
- table_columns: list[SkillDisplayTableColumn] = Field(
- default_factory=list,
- description="Columns used when mode is table",
- alias="tableColumns",
- )
- model_config = ConfigDict(extra="allow", populate_by_name=True)
-
-
-class SkillDisplayConfig(BaseModel):
- """Frontend display adapter for structured skill results."""
-
- enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
- mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
- default="none",
- description="How the frontend should render this skill's structured results",
- )
- result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
- title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
- summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
- content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
- source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
- author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
- type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
- time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
- score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
- url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
- id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
- link_strategy: Literal["auto", "url", "id"] = Field(
- default="auto",
- description="Whether links use a URL field, an ID template, or automatic fallback",
- alias="linkStrategy",
- )
- link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
- citation_token: Literal["doubleBracket", "singleBracket"] = Field(
- default="doubleBracket",
- description="Citation token style expected in model output",
- alias="citationToken",
- )
- max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
- custom_instruction: str = Field(
- default="",
- description="User-authored instruction for custom rendering schema generation",
- alias="customInstruction",
- )
- table_columns: list[SkillDisplayTableColumn] = Field(
- default_factory=list,
- description="Columns used when mode is table",
- alias="tableColumns",
- )
- result_sets: list[SkillDisplayResultSetConfig] = Field(
- default_factory=list,
- description="Optional per-result-array display adapters",
- alias="resultSets",
- )
- model_config = ConfigDict(extra="allow", populate_by_name=True)
-
-
-class SkillStateConfig(BaseModel):
- """Configuration for a single skill's state."""
-
- enabled: bool = Field(default=True, description="Whether this skill is enabled")
- display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
+class SkillDisplayTableColumn(BaseModel):
+ """A configured table column for structured skill result display."""
+
+ label: str = Field(description="Column label shown in the frontend table")
+ field: str = Field(description="Field name or dot path read from each result item")
+ width: str | None = Field(default=None, description="Optional CSS width for this column")
+ model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+
+class SkillDisplayResultSetConfig(BaseModel):
+ """Display adapter for one result array inside a skill response."""
+
+ key: str = Field(description="Stable key for this result set")
+ label: str | None = Field(default=None, description="Human-readable result set label")
+ enabled: bool = Field(default=True, description="Whether this result set is enabled")
+ mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
+ default="citation",
+ description="How the frontend should render this result set",
+ )
+ result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
+ title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
+ summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
+ content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
+ source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
+ author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
+ type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
+ time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
+ score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
+ url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
+ id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
+ link_strategy: Literal["auto", "url", "id"] = Field(
+ default="auto",
+ description="Whether links use a URL field, an ID template, or automatic fallback",
+ alias="linkStrategy",
+ )
+ link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
+ citation_token: Literal["doubleBracket", "singleBracket"] = Field(
+ default="singleBracket",
+ description="Citation token style expected in model output",
+ alias="citationToken",
+ )
+ max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
+ custom_instruction: str = Field(
+ default="",
+ description="User-authored instruction for custom rendering schema generation",
+ alias="customInstruction",
+ )
+ table_columns: list[SkillDisplayTableColumn] = Field(
+ default_factory=list,
+ description="Columns used when mode is table",
+ alias="tableColumns",
+ )
+ model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+
+class SkillDisplayConfig(BaseModel):
+ """Frontend display adapter for structured skill results."""
+
+ enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
+ mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
+ default="none",
+ description="How the frontend should render this skill's structured results",
+ )
+ result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
+ title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
+ summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
+ content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
+ source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
+ author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
+ type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
+ time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
+ score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
+ url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
+ id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
+ link_strategy: Literal["auto", "url", "id"] = Field(
+ default="auto",
+ description="Whether links use a URL field, an ID template, or automatic fallback",
+ alias="linkStrategy",
+ )
+ link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
+ citation_token: Literal["doubleBracket", "singleBracket"] = Field(
+ default="doubleBracket",
+ description="Citation token style expected in model output",
+ alias="citationToken",
+ )
+ max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
+ custom_instruction: str = Field(
+ default="",
+ description="User-authored instruction for custom rendering schema generation",
+ alias="customInstruction",
+ )
+ table_columns: list[SkillDisplayTableColumn] = Field(
+ default_factory=list,
+ description="Columns used when mode is table",
+ alias="tableColumns",
+ )
+ result_sets: list[SkillDisplayResultSetConfig] = Field(
+ default_factory=list,
+ description="Optional per-result-array display adapters",
+ alias="resultSets",
+ )
+ model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+
+class SkillStateConfig(BaseModel):
+ """Configuration for a single skill's state."""
+
+ enabled: bool = Field(default=True, description="Whether this skill is enabled")
+ display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
class ExtensionsConfig(BaseModel):
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py
new file mode 100644
index 00000000..d30d0c4c
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/knowledge_config.py
@@ -0,0 +1,75 @@
+"""Configuration for the product knowledge base (obsidian-wiki compatible)."""
+
+from pydantic import BaseModel, Field
+
+
+class KnowledgeExportConfig(BaseModel):
+ """Knowledge graph export switches (``wiki-export`` compatible output)."""
+
+ enabled: bool = Field(default=True, description="Master switch for graph export.")
+ graph_html_enabled: bool = Field(default=True, description="Generate wiki-export/graph.html.")
+ graph_json_enabled: bool = Field(default=True, description="Generate wiki-export/graph.json.")
+
+
+class KnowledgeQueryConfig(BaseModel):
+ """Layered query strategy flags (``wiki-query`` style)."""
+
+ use_hot_md: bool = Field(default=True, description="Consult hot.md when searching.")
+ use_index_md: bool = Field(default=True, description="Consult index.md when searching.")
+ use_frontmatter_scan: bool = Field(default=True, description="Scan frontmatter when searching.")
+
+
+class KnowledgeEmbeddingConfig(BaseModel):
+ """Vector embedding config (phase 2).
+
+ Vector RAG is opt-in: set ``enabled: true`` and point ``base_url`` at an
+ OpenAI-compatible ``/embeddings`` endpoint. When disabled or unreachable,
+ search transparently falls back to keyword scoring, so the feature degrades
+ gracefully on offline deployments without an embedding model.
+ """
+
+ enabled: bool = Field(default=False, description="Enable vector embeddings.")
+ provider: str = Field(default="openai", description="Embedding provider (openai-compatible).")
+ model: str = Field(default="", description="Embedding model name, e.g. text-embedding-3-small / bge-m3.")
+ base_url: str = Field(default="", description="OpenAI-compatible base URL (…/v1). Empty disables vector mode.")
+ api_key: str = Field(default="", description="API key for the embedding endpoint ($ENV resolved like other config).")
+ dimensions: int | None = Field(default=None, description="Optional embedding dimensions hint sent to the endpoint.")
+ batch_size: int = Field(default=32, description="Texts per embedding request.")
+ timeout_seconds: int = Field(default=30, description="HTTP timeout per embedding request.")
+
+
+class KnowledgeConfig(BaseModel):
+ """Product knowledge base configuration.
+
+ The first release ships a global, shared, obsidian-wiki compatible vault.
+ Knowledge metadata is mirrored into the database for fast listing/search,
+ while ``content_md`` is written to a Markdown vault that stays compatible
+ with ``ar9av/obsidian-wiki`` (index.md / log.md / hot.md / .manifest.json,
+ frontmatter, wikilinks, wiki-export graph artifacts).
+ """
+
+ enabled: bool = Field(default=True, description="Enable the knowledge base feature.")
+ provider: str = Field(default="obsidian_wiki", description="Knowledge organization provider.")
+ vault_dir: str = Field(
+ default="data/knowledge/obsidian-vault",
+ description="Vault directory (relative to DEER_FLOW_HOME, or absolute).",
+ )
+ project_name: str = Field(default="zncm", description="Project namespace for projects// pages.")
+ auto_ingest_enabled: bool = Field(default=False, description="Auto-sediment conversations in the background (phase 3).")
+ auto_ingest_min_chars: int = Field(default=300, description="Minimum characters before auto-ingest considers a turn.")
+ llm_extract_enabled: bool = Field(default=True, description="Distill notes with an LLM (wiki-capture style) instead of the rule-based fallback.")
+ extract_model_name: str | None = Field(default=None, description="Model (from models[]) used to distill knowledge. Falls back to models[0] when unset.")
+ extract_max_input_chars: int = Field(default=12000, description="Max characters of conversation transcript fed to the extraction LLM.")
+ max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.")
+ max_source_items: int = Field(default=20, description="Maximum source items recorded per note.")
+ related_notes_enabled: bool = Field(default=True, description="On capture, find existing notes on the same topic and link them under ## Related (real cross-references instead of placeholders).")
+ related_notes_limit: int = Field(default=3, description="Maximum real related notes linked into each captured note.")
+ multi_note_enabled: bool = Field(default=True, description="Allow the LLM extractor to split one conversation into multiple cross-linked knowledge notes.")
+ rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).")
+ rag_limit: int = Field(default=5, description="Maximum knowledge snippets injected per query.")
+ sensitive_redaction_enabled: bool = Field(default=True, description="Redact secrets/PII (tokens, keys, phone, id-card, email) before persisting notes (phase 3).")
+ dedup_similarity_threshold: float = Field(default=0.82, description="Title/summary similarity above which two notes are flagged as duplicates (phase 3).")
+ entity_extraction_enabled: bool = Field(default=True, description="Extract entities/relations during LLM distillation and feed them into the graph (phase 4).")
+ export: KnowledgeExportConfig = Field(default_factory=KnowledgeExportConfig, description="Graph export configuration.")
+ query: KnowledgeQueryConfig = Field(default_factory=KnowledgeQueryConfig, description="Layered query configuration.")
+ embedding: KnowledgeEmbeddingConfig = Field(default_factory=KnowledgeEmbeddingConfig, description="Embedding configuration.")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/__init__.py
new file mode 100644
index 00000000..ee3e83e5
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/__init__.py
@@ -0,0 +1,41 @@
+"""Product knowledge base module (obsidian-wiki compatible).
+
+Self-contained: knowledge ingestion, storage (DB mirror + Markdown vault),
+search and graph export all live here. The Gateway only depends on
+:class:`KnowledgeService` via :func:`make_knowledge_service`.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
+from deerflow.knowledge.repository import KnowledgeRepository
+from deerflow.knowledge.service import EmptyThreadError, KnowledgeService
+
+if TYPE_CHECKING:
+ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+ from deerflow.config.knowledge_config import KnowledgeConfig
+
+__all__ = [
+ "EmptyThreadError",
+ "KnowledgeRepository",
+ "KnowledgeService",
+ "ObsidianWikiAdapter",
+ "make_knowledge_service",
+]
+
+
+def make_knowledge_service(
+ session_factory: async_sessionmaker[AsyncSession] | None,
+ config: KnowledgeConfig,
+) -> KnowledgeService | None:
+ """Build the knowledge service.
+
+ Returns ``None`` when persistence is memory-only (no session factory) or
+ the feature is disabled — the router then responds 503.
+ """
+ if session_factory is None or not config.enabled:
+ return None
+ return KnowledgeService(KnowledgeRepository(session_factory), config)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/auto_ingest.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/auto_ingest.py
new file mode 100644
index 00000000..a3c5cef5
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/auto_ingest.py
@@ -0,0 +1,125 @@
+"""Background auto-sediment queue (phase 3).
+
+A single asyncio worker drains a bounded queue of finished conversations and
+sediments the worthwhile ones into the knowledge base — entirely off the chat
+request path, so it never adds latency to a reply. Enqueue happens from
+``KnowledgeRagMiddleware.after_agent`` when ``knowledge.auto_ingest_enabled``.
+
+Value judgment is intentionally cheap (min assistant chars + dedup via
+``skip_existing``); the LLM still distills the kept conversations via the normal
+``capture_thread`` path.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_QUEUE: AutoIngestQueue | None = None
+
+
+@dataclass
+class _QueueUser:
+ """Minimal CurrentUser stand-in (only ``.id`` is needed for context)."""
+
+ id: str
+
+
+@dataclass
+class AutoIngestItem:
+ thread_id: str
+ user_id: str
+ messages: list[dict[str, Any]]
+ thread_title: str | None = None
+ reference_batches: list[dict[str, Any]] = field(default_factory=list)
+
+
+class AutoIngestQueue:
+ """Bounded background queue + single worker for auto-sedimentation."""
+
+ def __init__(self, service_provider: Callable[[], Any], *, min_chars: int = 300, maxsize: int = 256) -> None:
+ self._provider = service_provider
+ self._min_chars = min_chars
+ self._queue: asyncio.Queue[AutoIngestItem] = asyncio.Queue(maxsize=maxsize)
+ self._task: asyncio.Task | None = None
+
+ def start(self) -> None:
+ if self._task is None or self._task.done():
+ self._task = asyncio.create_task(self._worker(), name="knowledge-auto-ingest")
+ logger.info("Knowledge auto-ingest worker started")
+
+ async def stop(self) -> None:
+ if self._task is not None:
+ self._task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._task
+ self._task = None
+
+ def enqueue(self, item: AutoIngestItem) -> None:
+ try:
+ self._queue.put_nowait(item)
+ except asyncio.QueueFull:
+ logger.warning("Knowledge auto-ingest queue full; dropping thread %s", item.thread_id)
+
+ def _worth_ingesting(self, item: AutoIngestItem) -> bool:
+ chars = 0
+ for m in item.messages:
+ if str(m.get("type") or m.get("role") or "").lower() == "ai":
+ content = m.get("content")
+ chars += len(content) if isinstance(content, str) else len(str(content or ""))
+ return chars >= self._min_chars
+
+ async def _process(self, item: AutoIngestItem) -> None:
+ service = self._provider()
+ if service is None:
+ return
+ if not self._worth_ingesting(item):
+ logger.debug("Auto-ingest skipped (below min chars) thread=%s", item.thread_id)
+ return
+ from deerflow.runtime.user_context import reset_current_user, set_current_user
+
+ token = set_current_user(_QueueUser(id=item.user_id))
+ try:
+ result = await service.capture_thread(
+ thread_id=item.thread_id,
+ messages=item.messages,
+ thread_title=item.thread_title,
+ status="draft", # auto-sedimented notes land as drafts for review
+ include_sources=True,
+ reference_batches=item.reference_batches,
+ created_by=item.user_id,
+ skip_existing=True,
+ )
+ if result.get("created"):
+ logger.info("Auto-ingested thread %s -> note %s", item.thread_id, result["note"]["id"])
+ except Exception:
+ logger.exception("Auto-ingest failed for thread %s", item.thread_id)
+ finally:
+ reset_current_user(token)
+
+ async def _worker(self) -> None:
+ while True:
+ item = await self._queue.get()
+ try:
+ await self._process(item)
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ logger.exception("Auto-ingest worker error")
+ finally:
+ self._queue.task_done()
+
+
+def set_auto_ingest_queue(queue: AutoIngestQueue | None) -> None:
+ global _QUEUE
+ _QUEUE = queue
+
+
+def get_auto_ingest_queue() -> AutoIngestQueue | None:
+ return _QUEUE
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/chunking.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/chunking.py
new file mode 100644
index 00000000..7897d8c1
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/chunking.py
@@ -0,0 +1,60 @@
+"""Markdown chunking for embedding/index (phase 2).
+
+Splits a note body into overlapping chunks along Markdown structure (headings →
+paragraphs) so each embedded chunk is semantically coherent and bounded in
+size. Frontmatter is assumed already stripped by the caller.
+"""
+
+from __future__ import annotations
+
+import re
+
+_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
+
+
+def _split_paragraphs(text: str) -> list[str]:
+ return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+
+
+def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
+ """Return a list of text chunks for ``content``.
+
+ Sections are cut at Markdown headings; long sections are further split into
+ ~``max_chars`` windows with ``overlap`` characters of carry-over.
+ """
+ text = (content or "").strip()
+ if not text:
+ return []
+
+ # Split into heading-delimited sections, keeping the heading with its body.
+ sections: list[str] = []
+ last = 0
+ for m in _HEADING_RE.finditer(text):
+ if m.start() > last:
+ sections.append(text[last : m.start()].strip())
+ last = m.start()
+ sections.append(text[last:].strip())
+ sections = [s for s in sections if s]
+
+ chunks: list[str] = []
+ for section in sections:
+ if len(section) <= max_chars:
+ chunks.append(section)
+ continue
+ # Pack paragraphs into windows, then hard-split anything still too long.
+ buf = ""
+ for para in _split_paragraphs(section):
+ if len(buf) + len(para) + 2 <= max_chars:
+ buf = f"{buf}\n\n{para}".strip()
+ else:
+ if buf:
+ chunks.append(buf)
+ if len(para) <= max_chars:
+ buf = para
+ else:
+ for i in range(0, len(para), max_chars - overlap):
+ chunks.append(para[i : i + max_chars])
+ buf = ""
+ if buf:
+ chunks.append(buf)
+ return [c for c in chunks if c.strip()]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/dedup.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/dedup.py
new file mode 100644
index 00000000..936bd9e0
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/dedup.py
@@ -0,0 +1,75 @@
+"""Duplicate detection for knowledge notes (phase 3).
+
+First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
+exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
+works on offline deployments. Returns *suggestions*; merging is an explicit,
+user-driven action.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from deerflow.knowledge.manifest import content_hash
+
+_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
+
+
+def _tokens(*texts: str) -> set[str]:
+ out: set[str] = set()
+ for t in texts:
+ out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
+ return out
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+ if not a or not b:
+ return 0.0
+ inter = len(a & b)
+ union = len(a | b)
+ return inter / union if union else 0.0
+
+
+def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
+ """Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
+
+ ``primary`` is the most recently updated note in each cluster.
+ """
+ notes = notes[:max_notes]
+ feats: list[tuple[dict[str, Any], set[str], str]] = []
+ for n in notes:
+ toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
+ feats.append((n, toks, content_hash(n.get("content_md") or "")))
+
+ used: set[str] = set()
+ groups: list[dict[str, Any]] = []
+ for i in range(len(feats)):
+ ni, ti, hi = feats[i]
+ if ni["id"] in used:
+ continue
+ dups: list[dict[str, Any]] = []
+ for j in range(i + 1, len(feats)):
+ nj, tj, hj = feats[j]
+ if nj["id"] in used:
+ continue
+ if hi and hi == hj:
+ score, reason = 1.0, "identical-content"
+ else:
+ score = _jaccard(ti, tj)
+ reason = "similar-title-summary"
+ if score >= threshold:
+ dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
+ used.add(nj["id"])
+ if dups:
+ used.add(ni["id"])
+ # Primary = newest by updated_at.
+ cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
+ primary = max(cluster, key=lambda x: x.get("updated_at") or "")
+ others = [
+ {"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
+ for c in cluster
+ if c["id"] != primary["id"]
+ ]
+ groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
+ return groups
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/embeddings.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/embeddings.py
new file mode 100644
index 00000000..b28843e4
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/embeddings.py
@@ -0,0 +1,79 @@
+"""Embedding generation for vector RAG (phase 2).
+
+Talks to an OpenAI-compatible ``/embeddings`` endpoint configured under
+``knowledge.embedding``. Everything here is **best-effort**: if embeddings are
+disabled, unconfigured, or the endpoint errors, callers get an empty result and
+the search layer falls back to keyword scoring — so vector RAG never breaks a
+deployment that has no embedding model.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from deerflow.config.knowledge_config import KnowledgeEmbeddingConfig
+
+logger = logging.getLogger(__name__)
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+ """Cosine similarity of two equal-length vectors (0 when degenerate)."""
+ if not a or not b or len(a) != len(b):
+ return 0.0
+ dot = sum(x * y for x, y in zip(a, b, strict=False))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ if na == 0 or nb == 0:
+ return 0.0
+ return dot / (na * nb)
+
+
+class EmbeddingClient:
+ """Thin async client over an OpenAI-compatible embeddings endpoint."""
+
+ def __init__(self, config: KnowledgeEmbeddingConfig) -> None:
+ self.config = config
+
+ def is_enabled(self) -> bool:
+ return bool(self.config.enabled and self.config.base_url and self.config.model)
+
+ async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+ """Return one vector per input text; ``[]`` on any failure/disabled."""
+ if not self.is_enabled() or not texts:
+ return []
+ try:
+ import httpx
+ except Exception: # pragma: no cover - httpx is a core dep
+ logger.warning("httpx unavailable; knowledge embeddings disabled")
+ return []
+
+ url = self.config.base_url.rstrip("/") + "/embeddings"
+ headers = {"Content-Type": "application/json"}
+ if self.config.api_key:
+ headers["Authorization"] = f"Bearer {self.config.api_key}"
+
+ vectors: list[list[float]] = []
+ batch = max(1, self.config.batch_size)
+ try:
+ async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
+ for start in range(0, len(texts), batch):
+ chunk = texts[start : start + batch]
+ payload: dict = {"model": self.config.model, "input": chunk}
+ if self.config.dimensions:
+ payload["dimensions"] = self.config.dimensions
+ resp = await client.post(url, headers=headers, json=payload)
+ resp.raise_for_status()
+ data = resp.json()
+ items = sorted(data.get("data", []), key=lambda d: d.get("index", 0))
+ vectors.extend([item.get("embedding", []) for item in items])
+ except Exception:
+ logger.exception("Knowledge embedding request failed; falling back to keyword search")
+ return []
+ return vectors
+
+ async def embed_query(self, text: str) -> list[float]:
+ vecs = await self.embed_texts([text])
+ return vecs[0] if vecs else []
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py
new file mode 100644
index 00000000..2f91a5ee
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py
@@ -0,0 +1,227 @@
+"""obsidian-wiki ``wiki-export`` compatible graph generation.
+
+Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
+tags (edges) and writes the four native artifacts into ``/wiki-export/``:
+``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
+``graph.html`` (no external CDN, safe to iframe offline).
+"""
+
+from __future__ import annotations
+
+import html
+import json
+from pathlib import Path
+from typing import Any
+
+from deerflow.knowledge.markdown_vault import extract_wikilinks
+
+
+def _vault_key(vault_path: str | None) -> str | None:
+ if not vault_path:
+ return None
+ key = vault_path.replace("\\", "/")
+ if key.endswith(".md"):
+ key = key[:-3]
+ return key
+
+
+def build_graph(
+ notes: list[dict[str, Any]],
+ entities: list[dict[str, Any]] | None = None,
+ relations: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+ """Return ``{nodes, edges}`` from notes (+ optional entities/relations).
+
+ Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
+ when ``entities``/``relations`` are supplied (phase 4), entity nodes and
+ note→entity / entity→entity edges are added on top of the wikilink and
+ shared-tag edges.
+ """
+ nodes = []
+ by_key: dict[str, str] = {}
+ note_ids: set[str] = set()
+ for n in notes:
+ nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
+ note_ids.add(n["id"])
+ key = _vault_key(n.get("vault_path"))
+ if key:
+ by_key[key] = n["id"]
+
+ # Only surface entities that are still mentioned by a LIVE note. Entities whose
+ # only referencing notes were deleted/archived would otherwise linger forever
+ # as orphan nodes ("graph doesn't follow my deletions").
+ mentioned: set[str] = set()
+ for rel in relations or []:
+ if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
+ mentioned.add(rel["to_entity_id"])
+
+ entity_ids: set[str] = set()
+ for ent in entities or []:
+ if ent["id"] not in mentioned:
+ continue
+ nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
+ entity_ids.add(ent["id"])
+
+ edges: list[dict[str, Any]] = []
+ edge_seen: set[tuple[str, str, str]] = set()
+
+ def add_edge(a: str, b: str, label: str, weight: float) -> None:
+ if a == b:
+ return
+ key = (min(a, b), max(a, b), label)
+ if key in edge_seen:
+ return
+ edge_seen.add(key)
+ edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
+
+ # Wikilink edges.
+ for n in notes:
+ for target in extract_wikilinks(n.get("content_md") or ""):
+ tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
+ if tid:
+ add_edge(n["id"], tid, "links", 2)
+
+ # Shared-tag edges.
+ tag_to_notes: dict[str, list[str]] = {}
+ for n in notes:
+ for tag in n.get("tags") or []:
+ tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
+ for tag, ids in tag_to_notes.items():
+ if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
+ continue
+ for i in range(len(ids)):
+ for j in range(i + 1, len(ids)):
+ add_edge(ids[i], ids[j], "related", 1)
+
+ # Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
+ valid = note_ids | entity_ids
+ for rel in relations or []:
+ src = rel.get("from_note_id") or rel.get("from_entity_id")
+ tgt = rel.get("to_note_id") or rel.get("to_entity_id")
+ if src in valid and tgt in valid:
+ add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
+
+ return {"nodes": nodes, "edges": edges}
+
+
+def to_graphml(graph: dict[str, Any]) -> str:
+ lines = [
+ '',
+ '',
+ ' ',
+ ' ',
+ ' ',
+ ]
+ for n in graph["nodes"]:
+ lines.append(f' {html.escape(str(n["label"]))}')
+ for e in graph["edges"]:
+ lines.append(
+ f' {html.escape(str(e["label"]))}'
+ )
+ lines += [" ", ""]
+ return "\n".join(lines)
+
+
+def to_cypher(graph: dict[str, Any]) -> str:
+ lines = []
+ for n in graph["nodes"]:
+ label = str(n["label"]).replace("'", "\\'")
+ lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
+ for e in graph["edges"]:
+ rel = str(e["label"]).upper()
+ lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
+ return "\n".join(lines)
+
+
+def to_html(graph: dict[str, Any]) -> str:
+ """Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
+ data_json = json.dumps(graph, ensure_ascii=False)
+ return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
+
+
+def export_all(
+ vault_root: Path | str,
+ notes: list[dict[str, Any]],
+ *,
+ entities: list[dict[str, Any]] | None = None,
+ relations: list[dict[str, Any]] | None = None,
+ write_html: bool = True,
+ write_json: bool = True,
+) -> dict[str, Any]:
+ """Generate graph artifacts under ``/wiki-export/`` and return stats."""
+ out_dir = Path(vault_root).resolve() / "wiki-export"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ graph = build_graph(notes, entities, relations)
+ written: list[str] = []
+ if write_json:
+ (out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
+ written.append("graph.json")
+ (out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
+ written.append("graph.graphml")
+ (out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
+ written.append("cypher.txt")
+ if write_html:
+ (out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
+ written.append("graph.html")
+ return {
+ "files": written,
+ "stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
+ "graph": graph,
+ }
+
+
+_HTML_TEMPLATE = """
+
+
+
+
+Knowledge Graph
+
+
+
+
+
+
+
+
+"""
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py
new file mode 100644
index 00000000..a9dbbe63
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/extractor.py
@@ -0,0 +1,262 @@
+"""Extract knowledge notes from threads / searches / tool results.
+
+First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
+keep high-value content (user questions, final assistant answers, cited
+sources), drop noise (empty/greeting/intermediate reasoning), and write a
+*declarative* note rather than a chat transcript. An LLM extractor can later
+replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from deerflow.knowledge.schemas import NoteDraft, SourceDraft
+
+# Greeting / low-value lines filtered from extracted turns.
+_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
+_WS_RE = re.compile(r"\s+")
+
+
+def _one_line(text: str, limit: int = 300) -> str:
+ """Collapse whitespace and clip ``text`` to a single line."""
+ collapsed = _WS_RE.sub(" ", text or "").strip()
+ return collapsed[:limit]
+
+
+def message_text(msg: dict[str, Any]) -> str:
+ """Return the plain-text content of a serialized LangChain message."""
+ content = msg.get("content")
+ if isinstance(content, str):
+ return content.strip()
+ if isinstance(content, list):
+ parts: list[str] = []
+ for block in content:
+ if isinstance(block, str):
+ parts.append(block)
+ elif isinstance(block, dict):
+ if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
+ parts.append(block["text"])
+ return "\n".join(p for p in parts if p).strip()
+ return ""
+
+
+def _msg_type(msg: dict[str, Any]) -> str:
+ return str(msg.get("type") or msg.get("role") or "").lower()
+
+
+def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
+ """Split messages into ``{user, ai}`` turns keyed by human messages.
+
+ For each turn we keep the last substantial AI message (with text body).
+ """
+ msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
+ turns: list[dict[str, str]] = []
+ current: dict[str, str] | None = None
+ for msg in msgs:
+ mtype = _msg_type(msg)
+ text = message_text(msg)
+ if mtype == "human":
+ if current and (current.get("user") or current.get("ai")):
+ turns.append(current)
+ current = {"user": text, "ai": ""}
+ elif mtype == "ai" and text:
+ if current is None:
+ current = {"user": "", "ai": text}
+ else:
+ current["ai"] = text # last substantial AI message wins
+ if current and (current.get("user") or current.get("ai")):
+ turns.append(current)
+ # Drop turns that are pure greetings with no answer.
+ cleaned = []
+ for t in turns:
+ user = (t.get("user") or "").strip()
+ ai = (t.get("ai") or "").strip()
+ if not ai and _GREETING_RE.match(user):
+ continue
+ if not user and not ai:
+ continue
+ cleaned.append({"user": user, "ai": ai})
+ return cleaned
+
+
+def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
+ if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
+ return thread_title.strip()[:200]
+ for t in turns:
+ if t.get("user"):
+ first = t["user"].splitlines()[0].strip()
+ return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
+ return "知识笔记"
+
+
+def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
+ """Simple rule-based summary: the first answer, clipped."""
+ for t in turns:
+ if t.get("ai"):
+ text = re.sub(r"\s+", " ", t["ai"]).strip()
+ return text[:limit] + ("…" if len(text) > limit else "")
+ for t in turns:
+ if t.get("user"):
+ text = re.sub(r"\s+", " ", t["user"]).strip()
+ return text[:limit] + ("…" if len(text) > limit else "")
+ return ""
+
+
+def related_links(project_name: str | None) -> list[str]: # noqa: ARG001
+ """Structural wikilinks every note used to carry.
+
+ Product decision: notes no longer auto-link the vault meta/hub pages
+ (``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every
+ note's ``## Related``. Only real, content-specific related notes (injected by
+ the service layer, which has DB access) are linked now. Kept as a function
+ (returns empty) so existing callers and the wiki-capture contract stay intact.
+ """
+ return []
+
+
+# Backwards-compatible internal alias.
+_related_links = related_links
+
+
+def _related_section_md(related_lines: list[str], source_lines: list[str]) -> str:
+ """Render the ``## Related`` section, or ``""`` when there is nothing to show."""
+ if not related_lines and not source_lines:
+ return ""
+ inner = "\n".join(related_lines)
+ if source_lines:
+ sources_block = "### Sources\n\n" + "\n".join(source_lines)
+ inner = (inner + "\n\n" + sources_block) if inner else sources_block
+ return f"## Related\n\n{inner}\n"
+
+
+def _build_body(
+ *,
+ title: str,
+ turns: list[dict[str, str]],
+ mode: str,
+ project_name: str | None,
+ sources: list[SourceDraft],
+) -> str:
+ """Render the wiki-capture 5-section declarative body."""
+ full = mode == "full"
+ answer_clip = 4000 if full else 800
+
+ context_lines = []
+ for t in turns:
+ if t.get("user"):
+ q = re.sub(r"\s+", " ", t["user"]).strip()
+ context_lines.append(f"- {q[:300]}")
+ context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
+
+ finding_lines = []
+ for i, t in enumerate(turns, 1):
+ if t.get("ai"):
+ ans = t["ai"].strip()
+ ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
+ finding_lines.append(f"### 结论 {i}\n\n{ans}")
+ finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
+
+ reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
+
+ implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
+
+ related_lines = [f"- {link}" for link in _related_links(project_name)]
+ source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
+ related_section = _related_section_md(related_lines, source_lines)
+
+ return (
+ f"# {title}\n\n"
+ f"## Context\n\n{context}\n\n"
+ f"## Finding / Decision\n\n{finding}\n\n"
+ f"## Reasoning\n\n{reasoning}\n\n"
+ f"## Implications\n\n{implications}\n\n"
+ f"{related_section}"
+ )
+
+
+def extract_thread(
+ messages: list[dict[str, Any]],
+ *,
+ thread_id: str,
+ thread_title: str | None = None,
+ mode: str = "summary",
+ title: str | None = None,
+ tags: list[str] | None = None,
+ project_name: str | None = "zncm",
+ sources: list[SourceDraft] | None = None,
+ max_messages: int = 80,
+) -> NoteDraft | None:
+ """Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
+ turns = split_turns(messages, max_messages=max_messages)
+ if not turns:
+ return None
+ final_title = (title or "").strip() or _auto_title(thread_title, turns)
+ summary = _summary(turns)
+ sources = sources or []
+ body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
+ base_tags = list(tags or [])
+ for default in (project_name or "zncm", "knowledge-base"):
+ if default and default not in base_tags:
+ base_tags.append(default)
+ return NoteDraft(
+ title=final_title,
+ summary=summary,
+ content_md=body,
+ category="synthesis",
+ source_type="thread",
+ source_id=thread_id,
+ tags=base_tags,
+ confidence=0.6,
+ related=_related_links(project_name),
+ sources=sources,
+ source_key=f"thread:{thread_id}",
+ )
+
+
+def extract_search(
+ query: str,
+ sources: list[SourceDraft],
+ *,
+ thread_id: str | None = None,
+ project_name: str | None = "zncm",
+ title: str | None = None,
+ tags: list[str] | None = None,
+) -> NoteDraft:
+ """Build a reference-category :class:`NoteDraft` from search/tool results."""
+ final_title = (title or "").strip() or f"检索:{query[:40]}"
+ body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
+ for s in sources:
+ label = s.title or s.url or "来源"
+ line = f"- **{label}**"
+ if s.url:
+ line += f" — {s.url}"
+ if s.snippet:
+ line += "\n - " + _one_line(s.snippet)
+ body_lines.append(line)
+ body_lines += [
+ "\n## Reasoning\n",
+ "本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
+ "## Implications\n",
+ "- 引用前请核对原始来源。\n",
+ "## Related\n",
+ *[f"- {link}" for link in _related_links(project_name)],
+ ]
+ base_tags = list(tags or [])
+ for default in (project_name or "zncm", "knowledge-base", "reference"):
+ if default and default not in base_tags:
+ base_tags.append(default)
+ return NoteDraft(
+ title=final_title,
+ summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
+ content_md="\n".join(body_lines),
+ category="references",
+ source_type="search",
+ source_id=f"search:{query[:120]}",
+ tags=base_tags,
+ confidence=0.4,
+ related=_related_links(project_name),
+ sources=sources,
+ source_key=None,
+ )
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py
new file mode 100644
index 00000000..5c86f697
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py
@@ -0,0 +1,354 @@
+"""LLM-based knowledge extraction (wiki-capture style).
+
+Turns a conversation into a *declarative* knowledge note instead of dumping the
+raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note,
+keep only high-value findings/decisions, drop greetings / intermediate
+reasoning / verbatim transcripts, mark inferred content, and emit structured
+JSON that we render into the Context / Finding / Reasoning / Implications /
+Related body.
+
+Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread`
+when the LLM is unavailable or returns something unusable, so capture never
+hard-fails.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+from deerflow.knowledge import extractor
+from deerflow.knowledge.schemas import NoteDraft, SourceDraft
+from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE
+
+logger = logging.getLogger(__name__)
+
+_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"}
+
+
+class LLMRejectedIngest(Exception):
+ """The LLM judged the conversation as having no knowledge value.
+
+ Raised (instead of returning ``[]``) when the model explicitly sets
+ ``should_ingest: false`` with no notes — so callers can *skip* ingestion
+ entirely rather than falling back to the rule-based verbatim dump.
+ """
+
+
+def _is_falsey(value: Any) -> bool:
+ if value is False:
+ return True
+ if isinstance(value, str):
+ return value.strip().lower() in ("false", "0", "no")
+ return False
+
+_SYSTEM_PROMPT = """\
+你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\
+**陈述式知识(declarative knowledge)**,而不是聊天流水账。
+
+要求:
+- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。
+- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。
+- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。
+- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
+- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
+references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
+- 用中文输出。
+
+一次对话可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
+- 默认只产出 **1 页**;只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
+- 每一页聚焦单一主题,标题与 summary 必须彼此区分,不要把同一结论拆到多页。
+- 不值得沉淀时返回空的 notes 数组。
+
+只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
+{
+ "should_ingest": true,
+ "reason": "为什么值得/不值得沉淀",
+ "notes": [
+ {
+ "title": "简洁的知识标题(不超过 40 字)",
+ "summary": "1-2 句话概括这页知识",
+ "category": "synthesis",
+ "context": "问题背景或触发场景,1-3 句",
+ "findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
+ "reasoning": "为什么成立、取舍、不确定性",
+ "implications": ["后续建议/风险/下一步", "..."],
+ "tags": ["标签1", "标签2"],
+ "entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
+ "relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
+ "confidence": 0.0
+ }
+ ]
+}"""
+
+
+def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
+ parts: list[str] = []
+ for i, t in enumerate(turns, 1):
+ if t.get("user"):
+ parts.append(f"[用户 {i}] {t['user']}")
+ if t.get("ai"):
+ parts.append(f"[助手 {i}] {t['ai']}")
+ if sources:
+ parts.append("\n[来源]")
+ for s in sources[:20]:
+ label = s.title or s.url or s.tool_name or "来源"
+ parts.append(f"- {label}" + (f" ({s.url})" if s.url else ""))
+ text = "\n\n".join(parts)
+ if len(text) > max_chars:
+ text = text[:max_chars] + "\n…(已截断)"
+ return text
+
+
+def _parse_json(raw: str) -> dict[str, Any] | None:
+ if not raw:
+ return None
+ text = raw.strip()
+ # Strip ```json fences if present.
+ fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
+ if fence:
+ text = fence.group(1).strip()
+ # Otherwise grab the outermost { ... }.
+ if not text.startswith("{"):
+ start = text.find("{")
+ end = text.rfind("}")
+ if start >= 0 and end > start:
+ text = text[start : end + 1]
+ try:
+ data = json.loads(text)
+ return data if isinstance(data, dict) else None
+ except json.JSONDecodeError:
+ logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based")
+ return None
+
+
+def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str:
+ def _bullets(items: Any) -> str:
+ if isinstance(items, str):
+ items = [items]
+ if not isinstance(items, list):
+ return ""
+ return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip())
+
+ context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]"
+ findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_"
+ reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)"
+ implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。"
+
+ related_lines = [f"- {link}" for link in related]
+ source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
+ related_section = extractor._related_section_md(related_lines, source_lines)
+
+ return (
+ f"# {title}\n\n"
+ f"## Context\n\n{context}\n\n"
+ f"## Finding / Decision\n\n{findings}\n\n"
+ f"## Reasoning\n\n{reasoning}\n\n"
+ f"## Implications\n\n{implications}\n\n"
+ f"{related_section}"
+ )
+
+
+def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
+ """Normalize the LLM JSON into a list of per-note dicts.
+
+ Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy
+ single-note shape (``{"title": ..., "findings": ...}``) for backward
+ compatibility. Notes without any title/findings are dropped.
+ """
+ raw_notes = data.get("notes")
+ if isinstance(raw_notes, list):
+ candidates = [n for n in raw_notes if isinstance(n, dict)]
+ elif data.get("title") or data.get("findings") or data.get("context"):
+ candidates = [data] # legacy single-object output
+ else:
+ candidates = []
+ return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")]
+
+
+def _build_note_draft(
+ note_data: dict[str, Any],
+ *,
+ thread_id: str,
+ thread_title: str | None,
+ turns: list[dict[str, str]],
+ title_override: str | None,
+ tags: list[str] | None,
+ project_name: str | None,
+ sources: list[SourceDraft],
+ source_key_suffix: str = "",
+) -> NoteDraft:
+ """Render one note dict into a :class:`NoteDraft`."""
+ category = str(note_data.get("category") or "synthesis").strip().lower()
+ if category not in _VALID_CATEGORIES:
+ category = "synthesis"
+
+ final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
+ summary = str(note_data.get("summary") or "").strip()
+ related = extractor.related_links(project_name)
+ body = _render_body(note_data, final_title, related, sources)
+
+ base_tags = list(tags or [])
+ llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else []
+ for tag in [*llm_tags, project_name or "zncm", "knowledge-base"]:
+ tag = str(tag).strip()
+ if tag and tag not in base_tags:
+ base_tags.append(tag)
+
+ try:
+ confidence = float(note_data.get("confidence") or 0.7)
+ except (TypeError, ValueError):
+ confidence = 0.7
+
+ entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else []
+ relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else []
+
+ return NoteDraft(
+ title=final_title,
+ summary=summary or final_title,
+ content_md=body,
+ category=category,
+ source_type="thread",
+ source_id=thread_id,
+ tags=base_tags,
+ confidence=max(0.0, min(confidence, 1.0)),
+ related=related,
+ sources=sources,
+ entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
+ relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
+ source_key=f"thread:{thread_id}{source_key_suffix}",
+ )
+
+
+async def extract_thread_llm_multi(
+ messages: list[dict[str, Any]],
+ *,
+ thread_id: str,
+ thread_title: str | None = None,
+ mode: str = "summary",
+ title: str | None = None,
+ tags: list[str] | None = None,
+ project_name: str | None = "zncm",
+ sources: list[SourceDraft] | None = None,
+ max_messages: int = 80,
+ model_name: str | None = None,
+ max_input_chars: int = 12000,
+ allow_multi: bool = True,
+ max_notes: int = 4,
+) -> list[NoteDraft]:
+ """Distill a thread into one *or several* cross-linkable knowledge notes.
+
+ The LLM may split a conversation that covers multiple independent topics into
+ several declarative notes. Returns ``[]`` when there is nothing to sediment
+ or the LLM path fails — callers should fall back to the rule-based extractor.
+ When ``allow_multi`` is false only the first note is kept.
+ """
+ turns = extractor.split_turns(messages, max_messages=max_messages)
+ if not turns:
+ return []
+ sources = sources or []
+
+ try:
+ from langchain_core.messages import HumanMessage, SystemMessage
+
+ from deerflow.models import create_chat_model
+
+ model = create_chat_model(name=model_name or None, thinking_enabled=False)
+ transcript = _build_transcript(turns, sources, max_chars=max_input_chars)
+ prompt = [
+ SystemMessage(content=_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
+ HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"),
+ ]
+ response = await model.ainvoke(prompt)
+ raw = response.content if isinstance(response.content, str) else str(response.content)
+ data = _parse_json(raw)
+ except Exception:
+ logger.exception("[knowledge] thread %s — LLM extraction call failed; using rule-based fallback", thread_id)
+ return []
+
+ if not data:
+ logger.warning("[knowledge] thread %s — LLM returned no parsable JSON; using rule-based fallback", thread_id)
+ return []
+
+ note_dicts = _notes_from_data(data)
+ if not note_dicts:
+ reason = str(data.get("reason") or "")[:200]
+ if _is_falsey(data.get("should_ingest")):
+ # The model deliberately judged this conversation worthless (greeting,
+ # meta chatter, no findings). Skip ingestion entirely — do NOT fall
+ # back to the rule-based verbatim dump.
+ logger.warning("[knowledge] thread %s SKIPPED — model judged not worth ingesting: %s", thread_id, reason)
+ raise LLMRejectedIngest(reason or "no knowledge value")
+ logger.warning(
+ "[knowledge] thread %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
+ thread_id,
+ data.get("should_ingest"),
+ reason,
+ )
+ return []
+ if not allow_multi:
+ note_dicts = note_dicts[:1]
+ else:
+ note_dicts = note_dicts[: max(1, max_notes)]
+
+ drafts: list[NoteDraft] = []
+ for i, nd in enumerate(note_dicts):
+ # The user-supplied title override only applies to a single-note result;
+ # multi-note splits keep each note's own LLM title.
+ override = title if (len(note_dicts) == 1) else None
+ suffix = "" if len(note_dicts) == 1 else f"#{i}"
+ drafts.append(
+ _build_note_draft(
+ nd,
+ thread_id=thread_id,
+ thread_title=thread_title,
+ turns=turns,
+ title_override=override,
+ tags=tags,
+ project_name=project_name,
+ sources=sources,
+ source_key_suffix=suffix,
+ )
+ )
+ return drafts
+
+
+async def extract_thread_llm(
+ messages: list[dict[str, Any]],
+ *,
+ thread_id: str,
+ thread_title: str | None = None,
+ mode: str = "summary",
+ title: str | None = None,
+ tags: list[str] | None = None,
+ project_name: str | None = "zncm",
+ sources: list[SourceDraft] | None = None,
+ max_messages: int = 80,
+ model_name: str | None = None,
+ max_input_chars: int = 12000,
+) -> NoteDraft | None:
+ """Distill a thread into a single :class:`NoteDraft` (backward-compatible).
+
+ Thin wrapper over :func:`extract_thread_llm_multi` that returns the first
+ note, or ``None`` when nothing was produced.
+ """
+ try:
+ drafts = await extract_thread_llm_multi(
+ messages,
+ thread_id=thread_id,
+ thread_title=thread_title,
+ mode=mode,
+ title=title,
+ tags=tags,
+ project_name=project_name,
+ sources=sources,
+ max_messages=max_messages,
+ model_name=model_name,
+ max_input_chars=max_input_chars,
+ allow_multi=False,
+ )
+ except LLMRejectedIngest:
+ return None
+ return drafts[0] if drafts else None
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/manifest.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/manifest.py
new file mode 100644
index 00000000..7dc67ab9
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/manifest.py
@@ -0,0 +1,74 @@
+""".manifest.json read/write for the vault.
+
+The manifest records, per page, the source key, content hash, vault path and
+update time — so batch ingest can skip already-sedimented threads (delta/dedup)
+exactly like obsidian-wiki's native ``.manifest.json``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+
+def content_hash(text: str) -> str:
+ """Stable sha256 hex digest of ``text`` (used for dedup)."""
+ return hashlib.sha256((text or "").encode("utf-8")).hexdigest()
+
+
+class Manifest:
+ """JSON manifest stored at ``/.manifest.json``."""
+
+ def __init__(self, root: Path | str) -> None:
+ self.path = Path(root).resolve() / ".manifest.json"
+
+ def load(self) -> dict[str, Any]:
+ if not self.path.exists():
+ return {"version": 1, "updated": None, "pages": {}, "sources": {}}
+ try:
+ data = json.loads(self.path.read_text(encoding="utf-8"))
+ except (json.JSONDecodeError, OSError):
+ return {"version": 1, "updated": None, "pages": {}, "sources": {}}
+ data.setdefault("pages", {})
+ data.setdefault("sources", {})
+ return data
+
+ def save(self, data: dict[str, Any]) -> None:
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ self.path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+ def record_page(
+ self,
+ *,
+ note_id: str,
+ vault_path: str,
+ source_key: str | None,
+ hash_value: str,
+ updated: str,
+ ) -> None:
+ data = self.load()
+ data["pages"][note_id] = {
+ "vault_path": vault_path,
+ "source": source_key,
+ "hash": hash_value,
+ "updated": updated,
+ }
+ if source_key:
+ data["sources"][source_key] = {"note_id": note_id, "hash": hash_value, "updated": updated}
+ data["updated"] = updated
+ self.save(data)
+
+ def remove_page(self, note_id: str) -> None:
+ data = self.load()
+ page = data["pages"].pop(note_id, None)
+ if page and page.get("source"):
+ data["sources"].pop(page["source"], None)
+ self.save(data)
+
+ def has_source(self, source_key: str) -> bool:
+ return source_key in self.load().get("sources", {})
+
+ def get_source(self, source_key: str) -> dict[str, Any] | None:
+ return self.load().get("sources", {}).get(source_key)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py
new file mode 100644
index 00000000..d36e028d
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/markdown_vault.py
@@ -0,0 +1,213 @@
+"""obsidian-wiki compatible Markdown vault writer/reader.
+
+Responsibilities (dev doc §10 ``markdown_vault.py``):
+
+- Generate obsidian-wiki compatible frontmatter.
+- Produce safe, path-traversal-proof file names/paths.
+- Write/update/read Markdown note files.
+- Parse frontmatter + extract ``[[wikilinks]]``.
+
+The vault layout follows §8 of the dev doc and stays compatible with
+``ar9av/obsidian-wiki`` (concepts/ references/ synthesis/ journal/ projects/…,
+plus index.md / log.md / hot.md / .manifest.json / _meta/taxonomy.md /
+wiki-export/).
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
+_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
+_UNSAFE_CHARS_RE = re.compile(r"[^\w一-鿿-]+")
+
+# Top-level vault directories that must exist (obsidian-wiki native layout).
+_VAULT_DIRS = ("concepts", "entities", "skills", "references", "synthesis", "journal", "projects", "_meta", "wiki-export")
+
+
+def slugify(title: str, *, max_len: int = 60) -> str:
+ """Return a filesystem-safe slug (ASCII + CJK), never empty."""
+ text = unicodedata.normalize("NFKC", (title or "").strip()).lower()
+ text = text.replace(" ", "-")
+ text = _UNSAFE_CHARS_RE.sub("-", text).strip("-")
+ text = re.sub(r"-{2,}", "-", text)
+ if len(text) > max_len:
+ text = text[:max_len].rstrip("-")
+ return text or "note"
+
+
+def extract_wikilinks(text: str) -> list[str]:
+ """Return the targets of every ``[[wikilink]]`` in ``text`` (de-duplicated)."""
+ seen: list[str] = []
+ for match in _WIKILINK_RE.finditer(text or ""):
+ target = match.group(1).strip()
+ if target and target not in seen:
+ seen.append(target)
+ return seen
+
+
+def normalize_wikilink_target(target: str) -> str:
+ """Canonicalize a ``[[wikilink]]`` target for resolution/comparison.
+
+ Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a
+ trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to
+ the same canonical ``index`` key used by the index/graph/resolver layers.
+ """
+ t = (target or "").strip()
+ if not t:
+ return ""
+ t = t.split("|", 1)[0].strip()
+ t = t.replace("\\", "/").lstrip("/")
+ if t.lower().endswith(".md"):
+ t = t[:-3]
+ return t
+
+
+class VaultPathError(ValueError):
+ """Raised when a requested vault path escapes the vault root."""
+
+
+class MarkdownVault:
+ """Reads/writes Markdown files inside a single vault root directory."""
+
+ def __init__(self, root: Path | str) -> None:
+ self.root = Path(root).resolve()
+
+ # -- path safety ------------------------------------------------------
+ def resolve(self, rel_path: str) -> Path:
+ """Resolve ``rel_path`` under the vault root, blocking traversal."""
+ rel = str(rel_path).replace("\\", "/").lstrip("/")
+ candidate = (self.root / rel).resolve()
+ try:
+ candidate.relative_to(self.root)
+ except ValueError as exc:
+ raise VaultPathError(f"Path escapes vault root: {rel_path!r}") from exc
+ return candidate
+
+ def note_relpath(self, *, category: str, project_name: str | None, slug: str, note_id: str) -> str:
+ """Build a vault-relative path for a note.
+
+ Project knowledge goes under ``projects///`` per
+ wiki-capture; a short id suffix guarantees uniqueness.
+ """
+ from deerflow.knowledge.skill_templates import category_to_dir
+
+ cat_dir = category_to_dir(category)
+ safe_slug = slugify(slug)
+ suffix = note_id.split("_")[-1][:8]
+ filename = f"{safe_slug}-{suffix}.md"
+ if project_name:
+ return f"projects/{slugify(project_name)}/{cat_dir}/{filename}"
+ return f"{cat_dir}/{filename}"
+
+ # -- scaffold ---------------------------------------------------------
+ def ensure_scaffold(self, *, project_name: str | None = None) -> None:
+ """Create the vault directory tree + base files if missing."""
+ self.root.mkdir(parents=True, exist_ok=True)
+ for d in _VAULT_DIRS:
+ (self.root / d).mkdir(parents=True, exist_ok=True)
+ if project_name:
+ base = self.root / "projects" / slugify(project_name)
+ for d in ("concepts", "references", "synthesis", "journal"):
+ (base / d).mkdir(parents=True, exist_ok=True)
+ # Base vault files.
+ index = self.root / "index.md"
+ if not index.exists():
+ index.write_text("# Knowledge Index\n\n_页面索引。每次写入新页面后自动更新。_\n\n", encoding="utf-8")
+ log = self.root / "log.md"
+ if not log.exists():
+ log.write_text("# Activity Log\n\n_CAPTURE / INGEST / QUERY / EXPORT 记录。_\n\n", encoding="utf-8")
+ hot = self.root / "hot.md"
+ if not hot.exists():
+ hot.write_text("# Hot\n\n_近期活动摘要。_\n\n", encoding="utf-8")
+ taxonomy = self.root / "_meta" / "taxonomy.md"
+ if not taxonomy.exists():
+ taxonomy.write_text(
+ "# Taxonomy\n\n- synthesis — 多步骤分析、方案、结论\n- concepts — 概念、框架、模型\n"
+ "- references — 外部来源、搜索结果、文章资料\n- decision — 架构或设计决策\n- journal — 完整会话摘要\n",
+ encoding="utf-8",
+ )
+
+ # -- frontmatter ------------------------------------------------------
+ @staticmethod
+ def build_frontmatter(
+ *,
+ note_id: str,
+ title: str,
+ category: str,
+ tags: list[str],
+ sources: list[str],
+ summary: str,
+ created: str,
+ updated: str,
+ base_confidence: float,
+ lifecycle: str,
+ provenance: dict[str, float] | None = None,
+ relationships: list[dict[str, str]] | None = None,
+ ) -> dict[str, Any]:
+ """Build an obsidian-wiki / wiki-capture compatible frontmatter dict."""
+ created_date = (created or "")[:10]
+ return {
+ "id": note_id,
+ "title": title,
+ "category": category,
+ "tags": list(tags or []),
+ "sources": list(sources or []),
+ "created": created,
+ "updated": updated,
+ "summary": summary or "",
+ "provenance": provenance or {"extracted": 0.7, "inferred": 0.2, "ambiguous": 0.1},
+ "base_confidence": round(float(base_confidence or 0.0), 2),
+ "lifecycle": lifecycle,
+ "lifecycle_changed": created_date,
+ "relationships": relationships or [],
+ }
+
+ # -- IO ---------------------------------------------------------------
+ def write_note(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> Path:
+ """Write ``frontmatter`` + ``body`` to ``rel_path`` (creates dirs)."""
+ path = self.resolve(rel_path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ fm = yaml.safe_dump(frontmatter, allow_unicode=True, sort_keys=False, default_flow_style=False)
+ path.write_text(f"---\n{fm}---\n\n{body.strip()}\n", encoding="utf-8")
+ return path
+
+ def read_note(self, rel_path: str) -> tuple[dict[str, Any], str] | None:
+ """Return ``(frontmatter, body)`` for a note, or ``None`` if missing."""
+ path = self.resolve(rel_path)
+ if not path.exists():
+ return None
+ return self.parse(path.read_text(encoding="utf-8"))
+
+ @staticmethod
+ def parse(text: str) -> tuple[dict[str, Any], str]:
+ """Split raw Markdown into ``(frontmatter dict, body)``."""
+ match = _FRONTMATTER_RE.match(text or "")
+ if not match:
+ return {}, text or ""
+ try:
+ fm = yaml.safe_load(match.group(1)) or {}
+ except yaml.YAMLError:
+ fm = {}
+ if not isinstance(fm, dict):
+ fm = {}
+ return fm, match.group(2) or ""
+
+ def delete_note(self, rel_path: str) -> None:
+ """Remove a note file if present (used on hard delete; archive keeps it)."""
+ path = self.resolve(rel_path)
+ if path.exists():
+ path.unlink()
+
+
+def now_iso_with_tz(dt: datetime) -> str:
+ """ISO-8601 string with timezone offset (frontmatter ``created``/``updated``)."""
+ if dt.tzinfo is None:
+ return dt.isoformat()
+ return dt.isoformat(timespec="seconds")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/models.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/models.py
new file mode 100644
index 00000000..07a4e338
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/models.py
@@ -0,0 +1,141 @@
+"""ORM models for the product knowledge base.
+
+Tables mirror the obsidian-wiki vault into the database so the product can
+list, filter, search and audit notes quickly without re-parsing Markdown. The
+vault Markdown remains the source-of-truth for ``content_md`` / frontmatter /
+graph; these rows are an index + audit layer.
+
+The knowledge base is **global / shared** — there is no ``user_id`` isolation.
+Writes record ``created_by`` / ``updated_by`` for auditing only.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from deerflow.persistence.base import Base
+from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
+
+
+class KnowledgeNoteRow(Base):
+ """Knowledge note main table (mirrors ``knowledge_notes``)."""
+
+ __tablename__ = "knowledge_notes"
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ title: Mapped[str] = mapped_column(String(512))
+ summary: Mapped[str | None] = mapped_column(Text)
+ content_md: Mapped[str] = mapped_column(PortableLongText())
+ vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
+ source_type: Mapped[str] = mapped_column(String(64), index=True)
+ source_id: Mapped[str | None] = mapped_column(String(256), index=True)
+ status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
+ confidence: Mapped[float] = mapped_column(Float, default=0.0)
+ created_by: Mapped[str | None] = mapped_column(String(128), index=True)
+ updated_by: Mapped[str | None] = mapped_column(String(128))
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
+ updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), index=True)
+
+
+class KnowledgeSourceRow(Base):
+ """Knowledge source table (mirrors ``knowledge_sources``)."""
+
+ __tablename__ = "knowledge_sources"
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
+ source_type: Mapped[str] = mapped_column(String(64))
+ thread_id: Mapped[str | None] = mapped_column(String(128), index=True)
+ message_id: Mapped[str | None] = mapped_column(String(128))
+ tool_name: Mapped[str | None] = mapped_column(String(128))
+ title: Mapped[str | None] = mapped_column(String(512))
+ url: Mapped[str | None] = mapped_column(Text)
+ snippet: Mapped[str | None] = mapped_column(Text)
+ raw_json: Mapped[str | None] = mapped_column(PortableLongText())
+ created_by: Mapped[str | None] = mapped_column(String(128))
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
+
+
+class KnowledgeTagRow(Base):
+ """Tag table (mirrors ``knowledge_tags``)."""
+
+ __tablename__ = "knowledge_tags"
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
+
+
+class KnowledgeNoteTagRow(Base):
+ """Note ↔ tag join table (mirrors ``knowledge_note_tags``)."""
+
+ __tablename__ = "knowledge_note_tags"
+ __table_args__ = (UniqueConstraint("note_id", "tag_id", name="uq_knowledge_note_tags"),)
+
+ note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), primary_key=True)
+ tag_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_tags.id", ondelete="CASCADE"), primary_key=True)
+
+
+class KnowledgeEmbeddingRow(Base):
+ """Per-chunk embedding index (phase 2, mirrors ``knowledge_embeddings``).
+
+ The vector is stored as a JSON float array; first-release similarity is
+ computed in Python (no external vector DB). ``vector_id`` is reserved for a
+ future external store mapping.
+ """
+
+ __tablename__ = "knowledge_embeddings"
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
+ chunk_index: Mapped[int] = mapped_column(Integer, default=0)
+ content: Mapped[str] = mapped_column(Text)
+ vector: Mapped[list] = mapped_column(PortableJSON(), default=list)
+ vector_id: Mapped[str | None] = mapped_column(String(256))
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
+
+
+class KnowledgeNoteVersionRow(Base):
+ """Immutable snapshot of a note before each update (phase 3 versioning)."""
+
+ __tablename__ = "knowledge_note_versions"
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
+ title: Mapped[str] = mapped_column(String(512))
+ summary: Mapped[str | None] = mapped_column(Text)
+ content_md: Mapped[str] = mapped_column(PortableLongText())
+ status: Mapped[str] = mapped_column(String(32))
+ tags_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
+ edited_by: Mapped[str | None] = mapped_column(String(128))
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
+
+
+class KnowledgeEntityRow(Base):
+ """Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
+
+ __tablename__ = "knowledge_entities"
+ __table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ name: Mapped[str] = mapped_column(String(256), index=True)
+ entity_type: Mapped[str | None] = mapped_column(String(64))
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
+
+
+class KnowledgeRelationRow(Base):
+ """Relation between notes/entities (phase 4, mirrors ``knowledge_relations``)."""
+
+ __tablename__ = "knowledge_relations"
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ from_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
+ to_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
+ from_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
+ to_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
+ relation_type: Mapped[str] = mapped_column(String(128))
+ weight: Mapped[float] = mapped_column(Float, default=1.0)
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/obsidian_adapter.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/obsidian_adapter.py
new file mode 100644
index 00000000..fef599bc
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/obsidian_adapter.py
@@ -0,0 +1,254 @@
+"""ObsidianWikiAdapter — the obsidian-wiki vault facade.
+
+Owns the Markdown vault: writes notes with wiki-capture frontmatter/body, keeps
+``index.md`` / ``log.md`` / ``hot.md`` / ``.manifest.json`` in sync after every
+write, and produces wiki-export graph artifacts. Output stays compatible with
+``ar9av/obsidian-wiki`` without ever invoking its CLI.
+
+This class is storage-only (no DB, no LLM). The :class:`KnowledgeService`
+combines it with the repository + extractor.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from deerflow.knowledge import exporter
+from deerflow.knowledge.manifest import Manifest, content_hash
+from deerflow.knowledge.markdown_vault import MarkdownVault
+from deerflow.knowledge.schemas import NoteDraft
+
+_INDEX_PAGES_HEADER = "## Pages"
+_HOT_HEADER = "## Recent"
+_HOT_MAX = 20
+
+
+class ObsidianWikiAdapter:
+ """Read/write an obsidian-wiki compatible vault."""
+
+ def __init__(self, vault_root: Path | str, *, project_name: str = "zncm", export_html: bool = True, export_json: bool = True) -> None:
+ self.vault = MarkdownVault(vault_root)
+ self.root = self.vault.root
+ self.manifest = Manifest(self.root)
+ self.project_name = project_name
+ self.export_html = export_html
+ self.export_json = export_json
+
+ def ensure_ready(self) -> None:
+ self.vault.ensure_scaffold(project_name=self.project_name)
+
+ # -- writes -----------------------------------------------------------
+ def write_note(
+ self,
+ note_id: str,
+ draft: NoteDraft,
+ *,
+ created: str,
+ updated: str,
+ status: str,
+ ) -> str:
+ """Write the note file, sync vault metadata, return the vault path."""
+ self.ensure_ready()
+ rel = self.vault.note_relpath(category=draft.category, project_name=self.project_name, slug=draft.title, note_id=note_id)
+ sources_fm = self._frontmatter_sources(draft)
+ relationships = [{"target": link, "type": "related"} for link in (draft.related or [])[:8]]
+ frontmatter = self.vault.build_frontmatter(
+ note_id=note_id,
+ title=draft.title,
+ category=draft.category,
+ tags=draft.tags,
+ sources=sources_fm,
+ summary=draft.summary,
+ created=created,
+ updated=updated,
+ base_confidence=draft.confidence,
+ lifecycle=status,
+ relationships=relationships,
+ )
+ self.vault.write_note(rel, frontmatter, draft.content_md)
+ self.manifest.record_page(
+ note_id=note_id,
+ vault_path=rel,
+ source_key=draft.source_key,
+ hash_value=content_hash(draft.content_md),
+ updated=updated,
+ )
+ self._update_index(note_id, rel, draft.title, draft.summary)
+ self._append_log("CAPTURE" if draft.source_type in ("thread", "search", "tool") else "INGEST", note_id, draft.title, updated)
+ self._update_hot(draft.title, rel, updated)
+ return rel
+
+ def update_note_file(
+ self,
+ note_id: str,
+ vault_path: str | None,
+ *,
+ title: str,
+ content_md: str,
+ tags: list[str],
+ status: str,
+ summary: str,
+ category: str,
+ created: str,
+ updated: str,
+ ) -> str:
+ """Rewrite a note file in place (frontmatter refreshed), sync metadata."""
+ self.ensure_ready()
+ rel = vault_path
+ if not rel:
+ rel = self.vault.note_relpath(category=category, project_name=self.project_name, slug=title, note_id=note_id)
+ # Preserve original `sources`/`provenance`/`category` frontmatter when present.
+ existing = self.vault.read_note(rel)
+ sources_fm = existing[0].get("sources", []) if existing else []
+ provenance = existing[0].get("provenance") if existing else None
+ relationships = existing[0].get("relationships") if existing else []
+ effective_category = (existing[0].get("category") if existing else None) or category
+ frontmatter = self.vault.build_frontmatter(
+ note_id=note_id,
+ title=title,
+ category=effective_category,
+ tags=tags,
+ sources=sources_fm,
+ summary=summary,
+ created=created,
+ updated=updated,
+ base_confidence=float((existing[0].get("base_confidence") if existing else 0.0) or 0.0),
+ lifecycle=status,
+ provenance=provenance,
+ relationships=relationships,
+ )
+ self.vault.write_note(rel, frontmatter, content_md)
+ self.manifest.record_page(
+ note_id=note_id,
+ vault_path=rel,
+ source_key=self.manifest.load().get("pages", {}).get(note_id, {}).get("source"),
+ hash_value=content_hash(content_md),
+ updated=updated,
+ )
+ self._update_index(note_id, rel, title, summary)
+ self._append_log("UPDATE", note_id, title, updated)
+ self._update_hot(title, rel, updated)
+ return rel
+
+ def archive_note(self, note_id: str, vault_path: str | None, *, updated: str) -> None:
+ """Mark a note archived in its frontmatter + log (soft delete)."""
+ if vault_path:
+ parsed = self.vault.read_note(vault_path)
+ if parsed:
+ fm, body = parsed
+ fm["lifecycle"] = "archived"
+ fm["updated"] = updated
+ self.vault.write_note(vault_path, fm, body)
+ self._remove_from_index(note_id)
+ self._append_log("ARCHIVE", note_id, "", updated)
+
+ def export_graph(
+ self,
+ notes: list[dict[str, Any]],
+ *,
+ entities: list[dict[str, Any]] | None = None,
+ relations: list[dict[str, Any]] | None = None,
+ updated: str,
+ ) -> dict[str, Any]:
+ self.ensure_ready()
+ result = exporter.export_all(
+ self.root,
+ notes,
+ entities=entities,
+ relations=relations,
+ write_html=self.export_html,
+ write_json=self.export_json,
+ )
+ self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
+ return result
+
+ def export_file_path(self, filename: str) -> Path | None:
+ allowed = {"graph.json", "graph.graphml", "cypher.txt", "graph.html"}
+ if filename not in allowed:
+ return None
+ path = (self.root / "wiki-export" / filename).resolve()
+ try:
+ path.relative_to(self.root)
+ except ValueError:
+ return None
+ return path if path.exists() else None
+
+ # -- frontmatter source serialization ---------------------------------
+ @staticmethod
+ def _frontmatter_sources(draft: NoteDraft) -> list[str]:
+ out: list[str] = []
+ if draft.source_type == "thread" and draft.source_id:
+ out.append(f"thread:{draft.source_id}")
+ for s in draft.sources[:20]:
+ if s.url:
+ out.append(f"url:{s.url}")
+ elif s.thread_id:
+ out.append(f"thread:{s.thread_id}")
+ elif s.tool_name:
+ out.append(f"tool:{s.tool_name}")
+ # De-dup preserving order.
+ seen: set[str] = set()
+ deduped = []
+ for item in out:
+ if item not in seen:
+ seen.add(item)
+ deduped.append(item)
+ return deduped
+
+ # -- index.md ---------------------------------------------------------
+ def _index_path(self) -> Path:
+ return self.root / "index.md"
+
+ def _update_index(self, note_id: str, rel: str, title: str, summary: str | None) -> None:
+ path = self._index_path()
+ text = path.read_text(encoding="utf-8") if path.exists() else "# Knowledge Index\n\n"
+ link = rel[:-3] if rel.endswith(".md") else rel
+ summary_txt = (summary or "").strip().replace("\n", " ")
+ if len(summary_txt) > 120:
+ summary_txt = summary_txt[:120] + "…"
+ line = f"- [[{link}|{title}]]" + (f" — {summary_txt}" if summary_txt else "") + f" "
+ lines = text.splitlines()
+ # Drop any existing line for this note id.
+ lines = [ln for ln in lines if f"" not in ln]
+ if _INDEX_PAGES_HEADER not in text:
+ lines.append("")
+ lines.append(_INDEX_PAGES_HEADER)
+ lines.append("")
+ # Insert after the Pages header.
+ try:
+ insert_at = lines.index(_INDEX_PAGES_HEADER) + 1
+ except ValueError:
+ insert_at = len(lines)
+ lines.insert(insert_at, line)
+ path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
+
+ def _remove_from_index(self, note_id: str) -> None:
+ path = self._index_path()
+ if not path.exists():
+ return
+ lines = [ln for ln in path.read_text(encoding="utf-8").splitlines() if f"" not in ln]
+ path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
+
+ # -- log.md -----------------------------------------------------------
+ def _append_log(self, action: str, note_id: str, label: str, updated: str) -> None:
+ path = self.root / "log.md"
+ text = path.read_text(encoding="utf-8") if path.exists() else "# Activity Log\n\n"
+ entry = f"- {updated} {action} {note_id}" + (f" — {label}" if label else "")
+ path.write_text(text.rstrip() + "\n" + entry + "\n", encoding="utf-8")
+
+ # -- hot.md -----------------------------------------------------------
+ def _update_hot(self, title: str, rel: str, updated: str) -> None:
+ path = self.root / "hot.md"
+ link = rel[:-3] if rel.endswith(".md") else rel
+ new_line = f"- {updated} [[{link}|{title}]]"
+ existing_lines: list[str] = []
+ if path.exists():
+ for ln in path.read_text(encoding="utf-8").splitlines():
+ if ln.startswith("- ") and "[[" in ln:
+ existing_lines.append(ln)
+ existing_lines = [ln for ln in existing_lines if f"[[{link}" not in ln]
+ existing_lines.insert(0, new_line)
+ existing_lines = existing_lines[:_HOT_MAX]
+ body = "# Hot\n\n_近期活动摘要。_\n\n" + _HOT_HEADER + "\n\n" + "\n".join(existing_lines) + "\n"
+ path.write_text(body, encoding="utf-8")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py
new file mode 100644
index 00000000..6481d79f
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/repository.py
@@ -0,0 +1,528 @@
+"""Database read/write for the knowledge base.
+
+This layer never calls an LLM and never touches the Markdown vault — it only
+persists/queries the relational mirror (notes, sources, tags). The vault and
+extraction live in :mod:`deerflow.knowledge.markdown_vault` and
+:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import delete, func, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from deerflow.knowledge.models import (
+ KnowledgeEmbeddingRow,
+ KnowledgeEntityRow,
+ KnowledgeNoteRow,
+ KnowledgeNoteTagRow,
+ KnowledgeNoteVersionRow,
+ KnowledgeRelationRow,
+ KnowledgeSourceRow,
+ KnowledgeTagRow,
+)
+from deerflow.knowledge.schemas import SourceDraft
+
+
+def _new_id(prefix: str) -> str:
+ return f"{prefix}_{uuid.uuid4().hex[:24]}"
+
+
+def _iso(value: Any) -> Any:
+ if isinstance(value, datetime):
+ return value.isoformat()
+ return value
+
+
+class KnowledgeRepository:
+ """SQLAlchemy-backed repository for knowledge notes/sources/tags."""
+
+ def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
+ self._sf = session_factory
+
+ # -- serialization ----------------------------------------------------
+ @staticmethod
+ def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
+ d = row.to_dict()
+ d["created_at"] = _iso(d.get("created_at"))
+ d["updated_at"] = _iso(d.get("updated_at"))
+ return d
+
+ @staticmethod
+ def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
+ d = row.to_dict()
+ d["created_at"] = _iso(d.get("created_at"))
+ return d
+
+ # -- tags -------------------------------------------------------------
+ async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
+ cleaned = []
+ seen: set[str] = set()
+ for raw in names or []:
+ name = (raw or "").strip()
+ if not name or name.lower() in seen:
+ continue
+ seen.add(name.lower())
+ cleaned.append(name)
+ if not cleaned:
+ return []
+ existing = {
+ r.name.lower(): r
+ for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
+ }
+ result: list[KnowledgeTagRow] = []
+ for name in cleaned:
+ row = existing.get(name.lower())
+ if row is None:
+ row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
+ session.add(row)
+ await session.flush()
+ existing[name.lower()] = row
+ result.append(row)
+ return result
+
+ async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
+ stmt = (
+ select(KnowledgeTagRow.name)
+ .join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
+ .where(KnowledgeNoteTagRow.note_id == note_id)
+ .order_by(KnowledgeTagRow.name)
+ )
+ return list((await session.execute(stmt)).scalars())
+
+ async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
+ if not note_ids:
+ return {}
+ stmt = (
+ select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
+ .join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
+ .where(KnowledgeNoteTagRow.note_id.in_(note_ids))
+ .order_by(KnowledgeTagRow.name)
+ )
+ out: dict[str, list[str]] = {nid: [] for nid in note_ids}
+ for note_id, name in (await session.execute(stmt)).all():
+ out.setdefault(note_id, []).append(name)
+ return out
+
+ async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
+ await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
+ tags = await self._get_or_create_tags(session, names)
+ for tag in tags:
+ session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
+
+ # -- notes ------------------------------------------------------------
+ async def create_note(
+ self,
+ *,
+ title: str,
+ summary: str | None,
+ content_md: str,
+ source_type: str,
+ source_id: str | None,
+ status: str,
+ confidence: float = 0.0,
+ vault_path: str | None = None,
+ tags: list[str] | None = None,
+ created_by: str | None = None,
+ ) -> dict[str, Any]:
+ note_id = _new_id("kb")
+ now = datetime.now(UTC)
+ row = KnowledgeNoteRow(
+ id=note_id,
+ title=title[:512],
+ summary=summary,
+ content_md=content_md,
+ vault_path=vault_path,
+ source_type=source_type,
+ source_id=source_id,
+ status=status,
+ confidence=confidence,
+ created_by=created_by,
+ updated_by=created_by,
+ created_at=now,
+ updated_at=now,
+ )
+ async with self._sf() as session:
+ session.add(row)
+ await session.flush()
+ await self._replace_note_tags(session, note_id, tags or [])
+ await session.commit()
+ await session.refresh(row)
+ d = self._note_to_dict(row)
+ d["tags"] = await self._tags_for(session, note_id)
+ return d
+
+ async def set_vault_path(self, note_id: str, vault_path: str) -> None:
+ async with self._sf() as session:
+ row = await session.get(KnowledgeNoteRow, note_id)
+ if row is None:
+ return
+ row.vault_path = vault_path
+ await session.commit()
+
+ async def update_note(
+ self,
+ note_id: str,
+ *,
+ title: str | None = None,
+ summary: str | None = None,
+ content_md: str | None = None,
+ status: str | None = None,
+ tags: list[str] | None = None,
+ vault_path: str | None = None,
+ updated_by: str | None = None,
+ ) -> dict[str, Any] | None:
+ async with self._sf() as session:
+ row = await session.get(KnowledgeNoteRow, note_id)
+ if row is None:
+ return None
+ if title is not None:
+ row.title = title[:512]
+ if summary is not None:
+ row.summary = summary
+ if content_md is not None:
+ row.content_md = content_md
+ if status is not None:
+ row.status = status
+ if vault_path is not None:
+ row.vault_path = vault_path
+ if updated_by is not None:
+ row.updated_by = updated_by
+ row.updated_at = datetime.now(UTC)
+ if tags is not None:
+ await self._replace_note_tags(session, note_id, tags)
+ await session.commit()
+ await session.refresh(row)
+ d = self._note_to_dict(row)
+ d["tags"] = await self._tags_for(session, note_id)
+ return d
+
+ async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
+ return await self.update_note(note_id, status=status, updated_by=updated_by)
+
+ async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
+ async with self._sf() as session:
+ row = await session.get(KnowledgeNoteRow, note_id)
+ if row is None:
+ return None
+ d = self._note_to_dict(row)
+ d["tags"] = await self._tags_for(session, note_id)
+ if with_sources:
+ stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
+ d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
+ return d
+
+ async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None:
+ """Return the first (non-archived) note whose vault_path matches.
+
+ ``vault_path`` is matched both with and without the ``.md`` suffix, since
+ wikilink targets are stored without it while ``vault_path`` keeps it.
+ """
+ stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path
+ candidates = {vault_path, stripped, f"{stripped}.md"}
+ async with self._sf() as session:
+ stmt = (
+ select(KnowledgeNoteRow)
+ .where(KnowledgeNoteRow.vault_path.in_(list(candidates)))
+ .where(KnowledgeNoteRow.status != "archived")
+ .limit(1)
+ )
+ row = (await session.execute(stmt)).scalars().first()
+ if row is None:
+ return None
+ d = self._note_to_dict(row)
+ d["tags"] = await self._tags_for(session, row.id)
+ return d
+
+ async def find_by_title(self, title: str) -> dict[str, Any] | None:
+ """Return the first (non-archived) note with a case-insensitive title match."""
+ name = (title or "").strip()
+ if not name:
+ return None
+ async with self._sf() as session:
+ stmt = (
+ select(KnowledgeNoteRow)
+ .where(func.lower(KnowledgeNoteRow.title) == name.lower())
+ .where(KnowledgeNoteRow.status != "archived")
+ .order_by(KnowledgeNoteRow.updated_at.desc())
+ .limit(1)
+ )
+ row = (await session.execute(stmt)).scalars().first()
+ if row is None:
+ return None
+ d = self._note_to_dict(row)
+ d["tags"] = await self._tags_for(session, row.id)
+ return d
+
+ async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
+ """Return the first (non-archived) note for a (source_type, source_id)."""
+ async with self._sf() as session:
+ stmt = (
+ select(KnowledgeNoteRow)
+ .where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
+ .where(KnowledgeNoteRow.status != "archived")
+ .limit(1)
+ )
+ row = (await session.execute(stmt)).scalars().first()
+ return self._note_to_dict(row) if row else None
+
+ async def list_notes(
+ self,
+ *,
+ keyword: str | None = None,
+ tag: str | None = None,
+ source_type: str | None = None,
+ status: str | None = None,
+ include_archived: bool = False,
+ limit: int = 20,
+ offset: int = 0,
+ ) -> tuple[list[dict[str, Any]], int]:
+ async with self._sf() as session:
+ stmt = select(KnowledgeNoteRow)
+ count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
+
+ conditions = []
+ if status:
+ conditions.append(KnowledgeNoteRow.status == status)
+ elif not include_archived:
+ conditions.append(KnowledgeNoteRow.status != "archived")
+ if source_type:
+ conditions.append(KnowledgeNoteRow.source_type == source_type)
+ if keyword:
+ like = f"%{keyword}%"
+ conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
+ if tag:
+ tag_subq = (
+ select(KnowledgeNoteTagRow.note_id)
+ .join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
+ .where(func.lower(KnowledgeTagRow.name) == tag.lower())
+ )
+ conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
+
+ for cond in conditions:
+ stmt = stmt.where(cond)
+ count_stmt = count_stmt.where(cond)
+
+ total = (await session.execute(count_stmt)).scalar_one()
+ stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
+ rows = list((await session.execute(stmt)).scalars())
+ ids = [r.id for r in rows]
+ tags_map = await self._tags_for_many(session, ids)
+ items = []
+ for r in rows:
+ d = self._note_to_dict(r)
+ d["tags"] = tags_map.get(r.id, [])
+ items.append(d)
+ return items, int(total)
+
+ async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
+ """Return every note (light fields + tags) for graph export."""
+ async with self._sf() as session:
+ stmt = select(KnowledgeNoteRow)
+ if not include_archived:
+ stmt = stmt.where(KnowledgeNoteRow.status != "archived")
+ rows = list((await session.execute(stmt)).scalars())
+ tags_map = await self._tags_for_many(session, [r.id for r in rows])
+ out = []
+ for r in rows:
+ d = self._note_to_dict(r)
+ d["tags"] = tags_map.get(r.id, [])
+ out.append(d)
+ return out
+
+ # -- embeddings (phase 2) --------------------------------------------
+ async def replace_embeddings(self, note_id: str, chunks: list[tuple[int, str, list[float]]]) -> None:
+ """Replace all embedding rows for a note with ``(index, content, vector)``."""
+ async with self._sf() as session:
+ await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
+ for idx, content, vector in chunks:
+ session.add(
+ KnowledgeEmbeddingRow(
+ id=_new_id("kbemb"),
+ note_id=note_id,
+ chunk_index=idx,
+ content=content,
+ vector=list(vector),
+ created_at=datetime.now(UTC),
+ )
+ )
+ await session.commit()
+
+ async def delete_embeddings(self, note_id: str) -> None:
+ async with self._sf() as session:
+ await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
+ await session.commit()
+
+ async def all_embeddings(self) -> list[dict[str, Any]]:
+ """Return every embedding chunk joined to its (non-archived) note."""
+ async with self._sf() as session:
+ stmt = (
+ select(KnowledgeEmbeddingRow, KnowledgeNoteRow.title, KnowledgeNoteRow.status)
+ .join(KnowledgeNoteRow, KnowledgeNoteRow.id == KnowledgeEmbeddingRow.note_id)
+ .where(KnowledgeNoteRow.status != "archived")
+ )
+ out = []
+ for emb, title, _status in (await session.execute(stmt)).all():
+ out.append({"note_id": emb.note_id, "chunk_index": emb.chunk_index, "content": emb.content, "vector": emb.vector or [], "title": title})
+ return out
+
+ async def has_embeddings(self) -> bool:
+ async with self._sf() as session:
+ row = (await session.execute(select(KnowledgeEmbeddingRow.id).limit(1))).first()
+ return row is not None
+
+ # -- versions (phase 3) ----------------------------------------------
+ async def add_version(self, note: dict[str, Any], *, edited_by: str | None) -> None:
+ async with self._sf() as session:
+ session.add(
+ KnowledgeNoteVersionRow(
+ id=_new_id("kbver"),
+ note_id=note["id"],
+ title=note.get("title") or "",
+ summary=note.get("summary"),
+ content_md=note.get("content_md") or "",
+ status=note.get("status") or "approved",
+ tags_json=note.get("tags") or [],
+ edited_by=edited_by,
+ created_at=datetime.now(UTC),
+ )
+ )
+ await session.commit()
+
+ async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
+ async with self._sf() as session:
+ stmt = select(KnowledgeNoteVersionRow).where(KnowledgeNoteVersionRow.note_id == note_id).order_by(KnowledgeNoteVersionRow.created_at.desc())
+ out = []
+ for r in (await session.execute(stmt)).scalars():
+ d = r.to_dict()
+ d["tags"] = d.pop("tags_json", [])
+ d["created_at"] = _iso(d.get("created_at"))
+ out.append(d)
+ return out
+
+ # -- entities & relations (phase 4) ----------------------------------
+ async def get_or_create_entity(self, name: str, entity_type: str | None) -> dict[str, Any]:
+ name = (name or "").strip()
+ async with self._sf() as session:
+ stmt = select(KnowledgeEntityRow).where(func.lower(KnowledgeEntityRow.name) == name.lower())
+ row = (await session.execute(stmt)).scalars().first()
+ if row is None:
+ row = KnowledgeEntityRow(id=_new_id("kbent"), name=name, entity_type=entity_type, created_at=datetime.now(UTC))
+ session.add(row)
+ await session.commit()
+ await session.refresh(row)
+ return {"id": row.id, "name": row.name, "entity_type": row.entity_type}
+
+ async def add_relation(
+ self,
+ *,
+ from_note_id: str | None = None,
+ to_note_id: str | None = None,
+ from_entity_id: str | None = None,
+ to_entity_id: str | None = None,
+ relation_type: str,
+ weight: float = 1.0,
+ ) -> None:
+ async with self._sf() as session:
+ session.add(
+ KnowledgeRelationRow(
+ id=_new_id("kbrel"),
+ from_note_id=from_note_id,
+ to_note_id=to_note_id,
+ from_entity_id=from_entity_id,
+ to_entity_id=to_entity_id,
+ relation_type=relation_type,
+ weight=weight,
+ created_at=datetime.now(UTC),
+ )
+ )
+ await session.commit()
+
+ async def clear_note_relations(self, note_id: str) -> None:
+ async with self._sf() as session:
+ await session.execute(delete(KnowledgeRelationRow).where(KnowledgeRelationRow.from_note_id == note_id))
+ await session.commit()
+
+ async def list_note_entities(self, note_id: str) -> list[dict[str, Any]]:
+ """Return the entities mentioned by a note (for note-detail display)."""
+ async with self._sf() as session:
+ stmt = (
+ select(KnowledgeEntityRow.name, KnowledgeEntityRow.entity_type)
+ .join(KnowledgeRelationRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
+ .where(KnowledgeRelationRow.from_note_id == note_id)
+ .order_by(KnowledgeEntityRow.name)
+ )
+ seen: set[str] = set()
+ out: list[dict[str, Any]] = []
+ for name, etype in (await session.execute(stmt)).all():
+ key = (name or "").lower()
+ if not name or key in seen:
+ continue
+ seen.add(key)
+ out.append({"name": name, "entity_type": etype})
+ return out
+
+ async def prune_orphan_entities(self) -> int:
+ """Delete entity rows no longer referenced by any relation. Returns count."""
+ async with self._sf() as session:
+ referenced: set[str] = set()
+ for col in (KnowledgeRelationRow.from_entity_id, KnowledgeRelationRow.to_entity_id):
+ for (eid,) in (await session.execute(select(col).where(col.isnot(None)))).all():
+ referenced.add(eid)
+ all_ids = list((await session.execute(select(KnowledgeEntityRow.id))).scalars())
+ orphans = [eid for eid in all_ids if eid not in referenced]
+ if orphans:
+ await session.execute(delete(KnowledgeEntityRow).where(KnowledgeEntityRow.id.in_(orphans)))
+ await session.commit()
+ return len(orphans)
+
+ async def list_entities(self) -> list[dict[str, Any]]:
+ async with self._sf() as session:
+ rows = (await session.execute(select(KnowledgeEntityRow))).scalars()
+ return [{"id": r.id, "name": r.name, "entity_type": r.entity_type} for r in rows]
+
+ async def list_relations(self) -> list[dict[str, Any]]:
+ async with self._sf() as session:
+ rows = (await session.execute(select(KnowledgeRelationRow))).scalars()
+ return [
+ {
+ "id": r.id,
+ "from_note_id": r.from_note_id,
+ "to_note_id": r.to_note_id,
+ "from_entity_id": r.from_entity_id,
+ "to_entity_id": r.to_entity_id,
+ "relation_type": r.relation_type,
+ "weight": r.weight,
+ }
+ for r in rows
+ ]
+
+ # -- sources ----------------------------------------------------------
+ async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
+ if not sources:
+ return
+ import json
+
+ async with self._sf() as session:
+ for src in sources:
+ session.add(
+ KnowledgeSourceRow(
+ id=_new_id("kbsrc"),
+ note_id=note_id,
+ source_type=src.source_type,
+ thread_id=src.thread_id,
+ message_id=src.message_id,
+ tool_name=src.tool_name,
+ title=(src.title or "")[:512] or None,
+ url=src.url,
+ snippet=src.snippet,
+ raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
+ created_by=created_by,
+ created_at=datetime.now(UTC),
+ )
+ )
+ await session.commit()
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/schemas.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/schemas.py
new file mode 100644
index 00000000..c46cfdec
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/schemas.py
@@ -0,0 +1,75 @@
+"""Internal domain models for the knowledge module.
+
+These are plain dataclasses passed between the extractor, the vault writer and
+the repository. The Gateway router defines its own Pydantic request/response
+models on top of the dicts the service returns.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+# Allowed note source types (see product-knowledge-base-dev.md §7.1).
+SOURCE_TYPES = ("thread", "search", "tool", "file", "manual", "report")
+# Allowed lifecycle states.
+STATUSES = ("draft", "approved", "archived")
+# obsidian-wiki wiki-capture categories.
+CATEGORIES = ("synthesis", "concepts", "references", "decision", "journal")
+
+
+@dataclass
+class SourceDraft:
+ """A normalized knowledge source (search result, tool call, thread, …)."""
+
+ source_type: str
+ thread_id: str | None = None
+ message_id: str | None = None
+ tool_name: str | None = None
+ title: str | None = None
+ url: str | None = None
+ snippet: str | None = None
+ raw: dict | None = None
+
+
+@dataclass
+class NoteDraft:
+ """A knowledge note ready to be persisted + written to the vault.
+
+ Produced by the extractor (from a thread / search / tool result) or built
+ directly for a manual note. ``content_md`` here is the *body* — the vault
+ writer prepends obsidian-wiki frontmatter.
+ """
+
+ title: str
+ summary: str
+ content_md: str
+ category: str = "synthesis"
+ source_type: str = "manual"
+ source_id: str | None = None
+ status: str = "approved"
+ confidence: float = 0.0
+ tags: list[str] = field(default_factory=list)
+ key_points: list[str] = field(default_factory=list)
+ actions: list[str] = field(default_factory=list)
+ related: list[str] = field(default_factory=list)
+ sources: list[SourceDraft] = field(default_factory=list)
+ # Phase 4: extracted graph signal. entities=[{name,type}],
+ # relations=[{from,to,type}] (names refer to entities or this note's title).
+ entities: list[dict] = field(default_factory=list)
+ relations: list[dict] = field(default_factory=list)
+ # Stable identity hint used for manifest dedup (e.g. ``thread:thread_id``).
+ source_key: str | None = None
+
+
+def normalize_status(status: str | None) -> str:
+ """Coerce an arbitrary status string to a valid lifecycle state."""
+ if status in STATUSES:
+ return status
+ return "approved"
+
+
+def normalize_source_type(source_type: str | None) -> str:
+ """Coerce an arbitrary source-type string to a valid value."""
+ if source_type in SOURCE_TYPES:
+ return source_type
+ return "manual"
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/search.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/search.py
new file mode 100644
index 00000000..f3430e17
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/search.py
@@ -0,0 +1,160 @@
+"""Knowledge search: keyword / vector / hybrid (phase 1 + 2).
+
+- ``keyword`` — title>tags>summary>body scoring over the DB mirror (always works).
+- ``vector`` — cosine over per-chunk embeddings (requires a configured
+ embedding endpoint; otherwise transparently falls back to ``keyword``).
+- ``hybrid`` — min-max-normalized blend of keyword + vector.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Any
+
+from deerflow.knowledge.embeddings import cosine_similarity
+from deerflow.knowledge.repository import KnowledgeRepository
+
+if TYPE_CHECKING:
+ from deerflow.knowledge.embeddings import EmbeddingClient
+
+_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
+
+
+def _tokens(text: str) -> list[str]:
+ return [t.lower() for t in _TOKEN_RE.findall(text or "")]
+
+
+def _snippet(content: str, terms: list[str], *, width: int = 160) -> str:
+ body = re.sub(r"\s+", " ", content or "").strip()
+ low = body.lower()
+ for term in terms:
+ idx = low.find(term)
+ if idx >= 0:
+ start = max(0, idx - width // 3)
+ end = min(len(body), start + width)
+ prefix = "…" if start > 0 else ""
+ suffix = "…" if end < len(body) else ""
+ return f"{prefix}{body[start:end]}{suffix}"
+ return body[:width] + ("…" if len(body) > width else "")
+
+
+async def _keyword_scores(repo: KnowledgeRepository, query: str, *, candidate_cap: int) -> dict[str, tuple[float, dict[str, Any]]]:
+ terms = _tokens(query)
+ if not terms:
+ return {}
+ candidates, _total = await repo.list_notes(limit=candidate_cap, offset=0)
+ scored: dict[str, tuple[float, dict[str, Any]]] = {}
+ for note in candidates:
+ title = (note.get("title") or "").lower()
+ summary = (note.get("summary") or "").lower()
+ content = (note.get("content_md") or "").lower()
+ tags = " ".join(note.get("tags") or []).lower()
+ score = 0.0
+ for term in terms:
+ if term in title:
+ score += 3.0
+ if term in tags:
+ score += 2.0
+ if term in summary:
+ score += 2.0
+ score += min(content.count(term), 5) * 0.5
+ if score > 0:
+ scored[note["id"]] = (score, note)
+ return scored
+
+
+async def _vector_scores(
+ repo: KnowledgeRepository,
+ query: str,
+ embedding_client: EmbeddingClient,
+) -> dict[str, tuple[float, dict[str, Any]]]:
+ qvec = await embedding_client.embed_query(query)
+ if not qvec:
+ return {}
+ embeddings = await repo.all_embeddings()
+ best: dict[str, tuple[float, str, str]] = {} # note_id -> (score, title, snippet)
+ for emb in embeddings:
+ sim = cosine_similarity(qvec, emb.get("vector") or [])
+ if sim <= 0:
+ continue
+ nid = emb["note_id"]
+ if nid not in best or sim > best[nid][0]:
+ best[nid] = (sim, emb.get("title") or "", emb.get("content") or "")
+ return {nid: (score, {"id": nid, "title": title, "_chunk": chunk}) for nid, (score, title, chunk) in best.items()}
+
+
+def _normalize(scores: dict[str, float]) -> dict[str, float]:
+ if not scores:
+ return {}
+ mx = max(scores.values())
+ if mx <= 0:
+ return {k: 0.0 for k in scores}
+ return {k: v / mx for k, v in scores.items()}
+
+
+async def search_notes(
+ repo: KnowledgeRepository,
+ query: str,
+ *,
+ mode: str = "keyword",
+ limit: int = 8,
+ candidate_cap: int = 500,
+ embedding_client: EmbeddingClient | None = None,
+) -> list[dict[str, Any]]:
+ """Score notes against ``query`` and return the top ``limit`` matches."""
+ terms = _tokens(query)
+ if not terms:
+ return []
+
+ want_vector = mode in ("vector", "hybrid") and embedding_client is not None and embedding_client.is_enabled()
+ keyword = await _keyword_scores(repo, query, candidate_cap=candidate_cap) if mode != "vector" or not want_vector else {}
+ vector = await _vector_scores(repo, query, embedding_client) if want_vector else {}
+
+ # Pure vector (when available); else keyword; hybrid blends both.
+ if mode == "vector" and want_vector:
+ combined = {nid: (sc, meta) for nid, (sc, meta) in vector.items()}
+ ranked = sorted(combined.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
+ return await _materialize(repo, ranked, terms)
+
+ if mode == "hybrid" and want_vector:
+ kw_norm = _normalize({k: v[0] for k, v in keyword.items()})
+ vec_norm = _normalize({k: v[0] for k, v in vector.items()})
+ ids = set(kw_norm) | set(vec_norm)
+ blended: dict[str, tuple[float, dict[str, Any]]] = {}
+ for nid in ids:
+ score = 0.5 * kw_norm.get(nid, 0.0) + 0.5 * vec_norm.get(nid, 0.0)
+ meta = keyword.get(nid, (0, None))[1] or vector.get(nid, (0, {}))[1]
+ blended[nid] = (score, meta)
+ ranked = sorted(blended.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
+ return await _materialize(repo, ranked, terms)
+
+ # keyword (default, and fallback when vector unavailable)
+ ranked = sorted(keyword.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
+ if not ranked and vector: # mode=vector requested but keyword empty: use vector
+ ranked = sorted(vector.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
+ return await _materialize(repo, ranked, terms)
+
+
+async def _materialize(repo: KnowledgeRepository, ranked: list[tuple[str, tuple[float, dict[str, Any]]]], terms: list[str]) -> list[dict[str, Any]]:
+ max_score = ranked[0][1][0] if ranked else 1.0
+ results: list[dict[str, Any]] = []
+ for nid, (score, meta) in ranked:
+ note = meta if (meta and meta.get("content_md")) else await repo.get_note(nid, with_sources=False)
+ if note is None:
+ continue
+ snippet_src = note.get("content_md") or meta.get("_chunk") or note.get("summary") or ""
+ results.append(
+ {
+ "note_id": nid,
+ "chunk_id": None,
+ "title": note.get("title") or meta.get("title"),
+ "summary": note.get("summary"),
+ "snippet": _snippet(snippet_src, terms),
+ "score": round(score / max_score, 4) if max_score else 0.0,
+ "tags": note.get("tags") or [],
+ "source_type": note.get("source_type"),
+ "updated_at": note.get("updated_at"),
+ "sources": [],
+ }
+ )
+ return results
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/sensitive.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/sensitive.py
new file mode 100644
index 00000000..ac7e3555
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/sensitive.py
@@ -0,0 +1,54 @@
+"""Sensitive-information redaction for knowledge content (phase 3).
+
+Best-effort masking of secrets / PII before a note is persisted: API keys,
+bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
+ID-card numbers and email addresses. Conservative by design — it favours not
+mangling legitimate prose over catching every possible secret.
+"""
+
+from __future__ import annotations
+
+import re
+
+_MASK = "[REDACTED]"
+
+# Order matters: more specific patterns first.
+_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+ ("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
+ ("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
+ ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
+ ("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
+ # key/secret/password/token assignment: keep the field name, mask the value.
+ ("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
+ ("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
+ ("cn_mobile", re.compile(r"(? tuple[str, list[str]]:
+ """Return ``(redacted_text, hit_kinds)``.
+
+ ``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
+ is falsy it is returned unchanged with no hits.
+ """
+ if not text:
+ return text or "", []
+ hits: list[str] = []
+ out = text
+ for kind, pattern in _PATTERNS:
+ def _sub(m: re.Match[str], _kind: str = kind) -> str:
+ # For assignments keep the field name + delimiter, mask only the value.
+ if _kind == "assignment":
+ head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
+ sep = ":" if ":" in m.group(0) else "="
+ return f"{head}{sep} {_MASK}"
+ if _kind == "bearer":
+ return f"Bearer {_MASK}"
+ return _MASK
+
+ new_out, n = pattern.subn(_sub, out)
+ if n:
+ hits.append(kind)
+ out = new_out
+ return out, hits
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py
new file mode 100644
index 00000000..d1fbe3f9
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py
@@ -0,0 +1,689 @@
+"""KnowledgeService — high-level orchestration for the knowledge base.
+
+Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
+(Markdown vault + index/log/hot/manifest + graph export) and the extractor
+(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
+service; everything obsidian-wiki specific stays inside the ``knowledge``
+module.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from deerflow.config.knowledge_config import KnowledgeConfig
+from deerflow.config.runtime_paths import runtime_home
+from deerflow.knowledge import extractor, llm_extractor
+from deerflow.knowledge.chunking import chunk_markdown
+from deerflow.knowledge.embeddings import EmbeddingClient
+from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target
+from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
+from deerflow.knowledge.repository import KnowledgeRepository
+from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
+from deerflow.knowledge.search import search_notes
+from deerflow.knowledge.sources import from_reference_batches, from_search_results
+
+logger = logging.getLogger(__name__)
+
+
+class EmptyThreadError(ValueError):
+ """Raised when a thread has no sediment-able content."""
+
+
+def _resolve_vault_dir(vault_dir: str) -> Path:
+ path = Path(vault_dir)
+ if path.is_absolute():
+ return path.resolve()
+ return (runtime_home() / vault_dir).resolve()
+
+
+def _inject_related_notes(body: str, links: list[str]) -> str:
+ """Insert real related-note ``[[wikilinks]]`` into the Related block.
+
+ Notes carry only content-specific related notes now (the structural meta-links
+ were dropped), so this is the sole writer of ``## Related`` links. When the
+ body has no ``## Related`` section yet, one is appended.
+ """
+ if not links:
+ return body
+ bullets = "\n".join(f"- {link}" for link in links)
+ marker = "## Related\n"
+ idx = body.find(marker)
+ if idx == -1:
+ return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n"
+ insert_at = idx + len(marker)
+ return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:]
+
+
+class KnowledgeService:
+ """Application-facing knowledge base API."""
+
+ def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
+ self.repo = repo
+ self.config = config
+ self.project_name = config.project_name
+ vault_root = _resolve_vault_dir(config.vault_dir)
+ self.adapter = ObsidianWikiAdapter(
+ vault_root,
+ project_name=config.project_name,
+ export_html=config.export.graph_html_enabled,
+ export_json=config.export.graph_json_enabled,
+ )
+ self.embedding_client = EmbeddingClient(config.embedding)
+ try:
+ self.adapter.ensure_ready()
+ except Exception: # pragma: no cover - vault init best-effort
+ logger.exception("Failed to initialize knowledge vault at %s", vault_root)
+
+ # -- internal helpers -------------------------------------------------
+ def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
+ """Mask secrets/PII in a draft before persistence (phase 3)."""
+ if not self.config.sensitive_redaction_enabled:
+ return draft
+ from deerflow.knowledge.sensitive import redact
+
+ draft.content_md, body_hits = redact(draft.content_md)
+ draft.summary, _ = redact(draft.summary)
+ for src in draft.sources:
+ if src.snippet:
+ src.snippet, _ = redact(src.snippet)
+ if body_hits:
+ logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
+ return draft
+
+ async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]:
+ """Find existing notes to cross-link under ## Related — by **shared tags**.
+
+ Matching is strictly tag-based: a candidate must share at least one of the
+ draft's *distinctive* tags (the per-note project / ``knowledge-base``
+ defaults are dropped, else everything matches everything). Free-text
+ title/summary matching was removed because CJK tokenization made generic
+ words (助手 / 知识图谱 / 页面) link unrelated notes. No distinctive tags →
+ no related notes (better empty than noisy). Best-effort.
+ """
+ if not self.config.related_notes_enabled:
+ return []
+ default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "zncm") if t}
+ distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags]
+ if not distinctive:
+ return []
+ own_title = (draft.title or "").strip().lower()
+ # Rank candidates by how many distinctive tags they share with the draft.
+ shared: dict[str, dict[str, Any]] = {}
+ for tag in distinctive:
+ try:
+ items, _total = await self.repo.list_notes(tag=tag, limit=50, offset=0)
+ except Exception:
+ logger.exception("Related-notes tag lookup failed for tag '%s'", tag)
+ continue
+ for it in items:
+ nid = it.get("id")
+ title = (it.get("title") or "").strip()
+ if not nid or not title or title.lower() == own_title:
+ continue
+ entry = shared.setdefault(nid, {"id": nid, "title": title, "count": 0})
+ entry["count"] += 1
+ ranked = sorted(shared.values(), key=lambda e: e["count"], reverse=True)
+ limit = max(1, self.config.related_notes_limit)
+ return [{"id": e["id"], "title": e["title"]} for e in ranked[:limit]]
+
+ async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
+ """Create a DB row, write the vault file, attach sources, return note."""
+ draft = self._redact_draft(draft)
+ related_notes = await self._find_related_notes(draft)
+ if related_notes:
+ real_links = [f"[[{n['title']}]]" for n in related_notes]
+ draft.content_md = _inject_related_notes(draft.content_md, real_links)
+ draft.related = list(dict.fromkeys([*real_links, *draft.related]))
+ note = await self.repo.create_note(
+ title=draft.title,
+ summary=draft.summary,
+ content_md=draft.content_md,
+ source_type=draft.source_type,
+ source_id=draft.source_id,
+ status=draft.status,
+ confidence=draft.confidence,
+ tags=draft.tags,
+ created_by=created_by,
+ )
+ note_id = note["id"]
+ try:
+ rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
+ await self.repo.set_vault_path(note_id, rel)
+ note["vault_path"] = rel
+ except Exception:
+ logger.exception("Failed to write knowledge note %s to vault", note_id)
+ if draft.sources:
+ await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
+ await self._reindex_embeddings(note_id, draft.content_md)
+ await self._store_entities(note_id, draft)
+ # Record note→note "related" edges (after _store_entities, which clears
+ # this note's relations) so the graph reflects the cross-links too.
+ for rel_note in related_notes:
+ try:
+ await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related")
+ except Exception:
+ logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"])
+ return await self.repo.get_note(note_id) or note
+
+ async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
+ """Persist extracted entities/relations and link them to the note (phase 4)."""
+ if not self.config.entity_extraction_enabled or not draft.entities:
+ return
+ try:
+ await self.repo.clear_note_relations(note_id)
+ name_to_id: dict[str, str] = {}
+ for ent in draft.entities:
+ name = str(ent.get("name") or "").strip()
+ if not name:
+ continue
+ row = await self.repo.get_or_create_entity(name, ent.get("type"))
+ name_to_id[name.lower()] = row["id"]
+ # note -> entity "mentions"
+ await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
+ for rel in draft.relations:
+ a = name_to_id.get(str(rel.get("from") or "").strip().lower())
+ b = name_to_id.get(str(rel.get("to") or "").strip().lower())
+ if a and b and a != b:
+ await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
+ except Exception:
+ logger.exception("Failed to store entities/relations for note %s", note_id)
+
+ async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
+ """Best-effort: (re)build the per-chunk embedding index for a note."""
+ if not self.embedding_client.is_enabled():
+ return
+ try:
+ chunks = chunk_markdown(content_md)
+ if not chunks:
+ await self.repo.delete_embeddings(note_id)
+ return
+ vectors = await self.embedding_client.embed_texts(chunks)
+ if not vectors or len(vectors) != len(chunks):
+ return
+ await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
+ except Exception:
+ logger.exception("Failed to (re)index embeddings for note %s", note_id)
+
+ # -- capture ----------------------------------------------------------
+ async def capture_thread(
+ self,
+ *,
+ thread_id: str,
+ messages: list[dict[str, Any]],
+ thread_title: str | None = None,
+ mode: str = "summary",
+ title: str | None = None,
+ status: str = "approved",
+ tags: list[str] | None = None,
+ include_sources: bool = True,
+ reference_batches: list[dict[str, Any]] | None = None,
+ created_by: str | None = None,
+ skip_existing: bool = False,
+ model_name: str | None = None,
+ ) -> dict[str, Any]:
+ """Sediment a thread into a knowledge note.
+
+ ``model_name`` optionally overrides the configured extraction model for
+ this one capture (falls back to ``config.extract_model_name``).
+
+ Returns ``{"note": , "created": bool}``. Raises
+ :class:`EmptyThreadError` when there is nothing worth keeping.
+ """
+ # Always-on entry log: proves a capture was triggered for this thread,
+ # regardless of how it ends (success / skip / fallback). Lets you tell
+ # "capture never ran" apart from "logs not visible" via one grep.
+ logger.info(
+ "[knowledge] capturing thread %s (msgs=%d, llm_extract=%s, skip_existing=%s)",
+ thread_id,
+ len(messages or []),
+ self.config.llm_extract_enabled,
+ skip_existing,
+ )
+ if skip_existing:
+ existing = await self.repo.find_by_source("thread", thread_id)
+ if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
+ existing = await self.repo.find_by_source("thread", thread_id)
+ if existing:
+ logger.info("[knowledge] thread %s already captured (note %s); skipping", thread_id, existing.get("id"))
+ return {"note": existing, "created": False}
+
+ sources = []
+ if include_sources and reference_batches:
+ sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
+
+ drafts: list[NoteDraft] = []
+ # Prefer LLM distillation (wiki-capture style) so notes are declarative
+ # knowledge, not raw chat. A single conversation may split into several
+ # cross-linked notes. Falls back to the rule-based extractor (always one
+ # note) when the LLM is disabled/unavailable or returns nothing usable.
+ llm_attempted = self.config.llm_extract_enabled
+ llm_fallback = False
+ if llm_attempted:
+ try:
+ drafts = await llm_extractor.extract_thread_llm_multi(
+ messages,
+ thread_id=thread_id,
+ thread_title=thread_title,
+ mode=mode,
+ title=title,
+ tags=tags,
+ project_name=self.project_name,
+ sources=sources,
+ max_messages=self.config.max_thread_messages,
+ model_name=model_name or self.config.extract_model_name,
+ max_input_chars=self.config.extract_max_input_chars,
+ allow_multi=self.config.multi_note_enabled,
+ )
+ except llm_extractor.LLMRejectedIngest as exc:
+ # The model judged this conversation worthless (greeting / meta
+ # chatter). Skip ingestion entirely — do NOT dump the raw chat.
+ raise EmptyThreadError(f"Thread {thread_id} judged not worth ingesting: {exc}") from exc
+ if not drafts:
+ # The LLM path was supposed to run but produced nothing usable
+ # (API error / non-JSON). We will fall back to the rule-based
+ # extractor, which yields a low-quality verbatim dump with no
+ # entities/tags — flag it so it lands as a draft.
+ llm_fallback = True
+ if not drafts:
+ single = extractor.extract_thread(
+ messages,
+ thread_id=thread_id,
+ thread_title=thread_title,
+ mode=mode,
+ title=title,
+ tags=tags,
+ project_name=self.project_name,
+ sources=sources,
+ max_messages=self.config.max_thread_messages,
+ )
+ drafts = [single] if single else []
+ if not drafts:
+ logger.info("[knowledge] thread %s has no sediment-able content; skipping", thread_id)
+ raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
+
+ # When the LLM was enabled but silently fell back to rule-based extraction,
+ # the note is a verbatim dump (no distillation / entities / tags). Land it
+ # as a *draft* for review instead of silently approving it, and log loudly
+ # so the extraction failure is diagnosable (see also the WARNING/EXCEPTION
+ # logged inside llm_extractor.extract_thread_llm_multi).
+ effective_status = status
+ if llm_fallback:
+ logger.warning(
+ "[knowledge] thread %s — capture fell back to rule-based extraction "
+ "(LLM distillation produced nothing usable); landing as DRAFT. "
+ "Check extraction model '%s' / earlier [knowledge] logs for the cause.",
+ thread_id,
+ self.config.extract_model_name or "models[0]",
+ )
+ if normalize_status(status) == "approved":
+ effective_status = "draft"
+
+ normalized = normalize_status(effective_status)
+ for draft in drafts:
+ draft.status = normalized
+ # Cross-link sibling notes from the same conversation by title, so the
+ # split pages reference each other (resolved once all are persisted).
+ if len(drafts) > 1:
+ for i, draft in enumerate(drafts):
+ siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
+ if siblings:
+ draft.content_md = _inject_related_notes(draft.content_md, siblings)
+ draft.related = list(dict.fromkeys([*siblings, *draft.related]))
+
+ notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
+ logger.info(
+ "[knowledge] thread %s captured → %d note(s) [%s], status=%s, llm_fallback=%s",
+ thread_id,
+ len(notes),
+ ", ".join(n.get("title", "") for n in notes),
+ normalized,
+ llm_fallback,
+ )
+ return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
+
+ async def capture_search(
+ self,
+ *,
+ query: str,
+ results: list[dict[str, Any]] | None,
+ thread_id: str | None = None,
+ message_id: str | None = None,
+ title: str | None = None,
+ tags: list[str] | None = None,
+ status: str = "approved",
+ created_by: str | None = None,
+ ) -> dict[str, Any]:
+ """Sediment search/tool results into a reference note."""
+ sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
+ draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
+ draft.status = normalize_status(status)
+ return await self._persist_draft(draft, created_by=created_by)
+
+ async def create_manual_note(
+ self,
+ *,
+ title: str,
+ content_md: str,
+ tags: list[str] | None = None,
+ status: str = "approved",
+ summary: str | None = None,
+ created_by: str | None = None,
+ ) -> dict[str, Any]:
+ """Create a note directly from product-edited Markdown."""
+ # Strip any frontmatter the user may have pasted; vault writer adds its own.
+ _, body = MarkdownVault.parse(content_md)
+ body = body or content_md
+ base_tags = list(tags or [])
+ for default in (self.project_name, "knowledge-base"):
+ if default and default not in base_tags:
+ base_tags.append(default)
+ draft = NoteDraft(
+ title=title.strip() or "知识笔记",
+ summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
+ content_md=body,
+ category="synthesis",
+ source_type="manual",
+ source_id=None,
+ status=normalize_status(status),
+ tags=base_tags,
+ confidence=0.5,
+ related=extractor.related_links(self.project_name),
+ )
+ return await self._persist_draft(draft, created_by=created_by)
+
+ # -- CRUD -------------------------------------------------------------
+ async def list_notes(
+ self,
+ *,
+ keyword: str | None = None,
+ tag: str | None = None,
+ source_type: str | None = None,
+ status: str | None = None,
+ limit: int = 20,
+ offset: int = 0,
+ ) -> dict[str, Any]:
+ items, total = await self.repo.list_notes(
+ keyword=keyword,
+ tag=tag,
+ source_type=normalize_source_type(source_type) if source_type else None,
+ status=status,
+ limit=limit,
+ offset=offset,
+ )
+ return {"items": items, "total": total}
+
+ async def get_note(self, note_id: str) -> dict[str, Any] | None:
+ note = await self.repo.get_note(note_id)
+ if note is not None:
+ note["entities"] = await self.repo.list_note_entities(note_id)
+ return note
+
+ async def update_note(
+ self,
+ note_id: str,
+ *,
+ title: str | None = None,
+ content_md: str | None = None,
+ tags: list[str] | None = None,
+ status: str | None = None,
+ summary: str | None = None,
+ updated_by: str | None = None,
+ ) -> dict[str, Any] | None:
+ current = await self.repo.get_note(note_id, with_sources=False)
+ if current is None:
+ return None
+ # Snapshot the pre-edit state for version history (phase 3).
+ try:
+ await self.repo.add_version(current, edited_by=updated_by)
+ except Exception:
+ logger.exception("Failed to snapshot version for note %s", note_id)
+ body = content_md
+ if body is not None:
+ _, parsed = MarkdownVault.parse(body)
+ body = parsed or body
+ note = await self.repo.update_note(
+ note_id,
+ title=title,
+ summary=summary,
+ content_md=body,
+ status=normalize_status(status) if status else None,
+ tags=tags,
+ updated_by=updated_by,
+ )
+ if note is None:
+ return None
+ try:
+ rel = self.adapter.update_note_file(
+ note_id,
+ current.get("vault_path"),
+ title=note["title"],
+ content_md=note["content_md"],
+ tags=note.get("tags") or [],
+ status=note["status"],
+ summary=note.get("summary") or "",
+ category="synthesis",
+ created=note["created_at"],
+ updated=note["updated_at"],
+ )
+ if rel and rel != current.get("vault_path"):
+ await self.repo.set_vault_path(note_id, rel)
+ except Exception:
+ logger.exception("Failed to update knowledge note %s in vault", note_id)
+ if note.get("content_md") is not None:
+ await self._reindex_embeddings(note_id, note["content_md"])
+ return await self.repo.get_note(note_id)
+
+ async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
+ return await self.repo.list_versions(note_id)
+
+ async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
+ """Soft delete (archive) by default; hard delete removes the vault file."""
+ current = await self.repo.get_note(note_id, with_sources=False)
+ if current is None:
+ return False
+ note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
+ try:
+ self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
+ if hard and current.get("vault_path"):
+ self.adapter.vault.delete_note(current["vault_path"])
+ self.adapter.manifest.remove_page(note_id)
+ await self.repo.delete_embeddings(note_id)
+ except Exception:
+ logger.exception("Failed to archive knowledge note %s in vault", note_id)
+ # Drop this note's entity mentions + any now-orphaned entities so the graph
+ # follows the deletion (no lingering nodes from removed notes).
+ try:
+ await self.repo.clear_note_relations(note_id)
+ pruned = await self.repo.prune_orphan_entities()
+ if pruned:
+ logger.info("[knowledge] deleted note %s → pruned %d orphan entities", note_id, pruned)
+ except Exception:
+ logger.exception("Failed to prune relations/entities for deleted note %s", note_id)
+ return True
+
+ # -- wikilinks --------------------------------------------------------
+ async def resolve_wikilink(self, target: str) -> dict[str, Any]:
+ """Resolve a single ``[[wikilink]]`` target to a navigable destination.
+
+ Returns one of:
+ - ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note
+ - ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page
+ - ``{"kind": "missing", "exists": False}`` — nothing matches
+
+ Resolution order: note id → note vault_path → note title → vault file.
+ """
+ norm = normalize_wikilink_target(target)
+ base = {"target": target, "normalized": norm}
+ if not norm:
+ return {**base, "kind": "missing", "exists": False}
+
+ # 1) Direct note id (e.g. "kb_abc...").
+ if norm.startswith("kb_"):
+ note = await self.repo.get_note(norm, with_sources=False)
+ if note and note.get("status") != "archived":
+ return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
+
+ # 2) Note by vault path (links from index.md / hot.md / structural hub).
+ note = await self.repo.find_by_vault_path(norm)
+ if note:
+ return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
+
+ # 3) Note by title (the real related-note cross-links use [[Title]]).
+ note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1])
+ if note:
+ return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
+
+ # 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…).
+ page = self.read_vault_page(norm)
+ if page is not None:
+ return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True}
+
+ return {**base, "kind": "missing", "exists": False}
+
+ async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]:
+ """Resolve many targets at once; returns ``{original_target: resolution}``."""
+ out: dict[str, dict[str, Any]] = {}
+ for target in dict.fromkeys(t for t in targets if t and t.strip()):
+ out[target] = await self.resolve_wikilink(target)
+ return out
+
+ async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None:
+ """Extract every ``[[wikilink]]`` in a note's body and resolve them.
+
+ Returns ``{target: resolution}`` or ``None`` when the note is missing.
+ Numeric ``[[123]]`` citation badges are skipped (handled by the renderer).
+ """
+ note = await self.repo.get_note(note_id, with_sources=False)
+ if note is None:
+ return None
+ targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()]
+ return await self.resolve_wikilinks(targets)
+
+ def read_vault_page(self, path: str) -> dict[str, Any] | None:
+ """Read a raw vault Markdown page (meta/hub pages without a DB note).
+
+ Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does
+ not exist or the path escapes the vault root.
+ """
+ norm = normalize_wikilink_target(path)
+ if not norm:
+ return None
+ rel = f"{norm}.md"
+ try:
+ parsed = self.adapter.vault.read_note(rel)
+ except VaultPathError:
+ return None
+ if parsed is None:
+ return None
+ fm, body = parsed
+ title = str(fm.get("title") or "").strip() or norm.split("/")[-1]
+ return {"path": norm, "title": title, "content_md": body}
+
+ # -- search & graph ---------------------------------------------------
+ async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
+ return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
+
+ async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
+ """Retrieve top knowledge for ``query`` and format an injectable block.
+
+ Returns ``{"text": , "items": [...]}``. Uses hybrid
+ search when embeddings are available, else keyword (wiki-query style).
+ """
+ mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
+ items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
+ if not items:
+ return {"text": "", "items": []}
+ lines = [
+ "以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
+ "",
+ ]
+ for i, it in enumerate(items, 1):
+ lines.append(f"[知识 {i}] {it.get('title') or ''}")
+ if it.get("snippet"):
+ lines.append(it["snippet"])
+ lines.append(f"来源:{it['note_id']}")
+ lines.append("")
+ return {"text": "\n".join(lines).strip(), "items": items}
+
+ # -- dedup & merge (phase 3) -----------------------------------------
+ async def find_duplicates(self) -> list[dict[str, Any]]:
+ from deerflow.knowledge.dedup import find_duplicate_groups
+
+ notes = await self.repo.list_all()
+ return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
+
+ async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
+ """Merge ``duplicate_ids`` into ``primary_id``.
+
+ Sources from each duplicate are re-attached to the primary, a wikilink
+ + "Merged from" note is appended to the primary body, the duplicates are
+ archived (soft delete), and a ``merged_into`` relation is recorded.
+ """
+ primary = await self.repo.get_note(primary_id)
+ if primary is None:
+ return None
+ merged_titles: list[str] = []
+ for dup_id in duplicate_ids:
+ if dup_id == primary_id:
+ continue
+ dup = await self.repo.get_note(dup_id)
+ if dup is None:
+ continue
+ # Move sources.
+ for src in dup.get("sources") or []:
+ await self.repo.add_sources(
+ primary_id,
+ [
+ SourceDraft(
+ source_type=src.get("source_type") or "tool",
+ thread_id=src.get("thread_id"),
+ message_id=src.get("message_id"),
+ tool_name=src.get("tool_name"),
+ title=src.get("title"),
+ url=src.get("url"),
+ snippet=src.get("snippet"),
+ )
+ ],
+ created_by=updated_by,
+ )
+ await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
+ await self.delete_note(dup_id, updated_by=updated_by)
+ merged_titles.append(dup.get("title") or dup_id)
+
+ if merged_titles:
+ appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
+ await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
+ return await self.repo.get_note(primary_id)
+
+ async def reindex_all(self) -> dict[str, int]:
+ """Rebuild embeddings for every non-archived note (admin/maintenance)."""
+ notes = await self.repo.list_all()
+ count = 0
+ for note in notes:
+ await self._reindex_embeddings(note["id"], note.get("content_md") or "")
+ count += 1
+ return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
+
+ async def export_graph(self) -> dict[str, Any]:
+ from deerflow.utils.time import now_iso
+
+ notes = await self.repo.list_all()
+ entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
+ relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
+ result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
+ return {"files": result["files"], "stats": result["stats"]}
+
+ async def get_graph(self) -> dict[str, Any]:
+ """Return product graph data ``{nodes, edges}`` (notes + entities)."""
+ from deerflow.knowledge import exporter
+
+ notes = await self.repo.list_all()
+ entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
+ relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
+ return exporter.build_graph(notes, entities, relations)
+
+ def export_file_path(self, filename: str) -> Path | None:
+ return self.adapter.export_file_path(filename)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/skill_templates.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/skill_templates.py
new file mode 100644
index 00000000..bad928f4
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/skill_templates.py
@@ -0,0 +1,51 @@
+"""Built-in obsidian-wiki skill templates.
+
+Mirrors the key intent of ``ar9av/obsidian-wiki`` skills so the backend can
+write vault content that stays compatible with the original tooling without
+shelling out to its CLI:
+
+- ``wiki-capture`` — how a conversation becomes a *declarative* knowledge note
+ (frontmatter schema + Context/Finding/Reasoning/Implications/Related body).
+- ``wiki-query`` — layered retrieval (hot.md → index.md → frontmatter → body).
+- ``wiki-export`` — graph artifacts (graph.json / graph.html / graphml / cypher).
+
+These are intentionally lightweight: the structures here are what
+:mod:`deerflow.knowledge.markdown_vault` and :mod:`deerflow.knowledge.extractor`
+produce, and what the first-release LLM prompt (future enhancement) should
+target.
+"""
+
+from __future__ import annotations
+
+# Category → vault sub-directory (wiki-capture page taxonomy, §8 of the dev doc).
+CATEGORY_DIRS = {
+ "synthesis": "synthesis",
+ "concepts": "concepts",
+ "concept": "concepts",
+ "references": "references",
+ "reference": "references",
+ "source": "references",
+ "decision": "synthesis",
+ "journal": "journal",
+ "session": "journal",
+}
+
+# Body section headings (wiki-capture). Declarative knowledge, never a chat log.
+BODY_SECTIONS = ("Context", "Finding / Decision", "Reasoning", "Implications", "Related")
+
+# Inline provenance markers reused from wiki-capture.
+INFERRED_MARKER = "^[inferred]"
+AMBIGUOUS_MARKER = "^[ambiguous]"
+
+WIKI_CAPTURE_GUIDANCE = """\
+Capture *declarative* knowledge, not a chat transcript. Classify the note as one
+of: synthesis, concept, source, decision, session. Prefer at least two
+[[wikilinks]]. Mark inferred content with ^[inferred] and uncertain/conflicting
+content with ^[ambiguous]. Project-specific knowledge belongs under
+projects//...
+"""
+
+
+def category_to_dir(category: str | None) -> str:
+ """Map a wiki-capture category to its vault sub-directory."""
+ return CATEGORY_DIRS.get((category or "").strip().lower(), "synthesis")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/sources.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/sources.py
new file mode 100644
index 00000000..824bc2c1
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/sources.py
@@ -0,0 +1,82 @@
+"""Normalize knowledge sources from threads / searches / tool results.
+
+Source items come from a few shapes: backend reference-batches (built by
+``deerflow.runtime.references``), raw search-result payloads, and tool-call
+messages. They are all normalized into :class:`SourceDraft` so the repository
+and frontend see one consistent structure.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from deerflow.knowledge.schemas import SourceDraft
+
+
+def _clip(value: Any, limit: int = 2000) -> str | None:
+ if value is None:
+ return None
+ text = str(value).strip()
+ if not text:
+ return None
+ return text[:limit]
+
+
+def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
+ """Convert backend reference-batches into :class:`SourceDraft`."""
+ out: list[SourceDraft] = []
+ seen: set[str] = set()
+ for batch in batches or []:
+ msg_id = batch.get("assistant_message_id") or batch.get("id")
+ for src in batch.get("sources", []) or []:
+ if not isinstance(src, dict):
+ continue
+ url = _clip(src.get("url"), 1024)
+ title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
+ dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
+ if dedup_key in seen:
+ continue
+ seen.add(dedup_key)
+ out.append(
+ SourceDraft(
+ source_type="search" if url else "tool",
+ thread_id=thread_id,
+ message_id=_clip(msg_id, 128),
+ tool_name=_clip(src.get("skillName"), 128),
+ title=title,
+ url=url,
+ snippet=_clip(src.get("snippet")),
+ raw=None,
+ )
+ )
+ if len(out) >= max_items:
+ return out
+ return out
+
+
+def from_search_results(
+ query: str,
+ results: list[dict[str, Any]] | None,
+ *,
+ thread_id: str | None = None,
+ message_id: str | None = None,
+ max_items: int = 20,
+) -> list[SourceDraft]:
+ """Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
+ out: list[SourceDraft] = []
+ for item in (results or [])[:max_items]:
+ if not isinstance(item, dict):
+ continue
+ out.append(
+ SourceDraft(
+ source_type="search",
+ thread_id=thread_id,
+ message_id=message_id,
+ tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
+ title=_clip(item.get("title"), 512) or query[:200],
+ url=_clip(item.get("url"), 1024),
+ snippet=_clip(item.get("snippet") or item.get("content")),
+ raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
+ )
+ )
+ return out
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/admin_stats/finalize.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/admin_stats/finalize.py
new file mode 100644
index 00000000..39212bae
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/admin_stats/finalize.py
@@ -0,0 +1,47 @@
+"""Pure scheduling helpers for the admin leaderboard daily snapshots.
+
+Kept import-light on purpose (only stdlib ``datetime``) so the decision logic
+is unit-testable without standing up the FastAPI app, the DB engine, or the
+ORM. The background scheduler in ``app/gateway/app.py`` wires this to the real
+snapshot rows.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+
+def should_finalize_previous_day(
+ *,
+ status: str | None,
+ generated_at: datetime | None,
+ now: datetime,
+ finalize_hour: int,
+) -> bool:
+ """Decide whether *yesterday's* snapshot should be force-recomputed once.
+
+ The previous day's snapshot is normally built before midnight, so it can
+ miss runs / tool calls whose rows only land in the database shortly before
+ or after the day boundary. Once per day, on/after ``finalize_hour`` (local
+ Beijing time), we recompute it a single time so the full day is captured.
+
+ ``now`` and ``generated_at`` are expected to be timezone-aware Beijing
+ datetimes (``generated_at`` may be ``None`` for legacy rows).
+
+ Returns ``True`` only when all hold:
+ - it is already on/after ``finalize_hour`` today;
+ - a snapshot row exists (``status`` is not ``None`` — a missing day is
+ left to the normal prewarm/queue path);
+ - that snapshot is not already ``queued``/``running`` (a finalize already
+ in flight, or normal processing);
+ - it was generated before today (i.e. not yet finalized for this day).
+ """
+ if status is None:
+ return False
+ if now.hour < finalize_hour:
+ return False
+ if status in ("queued", "running"):
+ return False
+ if generated_at is not None and generated_at.date() >= now.date():
+ return False
+ return True
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/base.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/base.py
index 0ff4ae33..d2fc32cd 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/base.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/base.py
@@ -119,18 +119,19 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
- async def list_paginated(
- self,
- *,
- user_id: str,
- scope: str | None,
- search: str | None,
- tag_ids: list[str] | None,
- page: int,
- page_size: int,
- admin: bool = False,
- sort: str = "time_desc",
- ) -> tuple[list[dict[str, Any]], int]:
+ async def list_paginated(
+ self,
+ *,
+ user_id: str,
+ scope: str | None,
+ search: str | None,
+ tag_ids: list[str] | None,
+ page: int,
+ page_size: int,
+ admin: bool = False,
+ sort: str = "time_desc",
+ owner: str | None = None,
+ ) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
``scope``:
@@ -142,13 +143,15 @@ class AgentStore(ABC):
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
``search`` matches the raw name OR its desensitized display form.
- ``tag_ids`` keeps agents carrying ANY of the tags (OR). Both filters,
- the COUNT, and LIMIT/OFFSET all execute in SQL.
+ ``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
+ only agents published by that user (``user_id`` equality) — built-in
+ agents (no owner) never match. All filters, the COUNT, and LIMIT/OFFSET
+ all execute in SQL.
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
- for the 1-based ``page``.
- """
- raise NotImplementedError
+ for the 1-based ``page``.
+ """
+ raise NotImplementedError
# --- Listing cache (skills / tool_groups / model) ---
@@ -199,28 +202,28 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
- async def delete_favorites_for_agent(self, agent_id: str) -> None:
- """Drop every user's favorite of ``agent_id`` (called on agent delete)."""
- raise NotImplementedError
-
- # --- Per-user pins ("置顶") ---
-
- @abstractmethod
- async def add_pin(self, user_id: str, agent_id: str) -> None:
- """Pin ``agent_id`` for ``user_id``. Idempotent."""
- raise NotImplementedError
-
- @abstractmethod
- async def remove_pin(self, user_id: str, agent_id: str) -> bool:
- """Remove a per-user pin. Returns ``False`` when none existed."""
- raise NotImplementedError
-
- @abstractmethod
- async def list_pin_ids(self, user_id: str) -> set[str]:
- """Return the set of agent ids pinned by ``user_id``."""
- raise NotImplementedError
-
- @abstractmethod
- async def delete_pins_for_agent(self, agent_id: str) -> None:
- """Drop every user's pin of ``agent_id`` (called on agent delete)."""
- raise NotImplementedError
+ async def delete_favorites_for_agent(self, agent_id: str) -> None:
+ """Drop every user's favorite of ``agent_id`` (called on agent delete)."""
+ raise NotImplementedError
+
+ # --- Per-user pins ("置顶") ---
+
+ @abstractmethod
+ async def add_pin(self, user_id: str, agent_id: str) -> None:
+ """Pin ``agent_id`` for ``user_id``. Idempotent."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def remove_pin(self, user_id: str, agent_id: str) -> bool:
+ """Remove a per-user pin. Returns ``False`` when none existed."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def list_pin_ids(self, user_id: str) -> set[str]:
+ """Return the set of agent ids pinned by ``user_id``."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def delete_pins_for_agent(self, agent_id: str) -> None:
+ """Drop every user's pin of ``agent_id`` (called on agent delete)."""
+ raise NotImplementedError
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/memory.py
index 762e3aa2..484dd7f9 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/memory.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/memory.py
@@ -21,9 +21,9 @@ def _copy(data: dict[str, Any]) -> dict[str, Any]:
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
- self._agents: dict[str, dict[str, Any]] = {}
- self._favorites: set[tuple[str, str]] = set()
- self._pins: set[tuple[str, str]] = set()
+ self._agents: dict[str, dict[str, Any]] = {}
+ self._favorites: set[tuple[str, str]] = set()
+ self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@@ -186,51 +186,54 @@ class MemoryAgentStore(AgentStore):
return bool(agent.get("is_private"))
return self._is_visible(agent, user_id)
- async def list_paginated(
- self,
- *,
- user_id: str,
- scope: str | None,
+ async def list_paginated(
+ self,
+ *,
+ user_id: str,
+ scope: str | None,
search: str | None,
- tag_ids: list[str] | None,
- page: int,
- page_size: int,
- admin: bool = False,
- sort: str = "time_desc",
- ) -> tuple[list[dict[str, Any]], int]:
+ tag_ids: list[str] | None,
+ page: int,
+ page_size: int,
+ admin: bool = False,
+ sort: str = "time_desc",
+ owner: str | None = None,
+ ) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
+ owner_key = owner.strip() if owner and owner.strip() else None
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
+ and (owner_key is None or a.get("user_id") == owner_key)
]
- if sort == "time_asc":
- matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
- elif sort == "updated_desc":
- matched.sort(
- key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
- reverse=True,
- )
- elif sort == "updated_asc":
- matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
- elif sort == "name_asc":
- matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
- elif sort == "name_desc":
- matched.sort(
- key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
- reverse=True,
- )
- else:
- matched.sort(
- key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
- reverse=True,
- )
- pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
- matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
- total = len(matched)
- start = max(page - 1, 0) * page_size
+ if sort == "time_asc":
+ matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
+ elif sort == "updated_desc":
+ matched.sort(
+ key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
+ reverse=True,
+ )
+ elif sort == "updated_asc":
+ matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
+ elif sort == "name_asc":
+ matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
+ elif sort == "name_desc":
+ matched.sort(
+ key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
+ reverse=True,
+ )
+ else:
+ matched.sort(
+ key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
+ reverse=True,
+ )
+ pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
+ matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
+ total = len(matched)
+ start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
@@ -275,23 +278,23 @@ class MemoryAgentStore(AgentStore):
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
- async def delete_favorites_for_agent(self, agent_id: str) -> None:
- self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
-
- # --- Pins ("置顶") ---
-
- async def add_pin(self, user_id: str, agent_id: str) -> None:
- self._pins.add((user_id, agent_id))
-
- async def remove_pin(self, user_id: str, agent_id: str) -> bool:
- key = (user_id, agent_id)
- if key not in self._pins:
- return False
- self._pins.remove(key)
- return True
-
- async def list_pin_ids(self, user_id: str) -> set[str]:
- return {agent_id for uid, agent_id in self._pins if uid == user_id}
-
- async def delete_pins_for_agent(self, agent_id: str) -> None:
- self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
+ async def delete_favorites_for_agent(self, agent_id: str) -> None:
+ self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
+
+ # --- Pins ("置顶") ---
+
+ async def add_pin(self, user_id: str, agent_id: str) -> None:
+ self._pins.add((user_id, agent_id))
+
+ async def remove_pin(self, user_id: str, agent_id: str) -> bool:
+ key = (user_id, agent_id)
+ if key not in self._pins:
+ return False
+ self._pins.remove(key)
+ return True
+
+ async def list_pin_ids(self, user_id: str) -> set[str]:
+ return {agent_id for uid, agent_id in self._pins if uid == user_id}
+
+ async def delete_pins_for_agent(self, agent_id: str) -> None:
+ self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/sql.py
index 965c93ae..aa0f5096 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/sql.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/agents/sql.py
@@ -6,19 +6,19 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
-from sqlalchemy import and_, case, delete, func, or_, select
+from sqlalchemy import and_, case, delete, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.agent_name_desensitize import source_terms_for_needle
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import (
- AgentExtrasRow,
- AgentFavoriteRow,
- AgentPinRow,
- AgentRow,
- AgentSkillRow,
-)
+ AgentExtrasRow,
+ AgentFavoriteRow,
+ AgentPinRow,
+ AgentRow,
+ AgentSkillRow,
+)
from deerflow.persistence.tags.model import TagAssignmentRow
@@ -278,18 +278,19 @@ class AgentRepository(AgentStore):
)
return AgentRow.id.in_(subq)
- async def list_paginated(
- self,
- *,
- user_id: str,
- scope: str | None,
+ async def list_paginated(
+ self,
+ *,
+ user_id: str,
+ scope: str | None,
search: str | None,
- tag_ids: list[str] | None,
- page: int,
- page_size: int,
- admin: bool = False,
- sort: str = "time_desc",
- ) -> tuple[list[dict[str, Any]], int]:
+ tag_ids: list[str] | None,
+ page: int,
+ page_size: int,
+ admin: bool = False,
+ sort: str = "time_desc",
+ owner: str | None = None,
+ ) -> tuple[list[dict[str, Any]], int]:
filters = []
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
if scope_filter is not None:
@@ -298,30 +299,33 @@ class AgentRepository(AgentStore):
filters.append(self._search_clause(search))
if tag_ids:
filters.append(self._tag_clause(tag_ids))
+ if owner and owner.strip():
+ # Filter to one publisher. Built-in agents (user_id IS NULL) never match.
+ filters.append(AgentRow.user_id == owner.strip())
count_stmt = select(func.count()).select_from(AgentRow)
- pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
- pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
-
- if sort == "time_asc":
- order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
- elif sort == "updated_desc":
- order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
- elif sort == "updated_asc":
- order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
- elif sort == "name_asc":
- order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
- elif sort == "name_desc":
- order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
- else:
- order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
-
- page_stmt = (
- select(AgentRow)
- .order_by(pin_rank.asc(), *order_by)
- .limit(page_size)
- .offset(max(page - 1, 0) * page_size)
- )
+ pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
+ pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
+
+ if sort == "time_asc":
+ order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
+ elif sort == "updated_desc":
+ order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
+ elif sort == "updated_asc":
+ order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
+ elif sort == "name_asc":
+ order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
+ elif sort == "name_desc":
+ order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
+ else:
+ order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
+
+ page_stmt = (
+ select(AgentRow)
+ .order_by(pin_rank.asc(), *order_by)
+ .limit(page_size)
+ .offset(max(page - 1, 0) * page_size)
+ )
if filters:
where = and_(*filters)
count_stmt = count_stmt.where(where)
@@ -441,49 +445,49 @@ class AgentRepository(AgentStore):
)
return {row[0] for row in result.all()}
- async def delete_favorites_for_agent(self, agent_id: str) -> None:
- async with self._sf() as session:
- await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
- await session.commit()
-
- # --- Pins ("置顶") ---
-
- async def add_pin(self, user_id: str, agent_id: str) -> None:
- async with self._sf() as session:
- existing = (
- await session.execute(
- select(AgentPinRow.id)
- .where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
- .limit(1)
- )
- ).scalar_one_or_none()
- if existing is not None:
- return
- session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
- try:
- await session.commit()
- except IntegrityError:
- await session.rollback()
-
- async def remove_pin(self, user_id: str, agent_id: str) -> bool:
- async with self._sf() as session:
- result = await session.execute(
- delete(AgentPinRow).where(
- AgentPinRow.user_id == user_id,
- AgentPinRow.agent_id == agent_id,
- )
- )
- await session.commit()
- return (result.rowcount or 0) > 0
-
- async def list_pin_ids(self, user_id: str) -> set[str]:
- async with self._sf() as session:
- result = await session.execute(
- select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
- )
- return {row[0] for row in result.all()}
-
- async def delete_pins_for_agent(self, agent_id: str) -> None:
- async with self._sf() as session:
- await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
- await session.commit()
+ async def delete_favorites_for_agent(self, agent_id: str) -> None:
+ async with self._sf() as session:
+ await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
+ await session.commit()
+
+ # --- Pins ("置顶") ---
+
+ async def add_pin(self, user_id: str, agent_id: str) -> None:
+ async with self._sf() as session:
+ existing = (
+ await session.execute(
+ select(AgentPinRow.id)
+ .where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
+ .limit(1)
+ )
+ ).scalar_one_or_none()
+ if existing is not None:
+ return
+ session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
+ try:
+ await session.commit()
+ except IntegrityError:
+ await session.rollback()
+
+ async def remove_pin(self, user_id: str, agent_id: str) -> bool:
+ async with self._sf() as session:
+ result = await session.execute(
+ delete(AgentPinRow).where(
+ AgentPinRow.user_id == user_id,
+ AgentPinRow.agent_id == agent_id,
+ )
+ )
+ await session.commit()
+ return (result.rowcount or 0) > 0
+
+ async def list_pin_ids(self, user_id: str) -> set[str]:
+ async with self._sf() as session:
+ result = await session.execute(
+ select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
+ )
+ return {row[0] for row in result.all()}
+
+ async def delete_pins_for_agent(self, agent_id: str) -> None:
+ async with self._sf() as session:
+ await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
+ await session.commit()
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260609_01_skill_name_zh.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260609_01_skill_name_zh.py
new file mode 100644
index 00000000..b660c0ac
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260609_01_skill_name_zh.py
@@ -0,0 +1,42 @@
+"""Add name_zh column to skills table.
+
+Revision ID: 20260609_01
+Revises: 20260608_01
+Create Date: 2026-06-09
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import inspect
+
+
+revision = "20260609_01"
+down_revision = "20260608_01"
+branch_labels = None
+depends_on = None
+
+
+def _has_table(table_name: str) -> bool:
+ inspector = inspect(op.get_bind())
+ return table_name in inspector.get_table_names()
+
+
+def _has_column(table_name: str, column_name: str) -> bool:
+ if not _has_table(table_name):
+ return False
+ inspector = inspect(op.get_bind())
+ return any(col["name"] == column_name for col in inspector.get_columns(table_name))
+
+
+def upgrade() -> None:
+ # ``skills`` may not exist yet on a brand-new DB — create_all handles that
+ # case with the column already present, so only patch existing tables.
+ if _has_table("skills") and not _has_column("skills", "name_zh"):
+ op.add_column("skills", sa.Column("name_zh", sa.String(length=128), nullable=True))
+
+
+def downgrade() -> None:
+ if _has_column("skills", "name_zh"):
+ op.drop_column("skills", "name_zh")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260609_02_merge_skill_name_zh_and_chain_heads.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260609_02_merge_skill_name_zh_and_chain_heads.py
new file mode 100644
index 00000000..f8095d73
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260609_02_merge_skill_name_zh_and_chain_heads.py
@@ -0,0 +1,30 @@
+"""Merge the skill_name_zh and chain-stages migration heads.
+
+Two lineages both descend from ``20260608_01`` (admin_leaderboard_daily_stats):
+``20260608_02`` … ``20260608_06`` (the chain-stages / parallel-agent branch) and
+``20260609_01`` (skills.name_zh), which branched directly off ``20260608_01``
+instead of off the latest head. That left the graph with two heads and made
+``alembic upgrade head`` fail with "Multiple head revisions are present". This
+no-op merge revision joins both lineages so ``upgrade head`` resolves to a single
+head again.
+
+Revision ID: 20260609_02
+Revises: 20260608_06, 20260609_01
+Create Date: 2026-06-10
+"""
+
+from __future__ import annotations
+
+
+revision = "20260609_02"
+down_revision = ("20260608_06", "20260609_01")
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ pass
+
+
+def downgrade() -> None:
+ pass
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
index 26e5cfef..18566d1f 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
@@ -25,6 +25,16 @@ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
+from deerflow.knowledge.models import (
+ KnowledgeEmbeddingRow,
+ KnowledgeEntityRow,
+ KnowledgeNoteRow,
+ KnowledgeNoteTagRow,
+ KnowledgeNoteVersionRow,
+ KnowledgeRelationRow,
+ KnowledgeSourceRow,
+ KnowledgeTagRow,
+)
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
@@ -63,6 +73,14 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
+ "KnowledgeEmbeddingRow",
+ "KnowledgeEntityRow",
+ "KnowledgeNoteRow",
+ "KnowledgeNoteTagRow",
+ "KnowledgeNoteVersionRow",
+ "KnowledgeRelationRow",
+ "KnowledgeSourceRow",
+ "KnowledgeTagRow",
"LlmCallMetricRow",
"NotificationRow",
"ParallelAgentTaskRow",
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/memory.py
index d774f560..8b341b18 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/memory.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/memory.py
@@ -70,6 +70,7 @@ class MemorySkillStore(SkillStore):
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", False)),
"detail": data.get("detail"),
+ "name_zh": data.get("name_zh"),
"created_at": _now(),
"updated_at": _now(),
}
@@ -85,6 +86,7 @@ class MemorySkillStore(SkillStore):
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", True)),
"detail": data.get("detail"),
+ "name_zh": data.get("name_zh"),
"created_at": _now(),
"updated_at": _now(),
}
@@ -102,6 +104,8 @@ class MemorySkillStore(SkillStore):
record["published"] = bool(data["published"])
if "detail" in data:
record["detail"] = data["detail"]
+ if "name_zh" in data:
+ record["name_zh"] = data["name_zh"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -115,6 +119,8 @@ class MemorySkillStore(SkillStore):
record["owner_user_id"] = data["owner_user_id"]
if "detail" in data:
record["detail"] = data["detail"]
+ if "name_zh" in data:
+ record["name_zh"] = data["name_zh"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/model.py
index ff981c15..b71a794f 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/model.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/model.py
@@ -29,6 +29,10 @@ class SkillRow(Base):
# kept in the DB so admins can annotate built-in skills without rewriting
# the shared on-disk SKILL.md file.
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
+ # User-configured or LLM-generated Chinese display name. Kept in the DB (not
+ # the on-disk SKILL.md) so it can be set for built-in skills too, and so the
+ # shared source file is never rewritten.
+ name_zh: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/sql.py
index e17dd015..3cefe180 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/sql.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/skills/sql.py
@@ -64,6 +64,7 @@ class SkillRepository(SkillStore):
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", False)),
detail=data.get("detail"),
+ name_zh=data.get("name_zh"),
created_at=now,
updated_at=now,
)
@@ -83,6 +84,7 @@ class SkillRepository(SkillStore):
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", True)),
detail=data.get("detail"),
+ name_zh=data.get("name_zh"),
created_at=now,
updated_at=now,
)
@@ -105,6 +107,8 @@ class SkillRepository(SkillStore):
row.published = bool(data["published"])
if "detail" in data:
row.detail = data["detail"]
+ if "name_zh" in data:
+ row.name_zh = data["name_zh"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -121,6 +125,8 @@ class SkillRepository(SkillStore):
row.owner_user_id = data["owner_user_id"]
if "detail" in data:
row.detail = data["detail"]
+ if "name_zh" in data:
+ row.name_zh = data["name_zh"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/runtime/checkpointer/async_provider.py b/offline-backend-20260512/backend/packages/harness/deerflow/runtime/checkpointer/async_provider.py
index ef5d412e..d7fa14e5 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/runtime/checkpointer/async_provider.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/runtime/checkpointer/async_provider.py
@@ -1,295 +1,302 @@
-"""Async checkpointer factory.
-
-Provides an **async context manager** for long-running async servers that need
-proper resource cleanup.
-
-Supported backends: memory, sqlite, postgres.
-
-Usage (e.g. FastAPI lifespan)::
-
- from deerflow.runtime.checkpointer.async_provider import make_checkpointer
-
- async with make_checkpointer() as checkpointer:
- app.state.checkpointer = checkpointer # InMemorySaver if not configured
-
-For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import logging
-from collections.abc import AsyncIterator
-import os
-from pathlib import Path
-
-from langgraph.types import Checkpointer
-
-from deerflow.config.app_config import AppConfig, get_app_config
-from deerflow.runtime.checkpointer.provider import (
- POSTGRES_CONN_REQUIRED,
- POSTGRES_INSTALL,
- SQLITE_INSTALL,
-)
-from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
-
-logger = logging.getLogger(__name__)
-
-
-def _csv_env(name: str) -> list[str]:
- raw = os.getenv(name, "")
- return [part.strip() for part in raw.split(",") if part.strip()]
-
-
-def _dual_read_enabled(config) -> bool:
- if getattr(config, "dual_read_enabled", False):
- return True
- return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _use_postgres_enabled(config) -> bool:
- raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
- if raw:
- return raw.strip().lower() not in {"0", "false", "no", "off"}
- return bool(getattr(config, "use_postgres", True))
-
-
-def _dual_read_paths(config) -> list[str]:
- paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
- paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
- resolved: list[str] = []
- seen: set[str] = set()
- for path in paths:
- conn_str = resolve_sqlite_conn_str(path)
- if conn_str == ":memory:" or conn_str.startswith("file:"):
- logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
- continue
- if conn_str not in seen:
- resolved.append(conn_str)
- seen.add(conn_str)
- return resolved
-
-
-def _sqlite_conn_str_for_postgres_fallback(config) -> str:
- configured = getattr(config, "sqlite_connection_string", None)
- if configured:
- return configured
- paths = _dual_read_paths(config)
- return paths[0] if paths else "store.db"
-
-
-@contextlib.asynccontextmanager
-async def _pg_pool_saver(conn_str: str) -> AsyncIterator[Checkpointer]:
- """Pool-backed ``AsyncPostgresSaver``, resilient to idle-closed connections.
-
- ``AsyncPostgresSaver.from_conn_string`` opens **one** long-lived connection.
- 对远程 Postgres(本项目 47.88.25.99 这类),空闲几分钟后连接会被服务端 / NAT /
- 连接池中间件断开,而 psycopg **不会自动重连** —— 下一次 checkpoint 操作就抛
- ``OperationalError: the connection is closed``,整个 run 直接失败(实测:圆桌第二步
- 完成后闲置 ~6 分钟,用户再人工干预 → leader run 读 checkpoint 时撞死连接 → 前端
- "(无文本输出)")。
-
- 用带健康检查的连接池根治:``check=AsyncConnectionPool.check_connection`` 在把连接交给
- 调用方前先验活,坏连接直接丢弃重建;``max_idle`` 主动回收久闲连接,赶在远程 idle 超时
- 之前。这样上层永远拿不到失效连接。``prepare_threshold=0`` 关预备语句,兼容 pgbouncer
- 一类连接池(8432 非标准端口很可能就是)。
- """
- try:
- from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
- except ImportError as exc:
- raise ImportError(POSTGRES_INSTALL) from exc
- from psycopg_pool import AsyncConnectionPool
-
- try:
- max_size = int(os.getenv("DEER_FLOW_PG_POOL_MAX", "20"))
- except ValueError:
- max_size = 20
-
- pool = AsyncConnectionPool(
- conninfo=conn_str,
- min_size=1,
- max_size=max_size,
- open=False,
- kwargs={"autocommit": True, "prepare_threshold": 0},
- check=AsyncConnectionPool.check_connection,
- max_idle=120.0,
- max_lifetime=1800.0,
- )
- await pool.open(wait=True)
- try:
- saver = AsyncPostgresSaver(pool)
- await saver.setup()
- logger.info("Checkpointer: Postgres connection pool ready (max_size=%d, health-checked)", max_size)
- yield saver
- finally:
- await pool.close()
-
-
-@contextlib.asynccontextmanager
-async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
- try:
- from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
- except ImportError as exc:
- raise ImportError(SQLITE_INSTALL) from exc
-
- conn_str = resolve_sqlite_conn_str(raw)
- await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
- async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
- await saver.setup()
- yield saver
-
-
-# ---------------------------------------------------------------------------
-# Async factory
-# ---------------------------------------------------------------------------
-
-
-@contextlib.asynccontextmanager
-async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
- """Async context manager that constructs and tears down a checkpointer."""
- if config.type == "memory":
- from langgraph.checkpoint.memory import InMemorySaver
-
- yield InMemorySaver()
- return
-
- if config.type == "sqlite":
- async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
- yield saver
- return
-
- if config.type == "postgres":
- if not _use_postgres_enabled(config):
- conn_str = _sqlite_conn_str_for_postgres_fallback(config)
- logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
- async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
- yield saver
- return
-
- if not config.connection_string:
- raise ValueError(POSTGRES_CONN_REQUIRED)
-
- # 连接池 + 健康检查,根治远程 Postgres 单连接空闲被关导致的 run 崩溃(见 _pg_pool_saver)。
- async with _pg_pool_saver(config.connection_string) as saver:
- if not _dual_read_enabled(config):
- yield saver
- return
-
- from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
-
- from deerflow.runtime.checkpointer.checkpoint_migration import (
- DualReadCheckpointer,
- SQLiteCheckpointSource,
- )
-
- sources: list[SQLiteCheckpointSource] = []
- for sqlite_path in _dual_read_paths(config):
- if not Path(sqlite_path).exists():
- logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
- continue
- ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
- source_saver = await ctx.__aenter__()
- await source_saver.setup()
- sources.append(
- SQLiteCheckpointSource(
- path=sqlite_path,
- saver=source_saver,
- context=ctx,
- progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
- )
- )
-
- if not sources:
- logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
- yield saver
- return
-
- proxy = DualReadCheckpointer(
- saver,
- sources,
- )
- try:
- yield proxy
- finally:
- await proxy.aclose_sources()
- return
-
- raise ValueError(f"Unknown checkpointer type: {config.type!r}")
-
-
-# ---------------------------------------------------------------------------
-# Public async context manager
-# ---------------------------------------------------------------------------
-
-
-@contextlib.asynccontextmanager
-async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
- """Async context manager that constructs a checkpointer from unified DatabaseConfig."""
- if db_config.backend == "memory":
- from langgraph.checkpoint.memory import InMemorySaver
-
- yield InMemorySaver()
- return
-
- if db_config.backend == "sqlite":
- try:
- from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
- except ImportError as exc:
- raise ImportError(SQLITE_INSTALL) from exc
-
- conn_str = db_config.checkpointer_sqlite_path
- ensure_sqlite_parent_dir(conn_str)
- async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
- await saver.setup()
- yield saver
- return
-
- if db_config.backend == "postgres":
- if not db_config.postgres_url:
- raise ValueError("database.postgres_url is required for the postgres backend")
-
- # 连接池 + 健康检查(见 _pg_pool_saver),避免单连接空闲被关。
- async with _pg_pool_saver(db_config.postgres_url) as saver:
- yield saver
- return
-
- raise ValueError(f"Unknown database backend: {db_config.backend!r}")
-
-
-@contextlib.asynccontextmanager
-async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
- """Async context manager that yields a checkpointer for the caller's lifetime.
- Resources are opened on enter and closed on exit -- no global state::
-
- async with make_checkpointer(app_config) as checkpointer:
- app.state.checkpointer = checkpointer
-
- Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
-
- Priority:
- 1. Legacy ``checkpointer:`` config section (backward compatible)
- 2. Unified ``database:`` config section
- 3. Default InMemorySaver
- """
-
- if app_config is None:
- app_config = get_app_config()
-
- # Legacy: standalone checkpointer config takes precedence
- if app_config.checkpointer is not None:
- async with _async_checkpointer(app_config.checkpointer) as saver:
- yield saver
- return
-
- # Unified database config
- db_config = getattr(app_config, "database", None)
- if db_config is not None and db_config.backend != "memory":
- async with _async_checkpointer_from_database(db_config) as saver:
- yield saver
- return
-
- # Default: in-memory
- from langgraph.checkpoint.memory import InMemorySaver
-
- yield InMemorySaver()
+"""Async checkpointer factory.
+
+Provides an **async context manager** for long-running async servers that need
+proper resource cleanup.
+
+Supported backends: memory, sqlite, postgres.
+
+Usage (e.g. FastAPI lifespan)::
+
+ from deerflow.runtime.checkpointer.async_provider import make_checkpointer
+
+ async with make_checkpointer() as checkpointer:
+ app.state.checkpointer = checkpointer # InMemorySaver if not configured
+
+For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from collections.abc import AsyncIterator
+import os
+from pathlib import Path
+
+from langgraph.types import Checkpointer
+
+from deerflow.config.app_config import AppConfig, get_app_config
+from deerflow.runtime.checkpointer.provider import (
+ POSTGRES_CONN_REQUIRED,
+ POSTGRES_INSTALL,
+ SQLITE_INSTALL,
+)
+from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
+
+logger = logging.getLogger(__name__)
+
+
+def _csv_env(name: str) -> list[str]:
+ raw = os.getenv(name, "")
+ return [part.strip() for part in raw.split(",") if part.strip()]
+
+
+def _dual_read_enabled(config) -> bool:
+ if getattr(config, "dual_read_enabled", False):
+ return True
+ return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _use_postgres_enabled(config) -> bool:
+ raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
+ if raw:
+ return raw.strip().lower() not in {"0", "false", "no", "off"}
+ return bool(getattr(config, "use_postgres", True))
+
+
+def _dual_read_paths(config) -> list[str]:
+ paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
+ paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
+ resolved: list[str] = []
+ seen: set[str] = set()
+ for path in paths:
+ conn_str = resolve_sqlite_conn_str(path)
+ if conn_str == ":memory:" or conn_str.startswith("file:"):
+ logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
+ continue
+ if conn_str not in seen:
+ resolved.append(conn_str)
+ seen.add(conn_str)
+ return resolved
+
+
+def _sqlite_conn_str_for_postgres_fallback(config) -> str:
+ configured = getattr(config, "sqlite_connection_string", None)
+ if configured:
+ return configured
+ paths = _dual_read_paths(config)
+ return paths[0] if paths else "store.db"
+
+
+def _pool_size_env(name: str, default: int) -> int:
+ raw = os.getenv(name, "").strip()
+ if not raw:
+ return default
+ try:
+ return max(0, int(raw))
+ except ValueError:
+ logger.warning("Invalid %s=%r; using default %d", name, raw, default)
+ return default
+
+
+@contextlib.asynccontextmanager
+async def _postgres_saver_from_pool(conn_string: str) -> AsyncIterator[Checkpointer]:
+ """Yield an ``AsyncPostgresSaver`` backed by a self-healing connection pool.
+
+ ``AsyncPostgresSaver.from_conn_string`` opens a *single* long-lived
+ connection. If that one connection dies (Postgres restart, idle timeout,
+ network blip, or a prior broken pipeline), every later checkpointer call
+ fails permanently with "the connection is closed" until the process is
+ restarted. A pool with ``check=check_connection`` validates each connection
+ on checkout and transparently replaces dead ones, so the runtime recovers
+ on its own.
+ """
+ from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
+ from psycopg.rows import dict_row
+ from psycopg_pool import AsyncConnectionPool
+
+ pool = AsyncConnectionPool(
+ conninfo=conn_string,
+ min_size=_pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
+ max_size=_pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
+ max_idle=300.0,
+ max_lifetime=3600.0,
+ check=AsyncConnectionPool.check_connection,
+ kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row},
+ open=False,
+ )
+ await pool.open(wait=True)
+ try:
+ saver = AsyncPostgresSaver(pool)
+ await saver.setup()
+ yield saver
+ finally:
+ await pool.close()
+
+
+@contextlib.asynccontextmanager
+async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
+ try:
+ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+ except ImportError as exc:
+ raise ImportError(SQLITE_INSTALL) from exc
+
+ conn_str = resolve_sqlite_conn_str(raw)
+ await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
+ async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
+ await saver.setup()
+ yield saver
+
+
+# ---------------------------------------------------------------------------
+# Async factory
+# ---------------------------------------------------------------------------
+
+
+@contextlib.asynccontextmanager
+async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
+ """Async context manager that constructs and tears down a checkpointer."""
+ if config.type == "memory":
+ from langgraph.checkpoint.memory import InMemorySaver
+
+ yield InMemorySaver()
+ return
+
+ if config.type == "sqlite":
+ async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
+ yield saver
+ return
+
+ if config.type == "postgres":
+ if not _use_postgres_enabled(config):
+ conn_str = _sqlite_conn_str_for_postgres_fallback(config)
+ logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
+ async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
+ yield saver
+ return
+
+ try:
+ from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
+ except ImportError as exc:
+ raise ImportError(POSTGRES_INSTALL) from exc
+
+ if not config.connection_string:
+ raise ValueError(POSTGRES_CONN_REQUIRED)
+
+ async with _postgres_saver_from_pool(config.connection_string) as saver:
+ if not _dual_read_enabled(config):
+ yield saver
+ return
+
+ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+
+ from deerflow.runtime.checkpointer.checkpoint_migration import (
+ DualReadCheckpointer,
+ SQLiteCheckpointSource,
+ )
+
+ sources: list[SQLiteCheckpointSource] = []
+ for sqlite_path in _dual_read_paths(config):
+ if not Path(sqlite_path).exists():
+ logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
+ continue
+ ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
+ source_saver = await ctx.__aenter__()
+ await source_saver.setup()
+ sources.append(
+ SQLiteCheckpointSource(
+ path=sqlite_path,
+ saver=source_saver,
+ context=ctx,
+ progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
+ )
+ )
+
+ if not sources:
+ logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
+ yield saver
+ return
+
+ proxy = DualReadCheckpointer(
+ saver,
+ sources,
+ )
+ try:
+ yield proxy
+ finally:
+ await proxy.aclose_sources()
+ return
+
+ raise ValueError(f"Unknown checkpointer type: {config.type!r}")
+
+
+# ---------------------------------------------------------------------------
+# Public async context manager
+# ---------------------------------------------------------------------------
+
+
+@contextlib.asynccontextmanager
+async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
+ """Async context manager that constructs a checkpointer from unified DatabaseConfig."""
+ if db_config.backend == "memory":
+ from langgraph.checkpoint.memory import InMemorySaver
+
+ yield InMemorySaver()
+ return
+
+ if db_config.backend == "sqlite":
+ try:
+ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+ except ImportError as exc:
+ raise ImportError(SQLITE_INSTALL) from exc
+
+ conn_str = db_config.checkpointer_sqlite_path
+ ensure_sqlite_parent_dir(conn_str)
+ async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
+ await saver.setup()
+ yield saver
+ return
+
+ if db_config.backend == "postgres":
+ try:
+ from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
+ except ImportError as exc:
+ raise ImportError(POSTGRES_INSTALL) from exc
+
+ if not db_config.postgres_url:
+ raise ValueError("database.postgres_url is required for the postgres backend")
+
+ async with _postgres_saver_from_pool(db_config.postgres_url) as saver:
+ yield saver
+ return
+
+ raise ValueError(f"Unknown database backend: {db_config.backend!r}")
+
+
+@contextlib.asynccontextmanager
+async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
+ """Async context manager that yields a checkpointer for the caller's lifetime.
+ Resources are opened on enter and closed on exit -- no global state::
+
+ async with make_checkpointer(app_config) as checkpointer:
+ app.state.checkpointer = checkpointer
+
+ Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
+
+ Priority:
+ 1. Legacy ``checkpointer:`` config section (backward compatible)
+ 2. Unified ``database:`` config section
+ 3. Default InMemorySaver
+ """
+
+ if app_config is None:
+ app_config = get_app_config()
+
+ # Legacy: standalone checkpointer config takes precedence
+ if app_config.checkpointer is not None:
+ async with _async_checkpointer(app_config.checkpointer) as saver:
+ yield saver
+ return
+
+ # Unified database config
+ db_config = getattr(app_config, "database", None)
+ if db_config is not None and db_config.backend != "memory":
+ async with _async_checkpointer_from_database(db_config) as saver:
+ yield saver
+ return
+
+ # Default: in-memory
+ from langgraph.checkpoint.memory import InMemorySaver
+
+ yield InMemorySaver()
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/runtime/store/async_provider.py b/offline-backend-20260512/backend/packages/harness/deerflow/runtime/store/async_provider.py
index 25156b01..94966e0d 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/runtime/store/async_provider.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/runtime/store/async_provider.py
@@ -28,6 +28,42 @@ from deerflow.runtime.store.provider import POSTGRES_CONN_REQUIRED, POSTGRES_STO
logger = logging.getLogger(__name__)
+
+def _pool_size_env(name: str, default: int) -> int:
+ import os
+
+ raw = os.getenv(name, "").strip()
+ if not raw:
+ return default
+ try:
+ return max(0, int(raw))
+ except ValueError:
+ logger.warning("Invalid %s=%r; using default %d", name, raw, default)
+ return default
+
+
+def _postgres_pool_config() -> dict:
+ """Pool config for the Postgres store.
+
+ A single long-lived connection (the default ``from_conn_string`` behavior)
+ dies permanently on a server restart / idle timeout / network blip — every
+ later store call then fails with "server closed the connection
+ unexpectedly". A pool with ``check=check_connection`` validates each
+ connection on checkout and replaces dead ones, so the runtime self-heals.
+ Extra keys (``check``/``max_idle``/``max_lifetime``) are passed straight
+ through to ``AsyncConnectionPool`` by ``from_conn_string``.
+ """
+ from psycopg_pool import AsyncConnectionPool
+
+ return {
+ "min_size": _pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
+ "max_size": _pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
+ "check": AsyncConnectionPool.check_connection,
+ "max_idle": 300.0,
+ "max_lifetime": 3600.0,
+ }
+
+
# ---------------------------------------------------------------------------
# Internal backend factory
# ---------------------------------------------------------------------------
@@ -71,9 +107,9 @@ async def _async_store(config) -> AsyncIterator[BaseStore]:
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
- async with AsyncPostgresStore.from_conn_string(config.connection_string) as store:
+ async with AsyncPostgresStore.from_conn_string(config.connection_string, pool_config=_postgres_pool_config()) as store:
await store.setup()
- logger.info("Store: using AsyncPostgresStore")
+ logger.info("Store: using AsyncPostgresStore (pooled, self-healing)")
yield store
return
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py
new file mode 100644
index 00000000..ed6797ae
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_act_tool.py
@@ -0,0 +1,225 @@
+"""Interactive browser action tool backed by the user's Chrome extension.
+
+Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
+*observe -> act* loop on a live page the extension keeps open:
+
+1. The model calls ``browser_act`` with an action (navigate/click/type/...).
+2. The extension performs the action on the session tab, then re-scans the page
+ and returns an *observation*: the final URL/title, an indexed list of
+ interactive elements, and the visible page text.
+3. The model reads the observation, picks the next element by its index, and
+ calls ``browser_act`` again with the same ``session_id``.
+
+The element ``index`` values are only valid for the most recent observation of a
+session; always act on indices from the latest result.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urlparse
+from uuid import uuid4
+
+from langchain.tools import tool
+
+from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
+from deerflow.config import get_app_config
+from deerflow.config.browser_context_config import BrowserContextConfig
+from deerflow.runtime.user_context import get_effective_user_id
+
+_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
+_NEEDS_INDEX = {"click", "type"}
+
+
+def _settings() -> BrowserContextConfig:
+ value = getattr(get_app_config(), "browser_context", None)
+ if isinstance(value, BrowserContextConfig):
+ return value
+ if isinstance(value, dict):
+ return BrowserContextConfig.model_validate(value)
+ return BrowserContextConfig()
+
+
+def _validate_url(raw_url: str) -> str | None:
+ url = (raw_url or "").strip()
+ parsed = urlparse(url)
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ return None
+ return url
+
+
+def _format_elements(elements: list[dict[str, Any]] | None) -> str:
+ if not elements:
+ return "(no interactive elements detected)"
+ lines: list[str] = []
+ for el in elements:
+ idx = el.get("index")
+ tag = str(el.get("tag") or "?")
+ el_type = str(el.get("type") or "").strip()
+ label = str(el.get("label") or "").strip().replace("\n", " ")
+ if len(label) > 120:
+ label = label[:120] + "…"
+ descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
+ extra = ""
+ if el.get("value"):
+ value = str(el["value"]).strip().replace("\n", " ")
+ extra = f' value="{value[:60]}"'
+ lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
+ return "\n".join(lines)
+
+
+def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
+ observation = observation or {}
+ status = str(observation.get("action_status") or "ok")
+ message = str(observation.get("message") or "").strip()
+ url = str(observation.get("url") or "").strip() or "(unknown)"
+ title = str(observation.get("title") or "").strip() or "(untitled)"
+ elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
+ text = str(observation.get("text") or "").strip()
+ text_limit = max(500, int(settings.observation_text_max_chars))
+ text_truncated = len(text) > text_limit
+ if text_truncated:
+ text = text[:text_limit]
+
+ parts = [
+ f"Browser session: {session_id}",
+ f"Last action: {action} -> {status}",
+ ]
+ if message:
+ parts.append(f"Note: {message}")
+ parts.extend(
+ [
+ f"URL: {url}",
+ f"Title: {title}",
+ "",
+ "Interactive elements (use the [index] with click/type):",
+ _format_elements(elements),
+ "",
+ "Page text:",
+ text or "(empty page text)",
+ ]
+ )
+ if text_truncated:
+ parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
+ parts.append(
+ "\nNext step: call browser_act again with this session_id and the index of the element to act on. "
+ "When the task is finished, call browser_act with action=\"close\" to release the tab."
+ )
+ return "\n".join(parts)
+
+
+@tool("browser_act", parse_docstring=True)
+async def browser_act_tool(
+ action: str,
+ session_id: str = "",
+ url: str = "",
+ index: int = -1,
+ text: str = "",
+ submit: bool = False,
+ wait_for: str = "",
+ wait_seconds: int = 0,
+) -> str:
+ """Drive an interactive observe/act loop on a page in the user's Chrome browser.
+
+ Use this to operate a real web page on the user's behalf — log in, fill forms,
+ click through navigation, and read pages that need the user's session or local
+ network. Start every task with ``action="navigate"`` and a URL to open a tab
+ and get the first observation, then keep calling this tool with the same
+ ``session_id``, choosing elements by the ``index`` shown in the observation.
+ Call ``action="close"`` when done to free the tab.
+
+ Args:
+ action: One of navigate, click, type, scroll, wait, extract, observe, close.
+ session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
+ url: Target http(s) URL. Required for navigate (and to start a new session).
+ index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
+ text: Text to enter for the type action.
+ submit: For type, press Enter / submit the form after typing.
+ wait_for: For wait, a CSS selector to wait for before observing.
+ wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
+ """
+
+ settings = _settings()
+ if not settings.enabled:
+ return "browser_act is disabled by config.yaml: browser_context.enabled is false."
+ if not settings.actions_enabled:
+ return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
+
+ action = (action or "").strip().lower()
+ if action not in _ACTIONS:
+ return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
+
+ session_id = (session_id or "").strip()
+ safe_url = _validate_url(url) if url else None
+
+ if action == "navigate" and safe_url is None:
+ return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
+
+ if not session_id:
+ if action != "navigate":
+ return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
+ session_id = uuid4().hex
+
+ if action in _NEEDS_INDEX and index < 0:
+ return f"browser_act {action} requires a non-negative element index from the latest observation."
+ if action == "type" and not text:
+ return "browser_act type requires the text to enter."
+
+ action_payload: dict[str, Any] = {"type": action}
+ if safe_url is not None:
+ action_payload["url"] = safe_url
+ if index >= 0:
+ action_payload["index"] = int(index)
+ if text:
+ action_payload["text"] = text
+ if submit:
+ action_payload["submit"] = True
+ if wait_for.strip():
+ action_payload["wait_for"] = wait_for.strip()[:300]
+
+ configured_timeout = max(5, int(settings.action_result_timeout_seconds))
+ requested_timeout = int(wait_seconds or configured_timeout)
+ timeout = min(max(5, requested_timeout), configured_timeout)
+ user_id = get_effective_user_id()
+
+ task = create_browser_task(
+ user_id=user_id,
+ url=safe_url or "",
+ instruction="",
+ kind="act",
+ session_id=session_id,
+ action=action_payload,
+ timeout_seconds=timeout + int(settings.task_lease_seconds),
+ max_content_chars=settings.max_content_chars,
+ metadata={"source": "browser_act_tool"},
+ )
+
+ completed = await wait_for_browser_task_result(
+ task_id=task["id"],
+ user_id=user_id,
+ timeout_seconds=timeout,
+ poll_interval_seconds=settings.task_poll_interval_seconds,
+ )
+ if completed is None:
+ return f"Browser task {task['id']} was not found after it was created."
+
+ status = completed.get("status")
+ if status == "error":
+ return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
+ if status == "expired":
+ return (
+ f"browser_act session {session_id}: the Chrome extension did not respond in time. "
+ "Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
+ )
+ if status != "done":
+ return (
+ f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
+ "Keep the Deer Web Clipper extension running with task polling enabled, then try again."
+ )
+
+ if action == "close":
+ return f"browser_act session {session_id}: tab closed."
+
+ result = completed.get("result") or {}
+ observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
+ return _format_observation(action, observation, session_id, settings)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py
new file mode 100644
index 00000000..6b065834
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/browser_context_tool.py
@@ -0,0 +1,117 @@
+"""Browser extension backed page fetch tool."""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from langchain.tools import tool
+
+from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
+from deerflow.config import get_app_config
+from deerflow.config.browser_context_config import BrowserContextConfig
+from deerflow.runtime.user_context import get_effective_user_id
+
+
+def _settings() -> BrowserContextConfig:
+ value = getattr(get_app_config(), "browser_context", None)
+ if isinstance(value, BrowserContextConfig):
+ return value
+ if isinstance(value, dict):
+ return BrowserContextConfig.model_validate(value)
+ return BrowserContextConfig()
+
+
+def _validate_url(raw_url: str) -> str | None:
+ url = (raw_url or "").strip()
+ parsed = urlparse(url)
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ return None
+ return url
+
+
+@tool("browser_fetch_page", parse_docstring=True)
+async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
+ """Ask the user's paired Chrome extension to fetch a page and return visible page text.
+
+ Use this when the user explicitly asks to use the browser extension or asks
+ for data from a page that may require their browser session, login state, or
+ local network access. The URL must be an exact http(s) URL provided by the
+ user. If the user did not provide a URL, ask for the URL instead of guessing.
+
+ Args:
+ url: Exact http(s) URL to open in the user's Chrome browser.
+ question: Optional instruction describing what data the user wants from the page.
+ wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
+ """
+
+ settings = _settings()
+ if not settings.enabled:
+ return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
+
+ safe_url = _validate_url(url)
+ if safe_url is None:
+ return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
+
+ configured_timeout = max(5, int(settings.task_result_timeout_seconds))
+ requested_timeout = int(wait_seconds or configured_timeout)
+ timeout = min(max(5, requested_timeout), configured_timeout)
+ user_id = get_effective_user_id()
+
+ task = create_browser_task(
+ user_id=user_id,
+ url=safe_url,
+ instruction=(question or "").strip(),
+ timeout_seconds=timeout + int(settings.task_lease_seconds),
+ max_content_chars=settings.max_content_chars,
+ metadata={"source": "browser_fetch_page_tool"},
+ )
+
+ completed = await wait_for_browser_task_result(
+ task_id=task["id"],
+ user_id=user_id,
+ timeout_seconds=timeout,
+ poll_interval_seconds=settings.task_poll_interval_seconds,
+ )
+ if completed is None:
+ return f"Browser task {task['id']} was not found after it was created."
+
+ status = completed.get("status")
+ if status == "error":
+ return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
+ if status == "expired":
+ return (
+ f"Browser task {task['id']} expired before the Chrome extension returned a result. "
+ "Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
+ )
+ if status != "done":
+ return (
+ f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
+ "Keep the Deer Web Clipper extension running with task polling enabled, then try again."
+ )
+
+ result = completed.get("result") or {}
+ content = str(result.get("content") or "").strip()
+ tool_limit = max(1_000, int(settings.tool_result_max_chars))
+ tool_truncated = len(content) > tool_limit
+ if tool_truncated:
+ content = content[:tool_limit]
+
+ notes: list[str] = []
+ if result.get("truncated"):
+ notes.append("The browser capture was truncated by browser_context.max_content_chars.")
+ if tool_truncated:
+ notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
+
+ parts = [
+ "Browser page fetched through the user's Chrome extension.",
+ f"Task ID: {task['id']}",
+ f"Title: {result.get('title') or '(untitled)'}",
+ f"URL: {result.get('url') or safe_url}",
+ ]
+ if question:
+ parts.extend(["", "Requested extraction:", question.strip()])
+ if notes:
+ parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
+ parts.extend(["", "Page content:", content or "(empty page text)"])
+ return "\n".join(parts)
+
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py
index 8a14e0a1..2bf592ee 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/tools.py
@@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry
logger = logging.getLogger(__name__)
+BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool"
+BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool"
+BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE}
+BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"}
+
BUILTIN_TOOLS = [
present_file_tool,
ask_clarification_tool,
@@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool:
return False
+def _is_browser_context_tool(tool: object) -> bool:
+ """Return True if the tool config represents the browser extension bridge."""
+
+ use = getattr(tool, "use", None)
+ name = getattr(tool, "name", None)
+ return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
+
+
def get_available_tools(
groups: list[str] | None = None,
include_mcp: bool = True,
@@ -64,6 +77,18 @@ def get_available_tools(
config = app_config or get_app_config()
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
+ browser_context_config = getattr(config, "browser_context", None)
+ if not getattr(browser_context_config, "enabled", False):
+ tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
+ elif not getattr(browser_context_config, "actions_enabled", True):
+ # browser_context is on, but interactive page actions are disabled — keep
+ # the read-only browser_fetch_page tool, drop only browser_act.
+ tool_configs = [
+ tool
+ for tool in tool_configs
+ if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
+ ]
+
# Do not expose host bash by default when LocalSandboxProvider is active.
if not is_host_bash_allowed(config):
tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)]
diff --git a/offline-backend-20260512/backend/scripts/sql/column_additions_mysql.sql b/offline-backend-20260512/backend/scripts/sql/column_additions_mysql.sql
new file mode 100644
index 00000000..51c1e20c
--- /dev/null
+++ b/offline-backend-20260512/backend/scripts/sql/column_additions_mysql.sql
@@ -0,0 +1,74 @@
+-- ============================================================================
+-- 给【已存在的表】补列(ADD COLUMN)—— MySQL 手动执行版
+-- ============================================================================
+--
+-- 背景:
+-- 本项目的 alembic 自动迁移是【关闭】的(engine.py 中
+-- `# await _run_alembic_upgrade(url)` 被注释)。`Base.metadata.create_all`
+-- 只会建【新表】,对【已存在的表】既不会补建索引、也不会补加新列。
+-- 因此当我们在某个 ORM model 上新增了一列时,【全新部署】会随建表一并带上,
+-- 但【已有数据的 MySQL 部署】上这一列并不会自动出现,代码读写该列会报错。
+--
+-- 约定(重要):
+-- 今后只要有代码给某张【已存在的表】新增字段(在 ORM model 上加列),就必须
+-- 把对应的 `ADD COLUMN` 语句以下面的【幂等】写法补进本文件,并在迁移目录里
+-- 保留对应的 alembic 版本文件,二者保持一致。运维在升级老库时整文件执行本脚本
+-- 即可补齐所有缺失的列。
+--
+-- 用法:
+-- 1) 先选中目标库(与 database.mysql_url 里的库名一致):
+-- USE your_deerflow_db;
+-- 2) 直接整文件执行即可。本脚本是【幂等】的:已存在的列会跳过,不会报错,
+-- 可重复执行。
+--
+-- 注意:
+-- * 列定义必须与 ORM model / alembic migration 完全一致(类型、长度、可空性)。
+-- * 新增列一律建议 NULL 或带 DEFAULT,避免在大表上回填导致长时间锁表。
+-- * MySQL 8.0 在线 DDL 默认 ALGORITHM=INPLACE 一般不阻塞写入,但仍建议低峰执行。
+-- ============================================================================
+
+-- ---- 通用幂等封装:仅当列不存在时才新增 --------------------------------------
+DELIMITER //
+DROP PROCEDURE IF EXISTS _add_column_if_missing //
+CREATE PROCEDURE _add_column_if_missing(
+ IN p_table VARCHAR(64),
+ IN p_column VARCHAR(64),
+ IN p_definition VARCHAR(255) -- 列定义,如 'VARCHAR(128) NULL DEFAULT NULL'
+)
+BEGIN
+ DECLARE v_exists INT DEFAULT 0;
+ SELECT COUNT(*) INTO v_exists
+ FROM information_schema.columns
+ WHERE table_schema = DATABASE()
+ AND table_name = p_table
+ AND column_name = p_column;
+
+ IF v_exists = 0 THEN
+ SET @ddl = CONCAT('ALTER TABLE `', p_table, '` ADD COLUMN `', p_column, '` ', p_definition);
+ PREPARE stmt FROM @ddl;
+ EXECUTE stmt;
+ DEALLOCATE PREPARE stmt;
+ SELECT CONCAT('ADDED ', p_table, '.', p_column) AS result;
+ ELSE
+ SELECT CONCAT('SKIPPED ', p_table, '.', p_column, ' (already exists)') AS result;
+ END IF;
+END //
+DELIMITER ;
+
+-- ---- skills ------------------------------------------------------------------
+-- name_zh: 技能的中文显示名(用户配置或 LLM 生成),对应 migration 20260609_01。
+-- ORM: SkillRow.name_zh = String(128), nullable=True。
+CALL _add_column_if_missing('skills', 'name_zh', 'VARCHAR(128) NULL DEFAULT NULL');
+-- detail: 技能的长文介绍,对应 migration 20260518_01。
+-- ORM: SkillRow.detail = Text, nullable=True。
+CALL _add_column_if_missing('skills', 'detail', 'TEXT NULL DEFAULT NULL');
+
+-- ---- 清理临时存储过程 --------------------------------------------------------
+DROP PROCEDURE IF EXISTS _add_column_if_missing;
+
+-- ---- 执行后自检:确认本批次涉及的列都已存在 ---------------------------------
+SELECT table_name, column_name
+ FROM information_schema.columns
+ WHERE table_schema = DATABASE()
+ AND (table_name = 'skills' AND column_name IN ('name_zh', 'detail'))
+ ORDER BY table_name, column_name;
diff --git a/offline-backend-20260512/backend/tests/test_browser_act.py b/offline-backend-20260512/backend/tests/test_browser_act.py
new file mode 100644
index 00000000..3dee216a
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_browser_act.py
@@ -0,0 +1,158 @@
+"""Tests for the interactive browser_act tool and its act-task queue support."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from deerflow.browser_context import tasks as task_queue
+from deerflow.config.browser_context_config import BrowserContextConfig
+from deerflow.tools.builtins import browser_act_tool as act_mod
+
+
+@pytest.fixture(autouse=True)
+def _isolated_task_home(tmp_path, monkeypatch):
+ """Point the file-backed task queue at a throwaway directory."""
+ monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
+ yield
+
+
+def _enable_actions(monkeypatch, *, enabled=True, actions_enabled=True):
+ cfg = BrowserContextConfig(
+ enabled=enabled,
+ actions_enabled=actions_enabled,
+ action_result_timeout_seconds=5,
+ task_poll_interval_seconds=0.25,
+ task_lease_seconds=30,
+ )
+ monkeypatch.setattr(act_mod, "_settings", lambda: cfg)
+ return cfg
+
+
+# --- task queue layer -------------------------------------------------------
+
+
+def test_act_task_round_trip_carries_session_and_observation():
+ task = task_queue.create_browser_task(
+ user_id="u1",
+ url="https://example.com",
+ kind="act",
+ session_id="sess-1",
+ action={"type": "navigate", "url": "https://example.com"},
+ )
+ assert task["kind"] == "act"
+ assert task["session_id"] == "sess-1"
+
+ claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1")
+ assert claimed is not None
+ assert claimed["kind"] == "act"
+ assert claimed["session_id"] == "sess-1"
+ assert claimed["action"] == {"type": "navigate", "url": "https://example.com"}
+
+ observation = {
+ "action_status": "ok",
+ "url": "https://example.com/home",
+ "title": "Home",
+ "elements": [{"index": 0, "tag": "button", "label": "Login"}],
+ "text": "welcome",
+ }
+ done = task_queue.complete_browser_task(
+ task_id=task["id"],
+ user_id="u1",
+ observation=observation,
+ )
+ assert done is not None
+ assert done["status"] == "done"
+ assert done["result"]["observation"] == observation
+
+
+def test_fetch_task_default_kind_is_fetch():
+ task = task_queue.create_browser_task(user_id="u1", url="https://example.com")
+ assert task["kind"] == "fetch"
+ claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="w1")
+ assert claimed["kind"] == "fetch"
+ assert claimed["session_id"] == ""
+ assert claimed["action"] is None
+
+
+# --- tool validation --------------------------------------------------------
+
+
+async def _invoke(**kwargs) -> str:
+ return await act_mod.browser_act_tool.ainvoke(kwargs)
+
+
+@pytest.mark.asyncio
+async def test_browser_act_disabled_when_actions_off(monkeypatch):
+ _enable_actions(monkeypatch, enabled=True, actions_enabled=False)
+ out = await _invoke(action="navigate", url="https://example.com")
+ assert "actions_enabled is false" in out
+
+
+@pytest.mark.asyncio
+async def test_browser_act_unknown_action(monkeypatch):
+ _enable_actions(monkeypatch)
+ out = await _invoke(action="frobnicate")
+ assert "unknown action" in out
+
+
+@pytest.mark.asyncio
+async def test_browser_act_navigate_requires_url(monkeypatch):
+ _enable_actions(monkeypatch)
+ out = await _invoke(action="navigate")
+ assert "requires an exact http(s) URL" in out
+
+
+@pytest.mark.asyncio
+async def test_browser_act_non_navigate_needs_session(monkeypatch):
+ _enable_actions(monkeypatch)
+ out = await _invoke(action="click", index=0)
+ assert "no session_id yet" in out
+
+
+@pytest.mark.asyncio
+async def test_browser_act_click_requires_index(monkeypatch):
+ _enable_actions(monkeypatch)
+ out = await _invoke(action="click", session_id="sess-1")
+ assert "requires a non-negative element index" in out
+
+
+# --- tool success path (simulated extension) --------------------------------
+
+
+@pytest.mark.asyncio
+async def test_browser_act_returns_formatted_observation(monkeypatch):
+ _enable_actions(monkeypatch)
+
+ async def fake_extension():
+ # Wait for the tool to create the pending act task, then complete it.
+ for _ in range(40):
+ claimed = task_queue.claim_next_browser_task(user_id="default", worker_id="ext")
+ if claimed is not None:
+ task_queue.complete_browser_task(
+ task_id=claimed["id"],
+ user_id="default",
+ observation={
+ "action_status": "ok",
+ "url": "https://example.com/home",
+ "title": "Home Page",
+ "elements": [
+ {"index": 0, "tag": "input", "type": "search", "label": "Search", "value": ""},
+ {"index": 1, "tag": "button", "label": "Go"},
+ ],
+ "text": "Welcome to the home page.",
+ },
+ )
+ return
+ await asyncio.sleep(0.1)
+
+ ext = asyncio.create_task(fake_extension())
+ out = await _invoke(action="navigate", url="https://example.com")
+ await ext
+
+ assert "Browser session:" in out
+ assert "Home Page" in out
+ assert "[0] input[search]" in out
+ assert "[1] button" in out
+ assert "Welcome to the home page." in out
diff --git a/offline-backend-20260512/config.yaml b/offline-backend-20260512/config.yaml
index 1956e4df..751c0156 100644
--- a/offline-backend-20260512/config.yaml
+++ b/offline-backend-20260512/config.yaml
@@ -1,76 +1,76 @@
config_version: 8
log_level: info
models:
-- name: deepseek-chat
- display_name: DeepSeek Chat
+# - name: deepseek-chat
+# display_name: DeepSeek Chat
+# use: langchain_openai:ChatOpenAI
+# model: deepseek-chat
+# api_key: sk-9808fb2375534112ac275541aec14b9b
+# base_url: https://api.deepseek.com
+# request_timeout: 600.0
+# max_retries: 2
+# max_tokens: 4096
+# temperature: 1.0
+# supports_vision: false
+# supports_thinking: true
+# - name: zai-org/GLM-5-FP8
+# display_name: zai-org/GLM-5-FP8
+# use: langchain_openai:ChatOpenAI
+# model: zai-org/GLM-5-FP8
+# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
+# base_url: https://api.us-west-2.modal.direct/v1
+# request_timeout: 600.0
+# max_retries: 2
+# max_tokens: 4096
+# temperature: 1.0
+# supports_vision: true
+# supports_thinking: true
+# - name: deepseek-v4-pro
+# display_name: DeepSeek V4 Pro
+# use: langchain_openai:ChatOpenAI
+# model: deepseek-v4-pro
+# api_key: sk-e1c22b7931dc446f8b67e57878edd105
+# base_url: https://api.deepseek.com
+# request_timeout: 600.0
+# max_retries: 2
+# max_tokens: 4096
+# temperature: 1.0
+# supports_vision: false
+# supports_thinking: true
+# - name: minimax-m2.7
+# display_name: MiniMax M2.7
+# use: langchain_openai:ChatOpenAI
+# model: MiniMax-M2.7
+# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
+# base_url: https://api.minimaxi.com
+# max_tokens: 4096
+# temperature: 1.0
+# supports_vision: true
+# supports_thinking: true
+# - name: minimax-m2.5
+# display_name: MiniMax M2.5
+# use: langchain_openai:ChatOpenAI
+# model: MiniMax-M2.5
+# api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
+# base_url: https://api.minimaxi.com
+# max_tokens: 4096
+# temperature: 1.0
+# supports_vision: true
+# supports_thinking: true
+# - name: deepseek-ai/DeepSeek-OCR
+# display_name: deepseek-ai/DeepSeek-OCR
+# use: langchain_openai:ChatOpenAI
+# model: deepseek-ai/DeepSeek-OCR
+# api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
+# base_url: https://api.siliconflow.cn/v1/
+# max_tokens: 4096
+# temperature: 1.0
+# supports_vision: true
+# supports_thinking: true
+- name: deepseek-ai/DeepSeek-V4-Flash
+ display_name: deepseek-ai/DeepSeek-V4-Flash
use: langchain_openai:ChatOpenAI
- model: deepseek-chat
- api_key: sk-9808fb2375534112ac275541aec14b9b
- base_url: https://api.deepseek.com
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: false
- supports_thinking: true
-- name: zai-org/GLM-5-FP8
- display_name: zai-org/GLM-5-FP8
- use: langchain_openai:ChatOpenAI
- model: zai-org/GLM-5-FP8
- api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
- base_url: https://api.us-west-2.modal.direct/v1
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
-- name: deepseek-v4-pro
- display_name: DeepSeek V4 Pro
- use: langchain_openai:ChatOpenAI
- model: deepseek-v4-pro
- api_key: sk-e1c22b7931dc446f8b67e57878edd105
- base_url: https://api.deepseek.com
- request_timeout: 600.0
- max_retries: 2
- max_tokens: 4096
- temperature: 1.0
- supports_vision: false
- supports_thinking: true
-- name: minimax-m2.7
- display_name: MiniMax M2.7
- use: langchain_openai:ChatOpenAI
- model: MiniMax-M2.7
- api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
- base_url: https://api.minimaxi.com
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
-- name: minimax-m2.5
- display_name: MiniMax M2.5
- use: langchain_openai:ChatOpenAI
- model: MiniMax-M2.5
- api_key: sk-cp-mrsAGtzDpUl0TpOBPVoiRG_pWF1kDDN9_mDAnONBs6HIPcNUexSKtSbx2QOxfKBvJT6Xw6hDTV6hdkJYFWfNA8UtHQ08TiGJiosdRoW6q1sGb6Cb7D85NHk
- base_url: https://api.minimaxi.com
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
-- name: deepseek-ai/DeepSeek-OCR
- display_name: deepseek-ai/DeepSeek-OCR
- use: langchain_openai:ChatOpenAI
- model: deepseek-ai/DeepSeek-OCR
- api_key: sk-beshfxfxfrllsczpqfrgjbugfbpzxgqgedbehceytpcvvtyb
- base_url: https://api.siliconflow.cn/v1/
- max_tokens: 4096
- temperature: 1.0
- supports_vision: true
- supports_thinking: true
-- name: Qwen/Qwen3.6-27B
- display_name: Qwen/Qwen3.6-27B
- use: langchain_openai:ChatOpenAI
- model: Qwen/Qwen3.6-27B
+ model: deepseek-ai/DeepSeek-V4-Flash
api_key: sk-qwwwvjompnzxqhndctlwmmomvqdgaqqqgxvibfxchsikvgod
base_url: https://api.siliconflow.cn/v1/
max_tokens: 4096
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
new file mode 100644
index 00000000..6ec1aeb7
--- /dev/null
+++ b/pnpm-lock.yaml
@@ -0,0 +1,246 @@
+lockfileVersion: '9.0'
+
+settings:
+ autoInstallPeers: true
+ excludeLinksFromLockfile: false
+
+importers:
+
+ .:
+ dependencies:
+ axios:
+ specifier: ^1.16.1
+ version: 1.17.0
+
+packages:
+
+ agent-base@6.0.2:
+ resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
+ engines: {node: '>= 6.0.0'}
+
+ asynckit@0.4.0:
+ resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+
+ axios@1.17.0:
+ resolution: {integrity: sha512-J8SwNxprqqpbfenehxWYXE7CW+wM1BB4w3+N+g+/Wx40xM4rsLrfPmHHxSWIxJLYDgSY/HqlFPIYb2/S3rxafw==}
+
+ call-bind-apply-helpers@1.0.2:
+ resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==}
+ engines: {node: '>= 0.4'}
+
+ combined-stream@1.0.8:
+ resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+ engines: {node: '>= 0.8'}
+
+ debug@4.4.3:
+ resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==}
+ engines: {node: '>=6.0'}
+ peerDependencies:
+ supports-color: '*'
+ peerDependenciesMeta:
+ supports-color:
+ optional: true
+
+ delayed-stream@1.0.0:
+ resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+ engines: {node: '>=0.4.0'}
+
+ dunder-proto@1.0.1:
+ resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
+ engines: {node: '>= 0.4'}
+
+ es-define-property@1.0.1:
+ resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==}
+ engines: {node: '>= 0.4'}
+
+ es-errors@1.3.0:
+ resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
+ engines: {node: '>= 0.4'}
+
+ es-object-atoms@1.1.2:
+ resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==}
+ engines: {node: '>= 0.4'}
+
+ es-set-tostringtag@2.1.0:
+ resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}
+ engines: {node: '>= 0.4'}
+
+ follow-redirects@1.16.0:
+ resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==}
+ engines: {node: '>=4.0'}
+ peerDependencies:
+ debug: '*'
+ peerDependenciesMeta:
+ debug:
+ optional: true
+
+ form-data@4.0.5:
+ resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==}
+ engines: {node: '>= 6'}
+
+ function-bind@1.1.2:
+ resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
+
+ get-intrinsic@1.3.0:
+ resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==}
+ engines: {node: '>= 0.4'}
+
+ get-proto@1.0.1:
+ resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}
+ engines: {node: '>= 0.4'}
+
+ gopd@1.2.0:
+ resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
+ engines: {node: '>= 0.4'}
+
+ has-symbols@1.1.0:
+ resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}
+ engines: {node: '>= 0.4'}
+
+ has-tostringtag@1.0.2:
+ resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
+ engines: {node: '>= 0.4'}
+
+ hasown@2.0.4:
+ resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==}
+ engines: {node: '>= 0.4'}
+
+ https-proxy-agent@5.0.1:
+ resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
+ engines: {node: '>= 6'}
+
+ math-intrinsics@1.1.0:
+ resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
+ engines: {node: '>= 0.4'}
+
+ mime-db@1.52.0:
+ resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
+ engines: {node: '>= 0.6'}
+
+ mime-types@2.1.35:
+ resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
+ engines: {node: '>= 0.6'}
+
+ ms@2.1.3:
+ resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+
+ proxy-from-env@2.1.0:
+ resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==}
+ engines: {node: '>=10'}
+
+snapshots:
+
+ agent-base@6.0.2:
+ dependencies:
+ debug: 4.4.3
+ transitivePeerDependencies:
+ - supports-color
+
+ asynckit@0.4.0: {}
+
+ axios@1.17.0:
+ dependencies:
+ follow-redirects: 1.16.0
+ form-data: 4.0.5
+ https-proxy-agent: 5.0.1
+ proxy-from-env: 2.1.0
+ transitivePeerDependencies:
+ - debug
+ - supports-color
+
+ call-bind-apply-helpers@1.0.2:
+ dependencies:
+ es-errors: 1.3.0
+ function-bind: 1.1.2
+
+ combined-stream@1.0.8:
+ dependencies:
+ delayed-stream: 1.0.0
+
+ debug@4.4.3:
+ dependencies:
+ ms: 2.1.3
+
+ delayed-stream@1.0.0: {}
+
+ dunder-proto@1.0.1:
+ dependencies:
+ call-bind-apply-helpers: 1.0.2
+ es-errors: 1.3.0
+ gopd: 1.2.0
+
+ es-define-property@1.0.1: {}
+
+ es-errors@1.3.0: {}
+
+ es-object-atoms@1.1.2:
+ dependencies:
+ es-errors: 1.3.0
+
+ es-set-tostringtag@2.1.0:
+ dependencies:
+ es-errors: 1.3.0
+ get-intrinsic: 1.3.0
+ has-tostringtag: 1.0.2
+ hasown: 2.0.4
+
+ follow-redirects@1.16.0: {}
+
+ form-data@4.0.5:
+ dependencies:
+ asynckit: 0.4.0
+ combined-stream: 1.0.8
+ es-set-tostringtag: 2.1.0
+ hasown: 2.0.4
+ mime-types: 2.1.35
+
+ function-bind@1.1.2: {}
+
+ get-intrinsic@1.3.0:
+ dependencies:
+ call-bind-apply-helpers: 1.0.2
+ es-define-property: 1.0.1
+ es-errors: 1.3.0
+ es-object-atoms: 1.1.2
+ function-bind: 1.1.2
+ get-proto: 1.0.1
+ gopd: 1.2.0
+ has-symbols: 1.1.0
+ hasown: 2.0.4
+ math-intrinsics: 1.1.0
+
+ get-proto@1.0.1:
+ dependencies:
+ dunder-proto: 1.0.1
+ es-object-atoms: 1.1.2
+
+ gopd@1.2.0: {}
+
+ has-symbols@1.1.0: {}
+
+ has-tostringtag@1.0.2:
+ dependencies:
+ has-symbols: 1.1.0
+
+ hasown@2.0.4:
+ dependencies:
+ function-bind: 1.1.2
+
+ https-proxy-agent@5.0.1:
+ dependencies:
+ agent-base: 6.0.2
+ debug: 4.4.3
+ transitivePeerDependencies:
+ - supports-color
+
+ math-intrinsics@1.1.0: {}
+
+ mime-db@1.52.0: {}
+
+ mime-types@2.1.35:
+ dependencies:
+ mime-db: 1.52.0
+
+ ms@2.1.3: {}
+
+ proxy-from-env@2.1.0: {}