feat: add skill and assistant knowledge bases

This commit is contained in:
Administrator committed 2026-09-02 00:49:13 +08:00
1 parent dd0ee282df
commit fbd8dec48f
45 files changed
+5262 -21

No files matched your search

@@ -0,0 +1,7 @@
# 技能知识归纳与“知识梳理”助手知识库
管理员在「技能管理 → 技能知识归纳」中选择一个或多个技能、一个或多个目标,系统按“技能 × 目标”创建后台任务。界面提供总体进度、单项失败重试、实体/关系审核、绑定重归纳、自动同步和两种解绑动作。技能列表通过一次 bindings 查询展示归纳目标,避免逐技能请求。
扫描仅处理知识资料。代码、脚本、锁文件、密钥文件以及 Markdown 代码块不会进入上传、摘要、提示词或知识指纹;ZIP 只读取逐项验证通过的知识成员。自动监听只比较该知识指纹,所以纯代码改动不会产生 stale 状态。
知识库页面新增「普通知识库 / 助手知识库」并行入口。管理员可将系统默认入口设为 v1(普通 WeKnora)或 v2(全局“知识梳理”)。助手库无需 WeKnora 即可初始化,所有用户可只读查看和搜索 Wiki;管理员可上传本地文件、从普通库导入、重新导入或删除来源、查看版本并回滚。来源删除只移除对应贡献,共享实体与关系继续保留。
@@ -0,0 +1,55 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { BookOpen, Database } from "lucide-react";
import { useLocation, useNavigate } from "react-router-dom";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { getStoredAuth } from "@/core/auth";
import {
loadKnowledgeBaseProductVersion,
type KnowledgeBaseProductVersion,
updateKnowledgeBaseProductVersion,
} from "@/core/system-settings/api";
export const KNOWLEDGE_PRODUCT_KEY = ["system-settings", "knowledge-base-product-version"] as const;
export function useKnowledgeBaseProductVersion() {
return useQuery({ queryKey: KNOWLEDGE_PRODUCT_KEY, queryFn: loadKnowledgeBaseProductVersion, staleTime: 60_000 });
}
export function KnowledgeProductNav() {
const location = useLocation();
const navigate = useNavigate();
const client = useQueryClient();
const version = useKnowledgeBaseProductVersion();
const isAdmin = getStoredAuth()?.system_role === "admin";
const save = useMutation({
mutationFn: updateKnowledgeBaseProductVersion,
onSuccess: (value) => {
client.setQueryData(KNOWLEDGE_PRODUCT_KEY, value);
toast.success(`默认知识库体验已切换为 ${value}`);
},
});
const assistant = location.pathname.includes("/knowledge/assistant");
return (
<div className="flex items-center gap-2 border-b bg-card px-6 py-2">
<Button size="sm" variant={assistant ? "ghost" : "secondary"} onClick={() => navigate("/page/workspace/knowledge")}>
<BookOpen />普通知识库
</Button>
<Button size="sm" variant={assistant ? "secondary" : "ghost"} onClick={() => navigate("/page/workspace/knowledge/assistant")}>
<Database />助手知识库
</Button>
{isAdmin && (
<div className="ml-auto flex items-center gap-2 text-xs text-muted-foreground">
默认产品版本
<Select value={version.data ?? "v1"} onValueChange={(value) => save.mutate(value as KnowledgeBaseProductVersion)}>
<SelectTrigger className="h-8 w-40"><SelectValue /></SelectTrigger>
<SelectContent><SelectItem value="v1">v1 普通知识库</SelectItem><SelectItem value="v2">v2 助手知识库</SelectItem></SelectContent>
</Select>
</div>
)}
</div>
);
}
@@ -92,7 +92,8 @@ import {
useSkillCompressionSettings,
useViewMode,
} from "@/core/system-settings/hooks";
import type { Skill } from "@/core/skills/type";
import type { Skill } from "@/core/skills/type";
import type { SkillKnowledgeBinding } from "@/strategy-components/api/skill-knowledge";
import { formatDateTime } from "@/core/utils/datetime";
import { env } from "@/env";
import { desensitize } from "@/lib/desensitize";
@@ -599,8 +600,9 @@ export function SkillRow({
skill,
currentUserId,
isAdmin,
layout = "row",
onPublisherClick,
layout = "row",
onPublisherClick,
knowledgeBindings = [],
}: {
skill: Skill;
currentUserId: string | null;
@@ -609,7 +611,9 @@ export function SkillRow({
layout?: "row" | "card";
/** Click the 发布者 line to filter the list to that publisher. Receives the
* owner_user_id + its desensitized display label. */
onPublisherClick?: (publisher: { id: string; label: string }) => void;
onPublisherClick?: (publisher: { id: string; label: string }) => void;
/** Admin list can inject its single batched binding query here (never N+1). */
knowledgeBindings?: SkillKnowledgeBinding[];
}) {
const isCard = layout === "card";
const { t } = useI18n();
@@ -849,7 +853,7 @@ export function SkillRow({
{t.settings.skills.badgePinned}
</Badge>
)}
{skill.always_on && (
{skill.always_on && (
<Badge
variant="outline"
className="border-amber-500/30 text-amber-700 dark:text-amber-300 text-xs"
@@ -857,7 +861,21 @@ export function SkillRow({
>
常驻
</Badge>
)}
)}
{knowledgeBindings.slice(0, 3).map((binding) => (
<Badge
key={binding.id}
variant="outline"
className="border-indigo-500/30 text-indigo-700 dark:text-indigo-300 text-xs"
title={`${binding.target_name} · ${binding.status}${binding.remote_business_key ? ` · ${binding.remote_business_key}` : ""}`}
>
知识库:{binding.target_name || binding.target_id}
{binding.status === "stale" ? "(待更新)" : ""}
</Badge>
))}
{knowledgeBindings.length > 3 && (
<Badge variant="secondary" className="text-xs">+{knowledgeBindings.length - 3}</Badge>
)}
</div>
);
@@ -0,0 +1,226 @@
import { CheckCircle2, Database, Loader2, RefreshCw, Sparkles } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Progress } from "@/components/ui/progress";
import { ScrollArea } from "@/components/ui/scroll-area";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useAllSkillsAdmin } from "@/core/skills/hooks";
import {
useAssistantKnowledgeActions,
useAssistantKnowledgeBase,
} from "@/strategy-components/api/assistant-knowledge";
import { useLlmWikiKnowledgeBases, useLlmWikiRuntime } from "@/strategy-components/api/llmwiki";
import {
type SkillKnowledgeTarget,
useSkillKnowledgeActions,
useSkillKnowledgeAutoSyncSettings,
useSkillKnowledgeBindings,
useSkillKnowledgeJobs,
useSkillKnowledgeReviews,
} from "@/strategy-components/api/skill-knowledge";
const PHASE_LABELS: Record<string, string> = {
scanning: "扫描知识文件",
parsing: "解析资料",
extracting: "抽取实体关系",
awaiting_review: "等待审核",
merging: "合并实体关系",
writing_wiki: "写入 Wiki",
activating: "激活新版本",
completed: "完成",
};
export function SkillKnowledgePanel() {
const { skills, isLoading } = useAllSkillsAdmin(true);
const liveSkills = useMemo(() => skills.filter((skill) => (skill.state ?? "active") === "active"), [skills]);
const [selectedSkills, setSelectedSkills] = useState<string[]>([]);
const [selectedTargetKeys, setSelectedTargetKeys] = useState<string[]>([]);
const [reviewMode, setReviewMode] = useState<"required" | "auto_high_confidence" | "off">("auto_high_confidence");
const runtime = useLlmWikiRuntime();
const knowledgeBases = useLlmWikiKnowledgeBases("all", runtime.data?.provider === "weknora");
const assistantBase = useAssistantKnowledgeBase();
const assistantActions = useAssistantKnowledgeActions();
const bindings = useSkillKnowledgeBindings();
const jobs = useSkillKnowledgeJobs();
const reviews = useSkillKnowledgeReviews();
const actions = useSkillKnowledgeActions();
const autoSettings = useSkillKnowledgeAutoSyncSettings();
const [scheduleCron, setScheduleCron] = useState("0 */6 * * *");
const [debounceSeconds, setDebounceSeconds] = useState(120);
useEffect(() => {
if (!autoSettings.data) return;
setScheduleCron(autoSettings.data.schedule_cron);
setDebounceSeconds(autoSettings.data.debounce_seconds);
}, [autoSettings.data]);
const targets = useMemo<SkillKnowledgeTarget[]>(() => {
const rows: SkillKnowledgeTarget[] = (knowledgeBases.data ?? []).map((kb) => ({
target_type: "weknora",
target_mode: kb.can_write ? kb.type === "faq" ? "faq" : kb.type === "wiki" ? "wiki" : "document" : "readonly_sidecar",
target_id: kb.id,
target_name: kb.name,
remote_business_key: `weknora:kb:${kb.id}`,
readonly: !kb.can_write,
}));
if (assistantBase.data) {
rows.unshift({
target_type: "assistant",
target_mode: "assistant",
target_id: assistantBase.data.id,
target_name: assistantBase.data.name,
remote_business_key: `assistant:base:${assistantBase.data.id}`,
});
}
return rows;
}, [assistantBase.data, knowledgeBases.data]);
const toggle = (values: string[], value: string, setter: (next: string[]) => void) =>
setter(values.includes(value) ? values.filter((item) => item !== value) : [...values, value]);
const submit = async () => {
const selectedTargets = targets.filter((target) =>
selectedTargetKeys.includes(`${target.target_type}:${target.target_id}`),
);
if (!selectedSkills.length || !selectedTargets.length) {
toast.error("请至少选择一个技能和一个目标知识库");
return;
}
try {
const job = await actions.createJob.mutateAsync({
skill_names: selectedSkills,
targets: selectedTargets,
review_mode: reviewMode,
});
toast.success(`已创建 ${job.total_count} 个归纳项`);
setSelectedSkills([]);
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
}
};
return (
<div className="grid size-full min-h-0 grid-cols-[minmax(320px,0.9fr)_minmax(420px,1.1fr)] gap-4 overflow-hidden p-6">
<section className="flex min-h-0 flex-col rounded-lg border bg-card">
<div className="border-b p-4">
<h2 className="font-semibold">批量归纳</h2>
<p className="mt-1 text-xs text-muted-foreground">按“技能 × 目标知识库”生成独立后台任务;代码与脚本不会被读取。</p>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-5 p-4">
<div>
<div className="mb-2 flex items-center justify-between">
<span className="text-sm font-medium">1. 选择技能({selectedSkills.length})</span>
<Button variant="ghost" size="sm" onClick={() => setSelectedSkills(selectedSkills.length === liveSkills.length ? [] : liveSkills.map((skill) => skill.name))}>全选/清空</Button>
</div>
<div className="max-h-60 space-y-1 overflow-y-auto rounded-md border p-2">
{isLoading ? <Loader2 className="m-4 size-5 animate-spin" /> : liveSkills.map((skill) => (
<label key={skill.name} className="flex cursor-pointer items-center gap-2 rounded px-2 py-1.5 hover:bg-muted">
<Checkbox checked={selectedSkills.includes(skill.name)} onCheckedChange={() => toggle(selectedSkills, skill.name, setSelectedSkills)} />
<span className="truncate text-sm">{skill.name_zh || skill.name}</span>
<span className="ml-auto truncate text-xs text-muted-foreground">{skill.name}</span>
</label>
))}
</div>
</div>
<div>
<div className="mb-2 flex items-center justify-between">
<span className="text-sm font-medium">2. 选择目标({selectedTargetKeys.length})</span>
{!assistantBase.data && (
<Button size="sm" variant="outline" disabled={assistantActions.initialize.isPending} onClick={() => assistantActions.initialize.mutate("知识梳理")}>创建知识梳理库</Button>
)}
</div>
<div className="space-y-1 rounded-md border p-2">
{targets.map((target) => {
const key = `${target.target_type}:${target.target_id}`;
return (
<label key={key} className="flex cursor-pointer items-center gap-2 rounded px-2 py-2 hover:bg-muted">
<Checkbox checked={selectedTargetKeys.includes(key)} onCheckedChange={() => toggle(selectedTargetKeys, key, setSelectedTargetKeys)} />
<Database className="size-4 text-muted-foreground" />
<span className="text-sm">{target.target_name}</span>
<Badge variant="outline" className="ml-auto text-[10px]">{target.target_type === "assistant" ? "助手库" : target.target_mode === "readonly_sidecar" ? "只读·本地投影" : target.target_mode}</Badge>
</label>
);
})}
{!targets.length && <p className="p-3 text-xs text-muted-foreground">未配置普通知识库;可先创建全局“知识梳理”助手库。</p>}
</div>
</div>
<div className="flex items-center gap-3">
<span className="text-sm font-medium">审核策略</span>
<Select value={reviewMode} onValueChange={(value) => setReviewMode(value as typeof reviewMode)}>
<SelectTrigger className="w-52"><SelectValue /></SelectTrigger>
<SelectContent>
<SelectItem value="auto_high_confidence">高置信自动通过</SelectItem>
<SelectItem value="required">全部逐条审核</SelectItem>
<SelectItem value="off">不阻塞写入</SelectItem>
</SelectContent>
</Select>
</div>
<Button className="w-full" disabled={actions.createJob.isPending} onClick={() => void submit()}>
{actions.createJob.isPending ? <Loader2 className="animate-spin" /> : <Sparkles />}
开始归纳({selectedSkills.length * selectedTargetKeys.length} 项)
</Button>
</div>
</ScrollArea>
</section>
<section className="flex min-h-0 flex-col gap-4 overflow-hidden">
<div className="flex min-h-0 flex-1 flex-col rounded-lg border bg-card">
<div className="flex items-center justify-between border-b p-4">
<div><h2 className="font-semibold">归纳任务</h2><p className="text-xs text-muted-foreground">后台执行、失败可单项重试</p></div>
<Button size="icon" variant="ghost" onClick={() => void jobs.refetch()}><RefreshCw className="size-4" /></Button>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-3 p-4">
{(jobs.data ?? []).map((job) => {
const done = job.success_count + job.partial_failed_count + job.failed_count;
const progress = job.total_count ? Math.round((done / job.total_count) * 100) : 0;
return (
<div key={job.id} className="rounded-md border p-3">
<div className="flex items-center gap-2 text-sm"><Badge variant={job.status === "completed" ? "default" : "secondary"}>{job.status}</Badge><span>{done}/{job.total_count} 项</span><span className="ml-auto text-xs text-muted-foreground">{new Date(job.created_at).toLocaleString()}</span></div>
<Progress className="my-2" value={progress} />
{job.items?.map((item) => (
<div key={item.id} className="mt-2 flex items-center gap-2 border-t pt-2 text-xs">
<span className="max-w-36 truncate font-medium">{item.skill_name}</span>
<span className="text-muted-foreground">{PHASE_LABELS[item.phase] || item.phase}</span>
<span className="ml-auto">{item.progress}%</span>
{item.status === "failed" && <Button size="sm" variant="outline" onClick={() => actions.retry.mutate(item.id)}>重试</Button>}
</div>
))}
</div>
);
})}
{!jobs.data?.length && <p className="py-10 text-center text-sm text-muted-foreground">暂无归纳任务</p>}
</div>
</ScrollArea>
</div>
{(reviews.data?.length ?? 0) > 0 && (
<div className="max-h-64 overflow-auto rounded-lg border bg-card p-4">
<div className="mb-3 flex items-center"><h2 className="font-semibold">待审核({reviews.data!.length})</h2><Button size="sm" className="ml-auto" onClick={() => actions.review.mutate({ review_ids: reviews.data!.map((row) => row.id), action: "approve" })}><CheckCircle2 />批量批准</Button></div>
{reviews.data!.map((review) => <div key={review.id} className="mb-2 flex items-center gap-2 rounded border p-2 text-xs"><Badge variant="outline">{review.object_type}</Badge><code className="truncate">{String(review.original.name || review.original.predicate || review.object_key)}</code><Button size="sm" variant="ghost" className="ml-auto" onClick={() => actions.review.mutate({ review_ids: [review.id], action: "approve" })}>批准</Button><Button size="sm" variant="ghost" onClick={() => actions.review.mutate({ review_ids: [review.id], action: "reject" })}>拒绝</Button></div>)}
</div>
)}
<div className="max-h-80 overflow-auto rounded-lg border bg-card p-3 text-xs">
<div className="mb-3 flex flex-wrap items-center gap-3 border-b pb-3">
<span className="font-medium">自动同步</span>
{autoSettings.data && <>
<label className="flex items-center gap-1"><Checkbox checked={autoSettings.data.watch_enabled} onCheckedChange={(checked) => actions.updateAutoSyncSettings.mutate({ watch_enabled: checked === true })} />文件监听</label>
<label className="flex items-center gap-1"><Checkbox checked={autoSettings.data.schedule_enabled} onCheckedChange={(checked) => actions.updateAutoSyncSettings.mutate({ schedule_enabled: checked === true })} />定时同步</label>
<label className="flex items-center gap-1"><Checkbox checked={autoSettings.data.write_remote_enabled} onCheckedChange={(checked) => actions.updateAutoSyncSettings.mutate({ write_remote_enabled: checked === true })} />自动写入目标</label>
<Input className="h-8 w-32" value={scheduleCron} onChange={(event) => setScheduleCron(event.target.value)} aria-label="自动同步 cron" />
<Input className="h-8 w-24" type="number" min={10} value={debounceSeconds} onChange={(event) => setDebounceSeconds(Number(event.target.value))} aria-label="防抖秒数" />
<Button size="sm" variant="outline" onClick={() => actions.updateAutoSyncSettings.mutate({ schedule_cron: scheduleCron, debounce_seconds: debounceSeconds }, { onSuccess: () => toast.success("自动同步设置已保存"), onError: (error) => toast.error(error.message) })}>保存</Button>
</>}
</div>
<div className="mb-2 text-muted-foreground">当前有效绑定:{bindings.data?.length ?? 0};同名同类型实体按来源贡献合并。</div>
<div className="space-y-2">{bindings.data?.map((binding) => <div key={binding.id} className="flex flex-wrap items-center gap-2 rounded border p-2"><span className="max-w-32 truncate font-medium">{binding.skill_name}</span><span className="max-w-40 truncate text-muted-foreground">→ {binding.target_name}</span><Badge variant="outline">{binding.status}</Badge><code className="max-w-40 truncate text-[10px] text-muted-foreground">{binding.remote_business_key}</code><label className="ml-auto flex items-center gap-1"><Checkbox checked={binding.auto_sync_enabled} onCheckedChange={(checked) => actions.setAutoSync.mutate({ id: binding.id, enabled: checked === true })} />自动</label><Button size="sm" variant="outline" disabled={actions.resync.isPending} onClick={() => actions.resync.mutate(binding.id)}>重归纳</Button><Button size="sm" variant="ghost" onClick={() => { if (window.confirm("解除关联并保留目标知识内容?")) actions.detach.mutate({ id: binding.id, deleteContributions: false }); }}>解绑并保留</Button><Button size="sm" variant="destructive" onClick={() => { if (window.confirm("解除关联并删除此技能贡献?共享知识仍会保留其他来源。")) actions.detach.mutate({ id: binding.id, deleteContributions: true }); }}>解绑并删除贡献</Button></div>)}</div>
</div>
</section>
</div>
);
}
@@ -8,6 +8,31 @@ export interface PromptPrefixSettings {
prompt_prefix: string;
}
export type KnowledgeBaseProductVersion = "v1" | "v2";
export async function loadKnowledgeBaseProductVersion(): Promise<KnowledgeBaseProductVersion> {
const response = await apiFetch(
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
);
if (!response.ok) throw new Error(`Failed to load knowledge base product version: ${response.status}`);
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
}
export async function updateKnowledgeBaseProductVersion(
version: KnowledgeBaseProductVersion,
): Promise<KnowledgeBaseProductVersion> {
const response = await apiFetch(
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ version }),
},
);
if (!response.ok) throw new Error(`Failed to update knowledge base product version: ${response.status}`);
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
}
export interface PluginDomainAgentMappingDto {
id: string;
domain: string;
@@ -3,6 +3,7 @@
import {
ChevronLeftIcon,
ChevronRightIcon,
DatabaseZap,
Replace,
SearchIcon,
SparklesIcon,
@@ -18,9 +19,11 @@ import {
WorkspaceHeader,
} from "@/components/workspace/workspace-container";
import { SkillRow } from "@/components/workspace/settings/skill-settings-page";
import { SkillKnowledgePanel } from "@/components/workspace/skills/SkillKnowledgePanel";
import { getStoredAuth } from "@/core/auth";
import { useAllSkillsAdmin } from "@/core/skills/hooks";
import { desensitize } from "@/lib/desensitize";
import { useSkillKnowledgeBindings } from "@/strategy-components/api/skill-knowledge";
import { SkillAddressesPanel } from "./SkillAddressesPage";
const PAGE_SIZE = 10;
@@ -29,6 +32,7 @@ export default function AdminSkillsPage() {
const [search, setSearch] = useState("");
const [page, setPage] = useState(1);
const { skills, isLoading, error } = useAllSkillsAdmin(true);
const knowledgeBindings = useSkillKnowledgeBindings();
const storedAuth = useMemo(() => getStoredAuth(), []);
const currentUserId = storedAuth?.user_id ?? null;
@@ -71,6 +75,10 @@ export default function AdminSkillsPage() {
<Replace />
技能地址管理
</TabsTrigger>
<TabsTrigger value="knowledge">
<DatabaseZap />
技能知识归纳
</TabsTrigger>
</TabsList>
</div>
</div>
@@ -126,6 +134,9 @@ export default function AdminSkillsPage() {
skill={skill}
currentUserId={currentUserId}
isAdmin
knowledgeBindings={(knowledgeBindings.data ?? []).filter(
(binding) => binding.skill_name === skill.name,
)}
/>
))}
</>
@@ -164,6 +175,12 @@ export default function AdminSkillsPage() {
>
<SkillAddressesPanel />
</TabsContent>
<TabsContent
value="knowledge"
className="mt-0 min-h-0 flex-1 overflow-hidden"
>
<SkillKnowledgePanel />
</TabsContent>
</Tabs>
</WorkspaceBody>
</WorkspaceContainer>
@@ -0,0 +1,74 @@
import { Database, FileUp, History, Loader2, RefreshCw, Search, ShieldCheck, Trash2 } from "lucide-react";
import { useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { ScrollArea } from "@/components/ui/scroll-area";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
import { KnowledgeProductNav } from "@/components/workspace/knowledge-product-nav";
import { getStoredAuth } from "@/core/auth";
import {
listAssistantWikiRevisions,
type AssistantWikiPage,
type AssistantWikiRevision,
useAssistantImportJobs,
useAssistantKnowledgeActions,
useAssistantKnowledgeBase,
useAssistantKnowledgeSources,
useAssistantWikiPages,
} from "@/strategy-components/api/assistant-knowledge";
export default function AssistantKnowledgePage() {
const isAdmin = getStoredAuth()?.system_role === "admin";
const [query, setQuery] = useState("");
const [selected, setSelected] = useState<AssistantWikiPage | null>(null);
const [revisions, setRevisions] = useState<AssistantWikiRevision[]>([]);
const fileRef = useRef<HTMLInputElement>(null);
const base = useAssistantKnowledgeBase();
const pages = useAssistantWikiPages(query);
const sources = useAssistantKnowledgeSources();
const jobs = useAssistantImportJobs();
const actions = useAssistantKnowledgeActions();
const activePage = selected ?? pages.data?.[0] ?? null;
const counts = useMemo(() => ({ pages: pages.data?.length ?? 0, sources: sources.data?.length ?? 0, jobs: jobs.data?.length ?? 0 }), [pages.data, sources.data, jobs.data]);
if (base.isLoading) return <div className="flex size-full flex-col"><KnowledgeProductNav /><div className="flex flex-1 items-center justify-center"><Loader2 className="animate-spin" /></div></div>;
if (!base.data) {
return <div className="flex size-full flex-col"><KnowledgeProductNav /><div className="flex flex-1 items-center justify-center p-8"><div className="max-w-lg rounded-xl border bg-card p-8 text-center"><Database className="mx-auto mb-4 size-10 text-primary" /><h1 className="text-xl font-semibold">创建“知识梳理”助手知识库</h1><p className="mt-2 text-sm text-muted-foreground">无需配置普通知识库服务。创建后所有用户可查看 Wiki 与搜索,只有管理员可导入和维护。</p>{isAdmin ? <Button className="mt-5" disabled={actions.initialize.isPending} onClick={() => actions.initialize.mutate("知识梳理")}>创建知识梳理库</Button> : <p className="mt-5 text-sm">请联系管理员初始化。</p>}</div></div></div>;
}
const uploadFiles = async (files: FileList | null) => {
if (!files?.length) return;
try {
await actions.upload.mutateAsync(Array.from(files));
toast.success("文件已安全扫描并导入");
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
} finally {
if (fileRef.current) fileRef.current.value = "";
}
};
const openHistory = async () => {
if (!activePage) return;
setRevisions(await listAssistantWikiRevisions(activePage.slug));
};
return (
<div className="flex size-full min-h-0 flex-col bg-background">
<KnowledgeProductNav />
<header className="flex items-center gap-4 border-b px-6 py-4"><div><h1 className="text-xl font-semibold">{base.data.name}</h1><p className="text-xs text-muted-foreground">全局助手知识库 · 普通用户只读</p></div><Badge variant="outline"><ShieldCheck className="mr-1 size-3" />实体对齐与版本管理</Badge><div className="ml-auto flex gap-2">{isAdmin && <><input ref={fileRef} className="hidden" type="file" multiple onChange={(event) => void uploadFiles(event.target.files)} /><Button variant="outline" onClick={() => fileRef.current?.click()}><FileUp />导入本地文件</Button></>}<Button variant="outline" disabled={!activePage} onClick={() => void openHistory()}><History />页面版本</Button></div></header>
<Tabs defaultValue="wiki" className="flex min-h-0 flex-1 flex-col">
<div className="flex items-center gap-4 border-b px-6"><TabsList className="my-2"><TabsTrigger value="wiki">Wiki({counts.pages})</TabsTrigger><TabsTrigger value="sources">来源({counts.sources})</TabsTrigger><TabsTrigger value="jobs">导入记录({counts.jobs})</TabsTrigger><TabsTrigger value="revisions">版本({revisions.length})</TabsTrigger></TabsList><div className="relative ml-auto w-72"><Search className="absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" /><Input value={query} onChange={(event) => setQuery(event.target.value)} placeholder="搜索标题、实体别名和正文" className="pl-8" /></div></div>
<TabsContent value="wiki" className="mt-0 grid min-h-0 flex-1 grid-cols-[300px_1fr]"><ScrollArea className="border-r"><div className="space-y-1 p-3">{pages.data?.map((page) => <button key={page.id} type="button" onClick={() => setSelected(page)} className={`w-full rounded-md px-3 py-2 text-left ${activePage?.id === page.id ? "bg-primary/10 text-primary" : "hover:bg-muted"}`}><div className="truncate text-sm font-medium">{page.title}</div><div className="mt-1 flex items-center gap-2 text-[11px] text-muted-foreground"><span>{page.page_type}</span><span>{new Date(page.updated_at).toLocaleString()}</span></div></button>)}</div></ScrollArea><ScrollArea><article className="mx-auto max-w-5xl p-8">{activePage ? <><h1 className="mb-1 text-2xl font-semibold">{activePage.title}</h1><p className="mb-6 text-xs text-muted-foreground">{activePage.slug}</p><MarkdownContent content={activePage.content_markdown} isLoading={false} rehypePlugins={[]} /></> : <p className="text-sm text-muted-foreground">暂无 Wiki 页面。</p>}</article></ScrollArea></TabsContent>
<TabsContent value="sources" className="mt-0 min-h-0 flex-1 overflow-auto p-6"><div className="grid gap-3 md:grid-cols-2 xl:grid-cols-3">{sources.data?.map((source) => <div key={source.id} className="rounded-lg border bg-card p-4"><div className="flex items-center gap-2"><Database className="size-4" /><span className="font-medium">{source.source_name}</span><Badge variant="outline" className="ml-auto">{source.source_type}</Badge></div><p className="mt-3 text-xs text-muted-foreground">业务键:{source.source_key}</p>{isAdmin && <div className="mt-4 flex justify-end gap-2"><Button size="sm" variant="outline" disabled={source.source_type === "local_file" || actions.reimportSource.isPending} onClick={() => actions.reimportSource.mutate(source.id, { onSuccess: () => toast.success("已发起重新导入"), onError: (error) => toast.error(error.message) })}><RefreshCw />重新导入</Button><Button size="sm" variant="destructive" disabled={actions.deleteSource.isPending} onClick={() => { if (window.confirm(`确认删除来源“${source.source_name}”的贡献?仍被其他来源引用的知识会保留。`)) actions.deleteSource.mutate(source.id, { onSuccess: () => toast.success("来源贡献已删除"), onError: (error) => toast.error(error.message) }); }}><Trash2 />删除来源</Button></div>}</div>)}</div></TabsContent>
<TabsContent value="jobs" className="mt-0 min-h-0 flex-1 overflow-auto p-6"><div className="space-y-3">{jobs.data?.map((job) => <div key={job.id} className="flex items-center gap-3 rounded-lg border bg-card p-4"><Badge>{job.status}</Badge><span className="text-sm">{job.trigger}</span><span className="text-xs text-muted-foreground">{Object.entries(job.counts || {}).map(([key, value]) => `${key} ${value}`).join(" · ")}</span><span className="ml-auto text-xs">{new Date(job.created_at).toLocaleString()}</span></div>)}</div></TabsContent>
<TabsContent value="revisions" className="mt-0 min-h-0 flex-1 overflow-auto p-6"><div className="space-y-3">{revisions.map((revision) => <div key={revision.id} className="flex items-center gap-3 rounded-lg border bg-card p-4"><Badge variant="outline">v{revision.revision_no}</Badge><span className="text-sm">{revision.change_type}</span><span className="text-xs text-muted-foreground">{revision.summary}</span><span className="ml-auto text-xs">{new Date(revision.created_at).toLocaleString()}</span>{isAdmin && activePage && revision.id !== activePage.current_revision_id && <Button size="sm" variant="outline" onClick={() => actions.rollback.mutate({ slug: activePage.slug, revisionId: revision.id })}>回滚到此版本</Button>}</div>)}{!revisions.length && <p className="text-sm text-muted-foreground">请先在 Wiki 页选择页面并点击“页面版本”。</p>}</div></TabsContent>
</Tabs>
</div>
);
}
@@ -3,6 +3,7 @@ import { useState } from "react";
import { Navigate, useParams, useSearchParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import { KnowledgeProductNav, useKnowledgeBaseProductVersion } from "@/components/workspace/knowledge-product-nav";
import {
Dialog,
DialogContent,
@@ -70,6 +71,7 @@ export default function LlmWikiKnowledgePage({
const { noteId = "" } = useParams();
const [searchParams] = useSearchParams();
const runtime = useLlmWikiRuntime();
const productVersion = useKnowledgeBaseProductVersion();
const wikiSlug = searchParams.get("wikiSlug")?.trim() || "";
if (mode === "detail" && noteId && wikiSlug) {
@@ -80,11 +82,15 @@ export default function LlmWikiKnowledgePage({
return <RuntimeLoading />;
}
if (mode === "list" && productVersion.data === "v2") {
return <Navigate to="/page/workspace/knowledge/assistant" replace />;
}
if (runtime.isError || runtime.data?.provider !== "weknora") {
if (mode === "graph") {
return <KnowledgeGraphPage />;
}
return <LegacyKnowledgePage />;
return mode === "list" ? <div className="flex size-full min-h-0 flex-col"><KnowledgeProductNav /><div className="min-h-0 flex-1"><LegacyKnowledgePage /></div></div> : <LegacyKnowledgePage />;
}
if (mode === "graph") {
@@ -95,5 +101,5 @@ export default function LlmWikiKnowledgePage({
return <WeKnoraKnowledgeDetailPage />;
}
return <WeKnoraKnowledgeWorkspace />;
return <div className="flex size-full min-h-0 flex-col"><KnowledgeProductNav /><div className="min-h-0 flex-1"><WeKnoraKnowledgeWorkspace /></div></div>;
}
@@ -40,6 +40,7 @@ import AdminSkillsPage from './AdminSkillsPage';
import ScheduledTaskRunDetailPage from './ScheduledTaskRunDetailPage';
import ScheduledTasksPage from './ScheduledTasksPage';
import LlmWikiKnowledgePage from './LlmWikiKnowledgePage';
import AssistantKnowledgePage from './AssistantKnowledgePage';
import WeKnoraWikiManagementPage from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage';
import WeKnoraWikiPageView from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView';
import MarkdownDocxEditorPage from './MarkdownDocxEditorPage';
@@ -689,6 +690,14 @@ export default function WorkspaceRoutes({
</WorkspaceLayout>
}
/>
<Route
path="knowledge/assistant"
element={
<WorkspaceLayout {...layoutProps}>
<AssistantKnowledgePage />
</WorkspaceLayout>
}
/>
<Route
path="knowledge/:noteId/wiki/view"
element={
@@ -0,0 +1,177 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
const BASE = () => `${getBackendBaseURL()}/api/assistant-knowledge`;
export interface AssistantKnowledgeBase {
id: string;
name: string;
status: string;
created_at: string;
updated_at: string;
}
export interface AssistantWikiPage {
id: string;
slug: string;
title: string;
page_type: string;
content_markdown: string;
summary: string;
current_revision_id?: string | null;
updated_at: string;
}
export interface AssistantWikiRevision {
id: string;
page_id: string;
revision_no: number;
summary: string;
content_markdown: string;
change_type: string;
created_by?: string | null;
created_at: string;
}
export interface AssistantKnowledgeSource {
id: string;
source_type: "weknora" | "skill" | "local_file";
source_name: string;
source_key: string;
status: string;
updated_at: string;
}
export interface AssistantImportJob {
id: string;
source_id: string;
trigger: string;
status: string;
phase: string;
counts: Record<string, number>;
created_at: string;
}
async function readJson<T>(response: Response): Promise<T> {
if (!response.ok) {
const data = await response.json().catch(() => null);
const detail = data && typeof data === "object" && "detail" in data
? (data as { detail: unknown }).detail
: `请求失败 (${response.status})`;
throw new Error(typeof detail === "string" ? detail : JSON.stringify(detail));
}
return (await response.json()) as T;
}
export async function getAssistantKnowledgeBase() {
const data = await readJson<{ base: AssistantKnowledgeBase | null }>(await apiFetch(`${BASE()}/base`));
return data.base;
}
export async function initializeAssistantKnowledgeBase(name = "知识梳理") {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/base/initialize`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name }),
}),
);
return data.base;
}
export async function listAssistantWikiPages(q = "") {
const params = new URLSearchParams();
if (q.trim()) params.set("q", q.trim());
return (await readJson<{ pages: AssistantWikiPage[] }>(await apiFetch(`${BASE()}/wiki-pages?${params}`))).pages;
}
export async function listAssistantKnowledgeSources() {
return (await readJson<{ sources: AssistantKnowledgeSource[] }>(await apiFetch(`${BASE()}/sources`))).sources;
}
export async function listAssistantImportJobs() {
return (await readJson<{ jobs: AssistantImportJob[] }>(await apiFetch(`${BASE()}/import-jobs`))).jobs;
}
export async function listAssistantWikiRevisions(slug: string) {
return (await readJson<{ revisions: AssistantWikiRevision[] }>(
await apiFetch(`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions`),
)).revisions;
}
export async function rollbackAssistantWikiRevision(slug: string, revisionId: string) {
return readJson<AssistantWikiRevision>(
await apiFetch(
`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions/${encodeURIComponent(revisionId)}/rollback`,
{ method: "POST" },
),
);
}
export async function uploadAssistantKnowledgeFiles(files: File[]) {
const form = new FormData();
files.forEach((file) => form.append("files", file));
return readJson<{ batch_id: string; jobs: AssistantImportJob[] }>(
await apiFetch(`${BASE()}/imports/files`, { method: "POST", body: form }),
);
}
export async function exportLlmWikiToAssistant(mappingId: string) {
return readJson<AssistantImportJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-to-assistant`,
{ method: "POST" },
),
);
}
export async function reimportAssistantKnowledgeSource(sourceId: string) {
return readJson<AssistantImportJob>(
await apiFetch(`${BASE()}/sources/${encodeURIComponent(sourceId)}/reimport`, {
method: "POST",
}),
);
}
export async function deleteAssistantKnowledgeSource(sourceId: string) {
const response = await apiFetch(`${BASE()}/sources/${encodeURIComponent(sourceId)}`, {
method: "DELETE",
});
if (!response.ok) await readJson(response);
}
const KEY = "assistant-knowledge";
export function useAssistantKnowledgeBase() {
return useQuery({ queryKey: [KEY, "base"], queryFn: getAssistantKnowledgeBase });
}
export function useAssistantWikiPages(query = "") {
return useQuery({ queryKey: [KEY, "pages", query], queryFn: () => listAssistantWikiPages(query) });
}
export function useAssistantKnowledgeSources() {
return useQuery({ queryKey: [KEY, "sources"], queryFn: listAssistantKnowledgeSources });
}
export function useAssistantImportJobs() {
return useQuery({ queryKey: [KEY, "jobs"], queryFn: listAssistantImportJobs });
}
export function useAssistantKnowledgeActions() {
const client = useQueryClient();
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
return {
initialize: useMutation({ mutationFn: initializeAssistantKnowledgeBase, onSuccess: invalidate }),
upload: useMutation({ mutationFn: uploadAssistantKnowledgeFiles, onSuccess: invalidate }),
reimportSource: useMutation({ mutationFn: reimportAssistantKnowledgeSource, onSuccess: invalidate }),
deleteSource: useMutation({ mutationFn: deleteAssistantKnowledgeSource, onSuccess: invalidate }),
rollback: useMutation({
mutationFn: ({ slug, revisionId }: { slug: string; revisionId: string }) =>
rollbackAssistantWikiRevision(slug, revisionId),
onSuccess: invalidate,
}),
};
}
@@ -0,0 +1,277 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
const BASE = () => `${getBackendBaseURL()}/api/skill-knowledge`;
export type SkillKnowledgeTargetType = "weknora" | "assistant";
export type SkillKnowledgeTargetMode =
| "wiki"
| "document"
| "faq"
| "conversation_archive"
| "readonly_sidecar"
| "assistant";
export interface SkillKnowledgeTarget {
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
target_name: string;
remote_business_key?: string | null;
readonly?: boolean;
}
export interface SkillKnowledgeBinding {
id: string;
skill_name: string;
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
target_name: string;
remote_business_key?: string | null;
remote_refs?: Array<{
provider: string;
remote_kind: string;
remote_id: string;
slug?: string;
}>;
status: string;
active: boolean;
auto_sync_enabled: boolean;
synced_digest?: string | null;
last_synced_at?: string | null;
last_error_message?: string | null;
}
export interface SkillKnowledgeSyncItem {
id: string;
job_id: string;
binding_id: string;
skill_name: string;
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
status: string;
phase: string;
progress: number;
attempts: number;
last_error_message?: string | null;
}
export interface SkillKnowledgeJob {
id: string;
status: string;
total_count: number;
success_count: number;
partial_failed_count: number;
failed_count: number;
review_mode: "required" | "auto_high_confidence" | "off";
created_at: string;
completed_at?: string | null;
items?: SkillKnowledgeSyncItem[];
}
export interface SkillKnowledgeReview {
id: string;
job_id: string;
item_id: string;
binding_id: string;
object_type: "entity" | "relation" | "wiki" | "alignment";
object_key: string;
original: Record<string, unknown>;
edited?: Record<string, unknown> | null;
status: string;
reason?: string | null;
}
export interface SkillKnowledgeAutoSyncSettings {
watch_enabled: boolean;
schedule_enabled: boolean;
schedule_cron: string;
debounce_seconds: number;
write_remote_enabled: boolean;
review_mode: "required" | "auto_high_confidence" | "off";
}
async function readJson<T>(response: Response): Promise<T> {
if (!response.ok) {
const data = await response.json().catch(() => null);
const detail =
data && typeof data === "object" && "detail" in data
? JSON.stringify((data as { detail: unknown }).detail)
: `请求失败 (${response.status})`;
throw new Error(detail.replace(/^"|"$/g, ""));
}
return (await response.json()) as T;
}
export async function listSkillKnowledgeBindings(
skillNames?: string[],
): Promise<SkillKnowledgeBinding[]> {
const params = new URLSearchParams();
for (const name of skillNames ?? []) params.append("skill_names", name);
const response = await apiFetch(`${BASE()}/bindings?${params.toString()}`);
return (await readJson<{ bindings: SkillKnowledgeBinding[] }>(response)).bindings;
}
export async function createSkillKnowledgeJob(input: {
skill_names: string[];
targets: SkillKnowledgeTarget[];
force?: boolean;
review_mode?: "required" | "auto_high_confidence" | "off";
}): Promise<SkillKnowledgeJob> {
return readJson(
await apiFetch(`${BASE()}/sync-jobs`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
export async function listSkillKnowledgeJobs(): Promise<SkillKnowledgeJob[]> {
const response = await apiFetch(`${BASE()}/sync-jobs`);
return (await readJson<{ jobs: SkillKnowledgeJob[] }>(response)).jobs;
}
export async function getSkillKnowledgeJob(id: string): Promise<SkillKnowledgeJob> {
return readJson(await apiFetch(`${BASE()}/sync-jobs/${encodeURIComponent(id)}`));
}
export async function listSkillKnowledgeReviews(): Promise<SkillKnowledgeReview[]> {
const response = await apiFetch(`${BASE()}/reviews?status=pending`);
return (await readJson<{ reviews: SkillKnowledgeReview[] }>(response)).reviews;
}
export async function reviewSkillKnowledge(input: {
review_ids: string[];
action: "approve" | "reject";
edits?: Record<string, Record<string, unknown>>;
reason?: string;
}) {
return readJson<{ updated: number; resumed_item_ids: string[] }>(
await apiFetch(`${BASE()}/reviews/actions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
export async function retrySkillKnowledgeItem(itemId: string) {
return readJson<SkillKnowledgeSyncItem>(
await apiFetch(`${BASE()}/sync-items/${encodeURIComponent(itemId)}/retry`, {
method: "POST",
}),
);
}
export async function detachSkillKnowledgeBinding(
bindingId: string,
deleteContributions: boolean,
) {
return readJson<SkillKnowledgeBinding>(
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/detach`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ delete_contributions: deleteContributions }),
}),
);
}
export async function resyncSkillKnowledgeBinding(bindingId: string) {
return readJson<SkillKnowledgeJob>(
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/resync`, {
method: "POST",
}),
);
}
export async function setSkillKnowledgeBindingAutoSync(
bindingId: string,
enabled: boolean,
) {
return readJson<SkillKnowledgeBinding>(
await apiFetch(
`${BASE()}/bindings/${encodeURIComponent(bindingId)}/${enabled ? "enable" : "disable"}-auto-sync`,
{ method: "POST" },
),
);
}
export async function getSkillKnowledgeAutoSyncSettings() {
return readJson<SkillKnowledgeAutoSyncSettings>(
await apiFetch(`${BASE()}/auto-sync-settings`),
);
}
export async function updateSkillKnowledgeAutoSyncSettings(
input: Partial<SkillKnowledgeAutoSyncSettings>,
) {
return readJson<SkillKnowledgeAutoSyncSettings>(
await apiFetch(`${BASE()}/auto-sync-settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
const KEY = "skill-knowledge";
export function useSkillKnowledgeBindings(skillNames?: string[]) {
return useQuery({
queryKey: [KEY, "bindings", ...(skillNames ?? [])],
queryFn: () => listSkillKnowledgeBindings(skillNames),
refetchOnWindowFocus: false,
});
}
export function useSkillKnowledgeJobs() {
return useQuery({
queryKey: [KEY, "jobs"],
queryFn: listSkillKnowledgeJobs,
refetchInterval: (query) =>
(query.state.data ?? []).some((job) => ["queued", "running"].includes(job.status))
? 1500
: false,
});
}
export function useSkillKnowledgeReviews() {
return useQuery({ queryKey: [KEY, "reviews"], queryFn: listSkillKnowledgeReviews });
}
export function useSkillKnowledgeAutoSyncSettings() {
return useQuery({
queryKey: [KEY, "auto-sync-settings"],
queryFn: getSkillKnowledgeAutoSyncSettings,
});
}
export function useSkillKnowledgeActions() {
const client = useQueryClient();
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
return {
createJob: useMutation({ mutationFn: createSkillKnowledgeJob, onSuccess: invalidate }),
review: useMutation({ mutationFn: reviewSkillKnowledge, onSuccess: invalidate }),
retry: useMutation({ mutationFn: retrySkillKnowledgeItem, onSuccess: invalidate }),
resync: useMutation({ mutationFn: resyncSkillKnowledgeBinding, onSuccess: invalidate }),
setAutoSync: useMutation({
mutationFn: ({ id, enabled }: { id: string; enabled: boolean }) =>
setSkillKnowledgeBindingAutoSync(id, enabled),
onSuccess: invalidate,
}),
updateAutoSyncSettings: useMutation({
mutationFn: updateSkillKnowledgeAutoSyncSettings,
onSuccess: invalidate,
}),
detach: useMutation({
mutationFn: ({ id, deleteContributions }: { id: string; deleteContributions: boolean }) =>
detachSkillKnowledgeBinding(id, deleteContributions),
onSuccess: invalidate,
}),
};
}
@@ -1,4 +1,4 @@
import { AlertCircle, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { AlertCircle, Database, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { Navigate, useNavigate, useParams, useSearchParams } from "react-router-dom";
import { toast } from "sonner";
@@ -14,6 +14,7 @@ import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { MarkdownCodeEditor } from "@/components/wiki/MarkdownCodeEditor";
import { getBackendBaseURL } from "@/core/config";
import { getStoredAuth } from "@/core/auth";
import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
import {
createLlmWikiWeKnoraEmbedSession,
@@ -23,6 +24,7 @@ import {
useLlmWikiWikiPage,
useLlmWikiRuntime,
} from "@/strategy-components/api/llmwiki";
import { exportLlmWikiToAssistant } from "@/strategy-components/api/assistant-knowledge";
const LIST_PATH = "/page/workspace/knowledge";
@@ -104,6 +106,7 @@ export function WeKnoraKnowledgeDetailPage() {
const [frameSrc, setFrameSrc] = useState("");
const [frameVersion, setFrameVersion] = useState(0);
const [frameError, setFrameError] = useState<string | null>(null);
const [assistantImporting, setAssistantImporting] = useState(false);
const runtime = useLlmWikiRuntime();
const details = useLlmWikiKnowledgeBase(knowledgeBaseId || null, !!knowledgeBaseId);
@@ -255,6 +258,23 @@ export function WeKnoraKnowledgeDetailPage() {
)}
</div>
<div className="flex shrink-0 items-center gap-2">
{getStoredAuth()?.system_role === "admin" && (
<Button
size="sm"
variant="outline"
disabled={assistantImporting}
onClick={() => {
setAssistantImporting(true);
void exportLlmWikiToAssistant(knowledgeBaseId)
.then(() => toast.success("已导入到助手知识库“知识梳理”"))
.catch((error: unknown) => toast.error(error instanceof Error ? error.message : String(error)))
.finally(() => setAssistantImporting(false));
}}
>
{assistantImporting ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Database className="mr-2 size-4" />}
导入到助手知识库
</Button>
)}
<Button
size="sm"
variant="outline"
@@ -1619,3 +1619,27 @@ write, log the exception, complete the rewrite with
`versionSnapshotFailed: true`, and do not emit a user-facing failure or restore
the original. The sandbox uses that signal only to enable its ordinary manual
save fallback.
## Skill/assistant knowledge invariants
Skill distillation is implemented in `deerflow.skill_knowledge` and
`deerflow.persistence.skill_knowledge`; the single native assistant base is in
`deerflow.assistant_knowledge` and
`deerflow.persistence.assistant_knowledge`. Gateway composition, remote adapters,
the lease dispatcher, watcher/scheduler, and APIs remain in `app.gateway` so the
harness-to-app import firewall is preserved.
Never execute a skill or read/upload/summarize its code and script files.
Knowledge fingerprints exclude those files and Markdown code fences. All
resyncs must stage and validate the knowledge digest before activation; the old
snapshot remains authoritative on failure. A binding may have only one active
item (`active_dedupe_key` plus atomic lease claims). Detach/delete operates on
source contributions, not canonical rows, and shared entities/relations are
retired only when their final contribution is gone.
There is exactly one global assistant base (`知识梳理`). Ordinary users have
read/search access and administrators own all mutations. WeKnora direct graph
writes require `llmwiki.weknora.graph.direct_write_enabled`, backend-only
credentials, and the fixed parameterized templates in
`deerflow.skill_knowledge.graph_writer`; never accept Cypher from an API, model,
skill, or frontend.
@@ -953,3 +953,30 @@ The final rewrite commit uses a deliberately short sibling temporary filename,
which keeps long Windows artifact paths below the path-length limit. It also
recreates a missing `outputs` parent before retrying, so transient
sandbox-directory recreation cannot discard a completed rewrite.
## Skill knowledge distillation and assistant knowledge
The Gateway now exposes an administrator-only skill distillation pipeline at
`/api/skill-knowledge`. A matrix job can send up to 200 skills to multiple
WeKnora or global assistant-knowledge targets. The scanner never reads or
uploads code/script files, strips Markdown code fences, blocks credential-like
files and content, safely inspects ZIP members, and converts supported
Office/PDF/spreadsheet files off the event loop. Jobs use database claims and
renewable leases, stage snapshots before activation, retain the prior active
version on failure, support per-item retry and entity/relation review, and keep
per-binding contribution records for safe detach/delete.
Bindings may opt into debounced file watching and cron synchronization through
`/api/skill-knowledge/auto-sync-settings`. Only non-code knowledge digests can
mark a binding stale. WeKnora Wiki, document, FAQ, conversation-archive and
read-only sidecar targets have separate projections. Optional Neo4j direct
writes are disabled by default under `llmwiki.weknora.graph`; when enabled they
use only fixed parameterized templates and Neo4j's transactional HTTP endpoint.
`/api/assistant-knowledge` owns exactly one global DeerFlow-native base named
`知识梳理`. Reads and search are available to authenticated users; initialization,
local-file import, WeKnora export/import, source reimport/delete, and revision
rollback are admin-only. Imports align canonical entities and relations while
retaining source contributions, publish source/entity/relation Wiki revisions,
and remove canonical knowledge only after its final contribution disappears.
The database schema is introduced by Alembic revision `20260901_01`.
@@ -30,9 +30,10 @@ from app.gateway.routers import (
ai_writing,
admin_active_runs,
admin_users,
agents,
artifacts,
assistants_compat,
agents,
artifacts,
assistant_knowledge,
assistants_compat,
auth,
browser_context,
business_mapping,
@@ -83,9 +84,10 @@ from app.gateway.routers import (
roundtable_task_drafts,
runs,
scheduled_tasks,
sensitive_words,
skill_addresses,
skills,
sensitive_words,
skill_addresses,
skill_knowledge,
skills,
suggestions,
system_settings,
tags,
@@ -1503,8 +1505,12 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Memory API is mounted at /api/memory
app.include_router(memory.router)
# Skills API is mounted at /api/skills
app.include_router(skills.router)
# Skills API is mounted at /api/skills
app.include_router(skills.router)
# Admin skill distillation matrix + global assistant knowledge base.
app.include_router(skill_knowledge.router)
app.include_router(assistant_knowledge.router)
app.include_router(assistant_knowledge.export_router)
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
app.include_router(artifacts.router)
@@ -315,6 +315,33 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.llmwiki_store = make_llmwiki_store(sf)
register_llmwiki_store(app.state.llmwiki_store)
# Skill distillation and the single global assistant knowledge base.
# Both repositories live in ``deerflow.*``; only this composition layer
# wires them to Gateway executors and remote adapters.
from deerflow.persistence.assistant_knowledge import make_assistant_knowledge_store
from deerflow.persistence.skill_knowledge import make_skill_knowledge_store
app.state.assistant_knowledge_store = make_assistant_knowledge_store(sf)
app.state.skill_knowledge_store = make_skill_knowledge_store(sf)
if app.state.skill_knowledge_store is not None and app.state.assistant_knowledge_store is not None:
from app.gateway.skill_knowledge_job_dispatcher import SkillKnowledgeJobDispatcher
from app.gateway.skill_knowledge_job_executor import SkillKnowledgeJobExecutor
app.state.skill_knowledge_executor = SkillKnowledgeJobExecutor(app)
app.state.skill_knowledge_dispatcher = SkillKnowledgeJobDispatcher(
app.state.skill_knowledge_store,
app.state.skill_knowledge_executor,
)
app.state.skill_knowledge_dispatcher.start()
from app.gateway.skill_knowledge_auto_sync import SkillKnowledgeAutoSyncService
app.state.skill_knowledge_auto_sync = SkillKnowledgeAutoSyncService(app)
app.state.skill_knowledge_auto_sync.start()
else:
app.state.skill_knowledge_executor = None
app.state.skill_knowledge_dispatcher = None
app.state.skill_knowledge_auto_sync = None
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
# migrations and status inspection work while feature flags are off;
# network/search services only exist when explicitly enabled.
@@ -580,10 +607,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)
if dispatcher is not None:
await dispatcher.stop()
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
if dr_dispatcher is not None:
await dr_dispatcher.stop()
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
if dr_dispatcher is not None:
await dr_dispatcher.stop()
skill_knowledge_auto_sync = getattr(app.state, "skill_knowledge_auto_sync", None)
if skill_knowledge_auto_sync is not None:
await skill_knowledge_auto_sync.close()
skill_knowledge_dispatcher = getattr(app.state, "skill_knowledge_dispatcher", None)
if skill_knowledge_dispatcher is not None:
await skill_knowledge_dispatcher.close()
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
if queue is not None:
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
@@ -0,0 +1,338 @@
"""Read-mostly global assistant knowledge API and admin import actions."""
from __future__ import annotations
import asyncio
import tempfile
from pathlib import Path
from typing import Any
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from deerflow.assistant_knowledge import export_package_from_skill_ir
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
router = APIRouter(prefix="/api/assistant-knowledge", tags=["assistant-knowledge"])
export_router = APIRouter(tags=["assistant-knowledge"])
async def _actor(request: Request, *, admin: bool = False) -> str | None:
user = await get_optional_user_from_request(request)
if user is None:
return None
if admin and getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="该操作仅管理员可用")
return str(user.id)
def _store(request: Request):
store = getattr(request.app.state, "assistant_knowledge_store", None)
if store is None:
raise HTTPException(status_code=503, detail="助手知识库存储不可用")
return store
class InitializeRequest(BaseModel):
name: str = Field(default="知识梳理", min_length=1, max_length=255)
class SkillImportRequest(BaseModel):
skill_names: list[str] = Field(min_length=1, max_length=200)
force: bool = False
@router.get("/base")
async def get_base(request: Request) -> dict[str, Any]:
await _actor(request)
return {"base": await _store(request).get_base()}
@router.post("/base/initialize", status_code=201)
async def initialize(request: Request, body: InitializeRequest) -> dict[str, Any]:
actor = await _actor(request, admin=True)
return {"base": await _store(request).initialize(created_by=actor, name=body.name)}
@router.get("/wiki-pages")
async def list_wiki_pages(request: Request, q: str | None = Query(default=None), page_type: str | None = Query(default=None), limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
await _actor(request)
pages = await _store(request).list_pages(query=q, page_type=page_type, limit=limit)
return {"pages": pages, "total": len(pages)}
@router.get("/wiki-pages/{slug:path}/revisions/compare")
async def compare_revisions(request: Request, slug: str, left: str = Query(...), right: str = Query(...)) -> dict[str, Any]:
await _actor(request)
result = await _store(request).compare_revisions(left, right)
if result is None:
raise HTTPException(status_code=404, detail="版本不存在或不属于同一页面")
return result
@router.get("/wiki-pages/{slug:path}/revisions")
async def list_revisions(request: Request, slug: str) -> dict[str, Any]:
await _actor(request)
return {"revisions": await _store(request).list_revisions(slug)}
@router.post("/wiki-pages/{slug:path}/revisions/{revision_id}/rollback")
async def rollback_revision(request: Request, slug: str, revision_id: str) -> dict[str, Any]:
actor = await _actor(request, admin=True)
row = await _store(request).rollback(slug=slug, revision_id=revision_id, actor=actor)
if row is None:
raise HTTPException(status_code=404, detail="页面或版本不存在")
return row
@router.get("/wiki-pages/{slug:path}")
async def get_wiki_page(request: Request, slug: str) -> dict[str, Any]:
await _actor(request)
row = await _store(request).get_page(slug)
if row is None:
raise HTTPException(status_code=404, detail="Wiki 页面不存在")
return row
@router.get("/search")
async def search(request: Request, q: str = Query(..., min_length=1), page_type: str | None = Query(default=None)) -> dict[str, Any]:
await _actor(request)
rows = await _store(request).list_pages(query=q, page_type=page_type, limit=100)
return {"results": rows, "total": len(rows)}
@router.get("/sources")
async def list_sources(request: Request) -> dict[str, Any]:
await _actor(request)
return {"sources": await _store(request).list_sources()}
@router.get("/entities/{entity_id}")
async def get_entity(request: Request, entity_id: str) -> dict[str, Any]:
await _actor(request)
row = await _store(request).get_entity(entity_id)
if row is None:
raise HTTPException(status_code=404, detail="实体不存在")
return row
@router.get("/relations/{relation_id}")
async def get_relation(request: Request, relation_id: str) -> dict[str, Any]:
await _actor(request)
row = await _store(request).get_relation(relation_id)
if row is None:
raise HTTPException(status_code=404, detail="关系不存在")
return row
@router.delete("/sources/{source_id}", status_code=204)
async def delete_source(request: Request, source_id: str) -> None:
await _actor(request, admin=True)
if not await _store(request).delete_source(source_id):
raise HTTPException(status_code=404, detail="来源不存在")
@router.get("/import-jobs")
async def list_import_jobs(request: Request, limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
await _actor(request)
return {"jobs": await _store(request).list_jobs(limit)}
@router.get("/import-jobs/{job_id}")
async def get_import_job(request: Request, job_id: str) -> dict[str, Any]:
await _actor(request)
row = await _store(request).get_job(job_id)
if row is None:
raise HTTPException(status_code=404, detail="导入任务不存在")
return row
@router.post("/imports/skills", status_code=202)
async def import_skills(request: Request, body: SkillImportRequest) -> dict[str, Any]:
actor = await _actor(request, admin=True)
base = await _store(request).initialize(created_by=actor)
target = {"target_type": "assistant", "target_mode": "assistant", "target_id": base["id"], "target_name": base["name"]}
job = await request.app.state.skill_knowledge_store.create_job(
skill_names=body.skill_names,
targets=[target],
created_by=actor,
force=body.force,
review_mode="auto_high_confidence",
trigger="skill_direct",
)
request.app.state.skill_knowledge_dispatcher.nudge()
return job
@router.post("/imports/files", status_code=201)
async def import_files(request: Request, files: list[UploadFile] = File(...)) -> dict[str, Any]:
actor = await _actor(request, admin=True)
if not files or len(files) > 50:
raise HTTPException(status_code=422, detail="一次可导入 1-50 个文件")
batch_id = str(uuid4())
jobs: list[dict[str, Any]] = []
with tempfile.TemporaryDirectory(prefix="assistant-knowledge-") as tmp:
root = Path(tmp)
for upload in files:
filename = Path(upload.filename or "upload.bin").name
content = await upload.read()
if len(content) > 50 * 1024 * 1024:
raise HTTPException(status_code=413, detail=f"文件过大:{filename}")
path = root / filename
await asyncio.to_thread(path.write_bytes, content)
scan = await asyncio.to_thread(scan_skill, f"local-{batch_id}", root)
scan = await enrich_converted_documents(scan)
if scan.blocked_files:
raise HTTPException(status_code=422, detail={"message": "文件命中敏感信息阻断规则", "blocked_files": scan.blocked_files})
ir = await asyncio.to_thread(build_skill_ir, scan, display_name="本地文件导入", description="管理员本地文件导入")
package = export_package_from_skill_ir(ir)
job = await _store(request).import_package(
package=package,
source_type="local_file",
source_key=batch_id,
source_name="、".join(Path(file.filename or "upload").name for file in files)[:255],
trigger="manual_file_upload",
created_by=actor,
upload_batch_id=batch_id,
)
jobs.append(job)
return {"batch_id": batch_id, "jobs": jobs}
async def _export_weknora_package(client: Any, remote_id: str, source: dict[str, Any]) -> dict[str, Any]:
pages: list[dict[str, Any]] = []
page_no = 1
while True:
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
batch = page.get("pages") or []
pages.extend(batch)
if len(pages) >= int(page.get("total") or len(pages)) or not batch:
break
page_no += 1
documents: list[dict[str, Any]] = []
chunks: list[dict[str, Any]] = []
page_no = 1
while True:
page = await client.list_documents(remote_id, page=page_no, page_size=100)
batch = page.get("items") or []
documents.extend(batch)
for document in batch:
document_id = str(document.get("id") or document.get("knowledge_id") or "")
if not document_id:
continue
chunk_page = 1
while True:
values = await client.list_chunks(document_id, page=chunk_page, page_size=100)
chunk_batch = values.get("items") or []
chunks.extend([{**row, "source_document_id": document_id} for row in chunk_batch])
if len(chunk_batch) < 100:
break
chunk_page += 1
if len(documents) >= int(page.get("total") or len(documents)) or not batch:
break
page_no += 1
try:
graph = await client.get_wiki_graph(remote_id, limit=500)
except WeKnoraError:
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
entities = [{"id": row.get("id"), "name": row.get("title") or row.get("name"), "type": row.get("type") or "concept", "confidence": 1.0} for row in graph.get("nodes") or [] if row.get("id") and (row.get("title") or row.get("name"))]
relations = [
{"source_entity_id": row.get("source"), "target_entity_id": row.get("target"), "predicate": row.get("predicate") or "references", "confidence": 1.0, "evidence": [{"source": "weknora_graph"}]} for row in graph.get("edges") or []
]
return {"source": source, "wiki_pages": pages, "documents": documents, "chunks": chunks, "entities": entities, "relations": relations, "graph": graph}
@export_router.post("/api/llmwiki/knowledge-bases/{mapping_id}/export-to-assistant", status_code=202)
async def export_to_assistant(request: Request, mapping_id: str) -> dict[str, Any]:
actor = await _actor(request, admin=True)
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
if not runtime.weknora_enabled:
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=False, is_admin=True)
if mapping is None:
raise HTTPException(status_code=404, detail="普通知识库不存在")
client = build_weknora_client(runtime)
package = await _export_weknora_package(
client,
str(mapping["weknora_id"]),
{
"type": "weknora",
"knowledge_base_id": mapping_id,
"remote_knowledge_base_id": mapping["weknora_id"],
"knowledge_base_name": mapping["name"],
},
)
return await _store(request).import_package(
package=package,
source_type="weknora",
source_key=mapping_id,
source_name=str(mapping["name"]),
trigger="manual_from_weknora",
created_by=actor,
mapping_id=mapping_id,
)
@router.post("/sources/{source_id}/reimport", status_code=202)
async def reimport_source(request: Request, source_id: str) -> dict[str, Any]:
actor = await _actor(request, admin=True)
store = _store(request)
source = await store.get_source(source_id)
if source is None:
raise HTTPException(status_code=404, detail="来源不存在")
if source["source_type"] == "skill":
base = await store.initialize(created_by=actor)
job = await request.app.state.skill_knowledge_store.create_job(
skill_names=[str(source.get("skill_name") or source["source_key"])],
targets=[
{
"target_type": "assistant",
"target_mode": "assistant",
"target_id": base["id"],
"target_name": base["name"],
}
],
created_by=actor,
force=True,
review_mode="auto_high_confidence",
trigger="assistant_source_reimport",
)
request.app.state.skill_knowledge_dispatcher.nudge()
return job
if source["source_type"] != "weknora":
raise HTTPException(status_code=409, detail="本地文件来源未保留原文件,请重新上传")
mapping_id = str(source.get("knowledge_base_mapping_id") or source["source_key"])
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
if not runtime.weknora_enabled:
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
mapping = await request.app.state.llmwiki_store.get_authorized(
mapping_id,
actor or "system",
write=False,
is_admin=True,
)
if mapping is None:
raise HTTPException(status_code=404, detail="来源普通知识库不存在")
package = await _export_weknora_package(
build_weknora_client(runtime),
str(mapping["weknora_id"]),
{
"type": "weknora",
"knowledge_base_id": mapping_id,
"remote_knowledge_base_id": mapping["weknora_id"],
"knowledge_base_name": mapping["name"],
},
)
return await store.import_package(
package=package,
source_type="weknora",
source_key=mapping_id,
source_name=str(mapping["name"]),
trigger="source_reimport",
created_by=actor,
mapping_id=mapping_id,
)
@@ -0,0 +1,334 @@
"""Administrator APIs for skill knowledge bindings and matrix jobs."""
from __future__ import annotations
import asyncio
from typing import Any, Literal
from croniter import croniter
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field, field_validator
from app.gateway.deps import get_optional_user_from_request
from app.gateway.skill_knowledge_job_executor import _skill_directory
from deerflow.config.system_settings import (
SkillKnowledgeAutoSyncSettings,
load_system_settings,
save_system_settings,
)
from deerflow.integrations.weknora.runtime import (
build_weknora_client,
get_resolved_llmwiki_runtime,
)
from deerflow.skill_knowledge import scan_skill
router = APIRouter(prefix="/api/skill-knowledge", tags=["skill-knowledge"])
async def _require_admin(request: Request) -> str | None:
user = await get_optional_user_from_request(request)
if user is None:
return None
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="技能知识归纳仅管理员可操作")
return str(user.id)
def _store(request: Request):
value = getattr(request.app.state, "skill_knowledge_store", None)
if value is None:
raise HTTPException(status_code=503, detail="技能知识归纳存储不可用")
return value
class SyncTarget(BaseModel):
target_type: Literal["weknora", "assistant"]
target_mode: Literal["wiki", "document", "faq", "conversation_archive", "readonly_sidecar", "assistant"]
target_id: str = Field(min_length=1, max_length=128)
target_name: str = Field(default="", max_length=255)
remote_business_key: str | None = Field(default=None, max_length=512)
readonly: bool = False
class CreateSyncJobRequest(BaseModel):
skill_names: list[str] = Field(min_length=1, max_length=200)
targets: list[SyncTarget] = Field(min_length=1, max_length=50)
force: bool = False
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
@field_validator("skill_names")
@classmethod
def validate_names(cls, values: list[str]) -> list[str]:
result = list(dict.fromkeys(value.strip() for value in values if value.strip()))
if not result:
raise ValueError("At least one skill is required")
return result
class ReviewRequest(BaseModel):
review_ids: list[str] = Field(min_length=1, max_length=500)
action: Literal["approve", "reject"]
edits: dict[str, dict[str, Any]] = Field(default_factory=dict)
reason: str | None = Field(default=None, max_length=1024)
class DetachRequest(BaseModel):
delete_contributions: bool
class AutoSyncSettingsUpdate(BaseModel):
watch_enabled: bool | None = None
schedule_enabled: bool | None = None
schedule_cron: str | None = Field(default=None, min_length=5, max_length=128)
debounce_seconds: int | None = Field(default=None, ge=10, le=86_400)
write_remote_enabled: bool | None = None
review_mode: Literal["required", "auto_high_confidence", "off"] | None = None
@field_validator("schedule_cron")
@classmethod
def validate_cron(cls, value: str | None) -> str | None:
if value is not None and not croniter.is_valid(value):
raise ValueError("无效的 cron 表达式")
return value
@router.get("/bindings")
async def list_bindings(
request: Request,
skill_names: list[str] | None = Query(default=None),
target_id: str | None = Query(default=None),
include_detached: bool = Query(default=False),
) -> dict[str, Any]:
await _require_admin(request)
rows = await _store(request).list_bindings(skill_names=skill_names, target_id=target_id, include_detached=include_detached)
return {"bindings": rows}
@router.get("/source-status")
async def source_status(request: Request, skill_names: list[str] = Query(...)) -> dict[str, Any]:
await _require_admin(request)
bindings = await _store(request).list_bindings(skill_names=skill_names)
by_skill: dict[str, dict[str, Any]] = {}
for name in list(dict.fromkeys(skill_names)):
try:
root = _skill_directory(request.app.state.config, name)
result = await asyncio.to_thread(scan_skill, name, root)
by_skill[name] = {"source_digest": result.source_digest, "counts": result.counts, "blocked_files": result.blocked_files}
except Exception as exc: # noqa: BLE001
by_skill[name] = {"error": str(exc)}
for binding in bindings:
status = by_skill.get(binding["skill_name"])
if status and status.get("source_digest") and status["source_digest"] != binding.get("synced_digest"):
binding["computed_status"] = "stale"
return {"skills": by_skill, "bindings": bindings}
@router.get("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
async def get_auto_sync_settings(request: Request) -> SkillKnowledgeAutoSyncSettings:
await _require_admin(request)
return load_system_settings().skill_knowledge_auto_sync
@router.put("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
async def update_auto_sync_settings(request: Request, body: AutoSyncSettingsUpdate) -> SkillKnowledgeAutoSyncSettings:
await _require_admin(request)
current = load_system_settings()
updated = current.skill_knowledge_auto_sync.model_copy(update=body.model_dump(exclude_unset=True))
saved = save_system_settings(current.model_copy(update={"skill_knowledge_auto_sync": updated}))
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
if service is not None:
service.nudge()
return saved.skill_knowledge_auto_sync
async def _set_binding_auto_sync(request: Request, binding_id: str, *, enabled: bool) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).set_binding_auto_sync(binding_id, enabled=enabled)
if row is None:
raise HTTPException(status_code=404, detail="绑定不存在")
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
if service is not None:
service.nudge()
return row
@router.post("/bindings/{binding_id}/enable-auto-sync")
async def enable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
return await _set_binding_auto_sync(request, binding_id, enabled=True)
@router.post("/bindings/{binding_id}/disable-auto-sync")
async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
return await _set_binding_auto_sync(request, binding_id, enabled=False)
@router.post("/sync-jobs", status_code=202)
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
actor = await _require_admin(request)
skill_store = request.app.state.skill_store
missing = [name for name in body.skill_names if await skill_store.get_any(name) is None]
if missing:
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
targets: list[dict[str, Any]] = []
for target in body.targets:
data = target.model_dump()
if target.target_type == "weknora":
mapping = await request.app.state.llmwiki_store.get_authorized(target.target_id, actor or "system", write=True, is_admin=True)
if mapping is None:
raise HTTPException(status_code=404, detail=f"普通知识库不存在:{target.target_id}")
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
else:
base = await request.app.state.assistant_knowledge_store.get_base()
if base is None:
base = await request.app.state.assistant_knowledge_store.initialize(created_by=actor)
if target.target_id not in {base["id"], "global"}:
raise HTTPException(status_code=404, detail="助手知识库仅允许选择全局知识梳理库")
data["target_id"] = base["id"]
data["target_name"] = base["name"]
data["remote_business_key"] = f"assistant:base:{base['id']}"
targets.append(data)
job = await _store(request).create_job(
skill_names=body.skill_names,
targets=targets,
created_by=actor,
force=body.force,
review_mode=body.review_mode,
)
dispatcher = getattr(request.app.state, "skill_knowledge_dispatcher", None)
if dispatcher is not None:
dispatcher.nudge()
return job
@router.get("/sync-jobs")
async def list_sync_jobs(request: Request, limit: int = Query(default=50, ge=1, le=200)) -> dict[str, Any]:
await _require_admin(request)
return {"jobs": await _store(request).list_jobs(limit=limit)}
@router.get("/sync-jobs/{job_id}")
async def get_sync_job(request: Request, job_id: str) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).get_job(job_id)
if row is None:
raise HTTPException(status_code=404, detail="归纳任务不存在")
return row
@router.post("/sync-jobs/{job_id}/cancel")
async def cancel_sync_job(request: Request, job_id: str) -> dict[str, bool]:
await _require_admin(request)
if not await _store(request).cancel_job(job_id):
raise HTTPException(status_code=404, detail="归纳任务不存在")
return {"canceled": True}
@router.post("/sync-items/{item_id}/retry", status_code=202)
async def retry_sync_item(request: Request, item_id: str) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).requeue_item(item_id)
if row is None:
raise HTTPException(status_code=404, detail="归纳任务项不存在")
request.app.state.skill_knowledge_dispatcher.nudge()
return row
@router.post("/bindings/{binding_id}/resync", status_code=202)
async def resync_binding(request: Request, binding_id: str) -> dict[str, Any]:
actor = await _require_admin(request)
binding = await _store(request).get_binding(binding_id)
if binding is None or not binding.get("active"):
raise HTTPException(status_code=404, detail="绑定不存在")
target = {
"target_type": binding["target_type"],
"target_mode": binding["target_mode"],
"target_id": binding["target_id"],
"target_name": binding["target_name"],
"remote_business_key": binding.get("remote_business_key"),
}
job = await _store(request).create_job(
skill_names=[binding["skill_name"]],
targets=[target],
created_by=actor,
force=True,
review_mode="auto_high_confidence",
)
request.app.state.skill_knowledge_dispatcher.nudge()
return job
@router.post("/bindings/{binding_id}/detach")
async def detach_binding(request: Request, binding_id: str, body: DetachRequest) -> dict[str, Any]:
await _require_admin(request)
store = _store(request)
binding = await store.get_binding(binding_id)
if binding is None:
raise HTTPException(status_code=404, detail="绑定不存在")
if await store.has_active_item(binding_id):
raise HTTPException(status_code=409, detail="该绑定仍有运行中的归纳项,请先取消任务")
cleanup_failed = False
if body.delete_contributions:
try:
if binding["target_type"] == "assistant":
sources = await request.app.state.assistant_knowledge_store.list_sources()
for source in sources:
if source.get("source_type") == "skill" and source.get("source_key") == binding["skill_name"]:
await request.app.state.assistant_knowledge_store.delete_source(source["id"])
elif binding["target_type"] == "weknora":
mapping = await request.app.state.llmwiki_store.get_authorized(
str(binding["target_id"]),
"system",
write=True,
is_admin=True,
)
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
if mapping is None or not runtime.weknora_enabled:
raise RuntimeError("远端普通知识库当前不可写")
client = build_weknora_client(runtime)
remote_kb_id = str(mapping["weknora_id"])
for artifact in await store.list_artifacts(binding_id):
if artifact["kind"] == "wiki_page" and artifact.get("slug"):
await client.delete_wiki_page(remote_kb_id, artifact["slug"])
elif artifact.get("remote_id") and artifact["kind"] in {
"raw_source",
"summary_document",
"faq_projection",
"conversation_archive",
"graph_projection",
}:
await client.delete_document(str(artifact["remote_id"]))
except Exception: # noqa: BLE001
cleanup_failed = True
row = await store.detach_binding(
binding_id,
delete_contributions=body.delete_contributions,
cleanup_failed=cleanup_failed,
)
if row is None:
raise HTTPException(status_code=404, detail="绑定不存在")
return row
@router.get("/reviews")
async def list_reviews(
request: Request,
job_id: str | None = Query(default=None),
item_id: str | None = Query(default=None),
status: str | None = Query(default="pending"),
) -> dict[str, Any]:
await _require_admin(request)
return {"reviews": await _store(request).list_reviews(job_id=job_id, item_id=item_id, status=status)}
@router.post("/reviews/actions")
async def review_actions(request: Request, body: ReviewRequest) -> dict[str, Any]:
actor = await _require_admin(request)
item_ids = await _store(request).review(body.review_ids, action=body.action, actor=actor, edits=body.edits, reason=body.reason)
resumed: list[str] = []
executor = request.app.state.skill_knowledge_executor
for item_id in item_ids:
if await _store(request).pending_review_count(item_id) == 0:
asyncio.create_task(executor.resume_reviewed_item(item_id))
resumed.append(item_id)
return {"updated": len(body.review_ids), "resumed_item_ids": resumed}
@@ -44,6 +44,14 @@ class PromptPrefixResponse(BaseModel):
prompt_prefix: str = Field(..., description="Prefix text prepended to the first user message")
class KnowledgeBaseProductVersionResponse(BaseModel):
version: Literal["v1", "v2"] = "v1"
class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
version: Literal["v1", "v2"]
class PromptPrefixUpdateRequest(BaseModel):
enabled: bool | None = None
switch_label: str | None = None
@@ -167,6 +175,21 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
return _to_response(saved.prompt_prefix)
@router.get("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
async def get_knowledge_base_product_version() -> KnowledgeBaseProductVersionResponse:
return KnowledgeBaseProductVersionResponse(version=load_system_settings().knowledge_base_product_version)
@router.put("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
async def update_knowledge_base_product_version(
body: KnowledgeBaseProductVersionUpdateRequest,
request: Request,
) -> KnowledgeBaseProductVersionResponse:
await _require_admin(request)
saved = save_system_settings(load_system_settings().model_copy(update={"knowledge_base_product_version": body.version}))
return KnowledgeBaseProductVersionResponse(version=saved.knowledge_base_product_version)
@router.get("/plugin-management", response_model=PluginManagementResponse)
async def get_plugin_management() -> PluginManagementResponse:
"""Return the iframe plugin domain -> agent mappings."""
@@ -0,0 +1,155 @@
"""Debounced file watcher and cron scheduler for opted-in knowledge bindings."""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from typing import Any
from croniter import croniter
from app.gateway.skill_knowledge_job_executor import _skill_directory
from deerflow.config.system_settings import load_system_settings
from deerflow.skill_knowledge import scan_skill
logger = logging.getLogger(__name__)
class SkillKnowledgeAutoSyncService:
def __init__(self, app: Any, *, poll_seconds: float = 10.0) -> None:
self.app = app
self.store = app.state.skill_knowledge_store
self.poll_seconds = poll_seconds
self._task: asyncio.Task | None = None
self._wake = asyncio.Event()
self._observed: dict[str, tuple[str, datetime]] = {}
self._schedule_expression = ""
self._next_schedule: datetime | None = None
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._loop())
def nudge(self) -> None:
self._wake.set()
async def close(self) -> None:
if self._task is None:
return
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
async def _queue_skill(
self,
skill_name: str,
bindings: list[dict[str, Any]],
*,
trigger: str,
review_mode: str,
) -> None:
targets = [
{
"target_type": binding["target_type"],
"target_mode": binding["target_mode"],
"target_id": binding["target_id"],
"target_name": binding["target_name"],
"remote_business_key": binding.get("remote_business_key"),
}
for binding in bindings
]
job = await self.store.create_job(
skill_names=[skill_name],
targets=targets,
created_by=None,
trigger=trigger,
review_mode=review_mode,
)
if job["total_count"]:
self.app.state.skill_knowledge_dispatcher.nudge()
async def _watch(
self,
bindings_by_skill: dict[str, list[dict[str, Any]]],
*,
debounce_seconds: int,
write_remote_enabled: bool,
review_mode: str,
) -> None:
now = datetime.now(UTC)
for skill_name, bindings in bindings_by_skill.items():
try:
root = _skill_directory(self.app.state.config, skill_name)
scan = await asyncio.to_thread(scan_skill, skill_name, root)
except Exception:
logger.exception("Failed to fingerprint skill %s", skill_name)
continue
synced = {str(binding.get("synced_digest") or "") for binding in bindings}
if synced == {scan.source_digest}:
self._observed.pop(skill_name, None)
continue
await self.store.mark_skill_stale(skill_name)
previous = self._observed.get(skill_name)
if previous is None or previous[0] != scan.source_digest:
self._observed[skill_name] = (scan.source_digest, now)
continue
if (now - previous[1]).total_seconds() < debounce_seconds:
continue
if write_remote_enabled:
current = await self.store.list_bindings(skill_names=[skill_name])
stale = [binding for binding in current if binding.get("auto_sync_enabled") and binding.get("status") == "stale"]
if stale:
await self._queue_skill(
skill_name,
stale,
trigger="watcher",
review_mode=review_mode,
)
self._observed.pop(skill_name, None)
def _schedule_due(self, expression: str, now: datetime) -> bool:
if expression != self._schedule_expression or self._next_schedule is None:
self._schedule_expression = expression
self._next_schedule = croniter(expression, now).get_next(datetime)
return False
if now < self._next_schedule:
return False
self._next_schedule = croniter(expression, now).get_next(datetime)
return True
async def _tick(self) -> None:
settings = (await asyncio.to_thread(load_system_settings)).skill_knowledge_auto_sync
bindings = [row for row in await self.store.list_bindings() if row.get("auto_sync_enabled")]
by_skill: dict[str, list[dict[str, Any]]] = {}
for binding in bindings:
by_skill.setdefault(str(binding["skill_name"]), []).append(binding)
if settings.watch_enabled:
await self._watch(
by_skill,
debounce_seconds=settings.debounce_seconds,
write_remote_enabled=settings.write_remote_enabled,
review_mode=settings.review_mode,
)
if settings.schedule_enabled and settings.write_remote_enabled and by_skill and self._schedule_due(settings.schedule_cron, datetime.now(UTC)):
for skill_name, skill_bindings in by_skill.items():
await self._queue_skill(
skill_name,
skill_bindings,
trigger="scheduled",
review_mode=settings.review_mode,
)
async def _loop(self) -> None:
while True:
try:
await self._tick()
except Exception:
logger.exception("Skill knowledge automatic sync tick failed")
try:
await asyncio.wait_for(self._wake.wait(), timeout=self.poll_seconds)
except TimeoutError:
pass
self._wake.clear()
@@ -0,0 +1,96 @@
"""Small persistent-queue dispatcher for skill knowledge items."""
from __future__ import annotations
import asyncio
import logging
import socket
from typing import Any
from uuid import uuid4
logger = logging.getLogger(__name__)
class SkillKnowledgeJobDispatcher:
def __init__(
self,
store: Any,
executor: Any,
*,
interval_seconds: float = 2.0,
concurrency: int = 3,
lease_seconds: int = 900,
) -> None:
self.store = store
self.executor = executor
self.interval_seconds = interval_seconds
self.lease_seconds = lease_seconds
self.owner = f"{socket.gethostname()}:{uuid4()}"
self.semaphore = asyncio.Semaphore(concurrency)
self._wake = asyncio.Event()
self._task: asyncio.Task | None = None
self._running: set[str] = set()
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._loop())
def nudge(self) -> None:
self._wake.set()
async def close(self) -> None:
if self._task is not None:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
async def _run(self, item: dict) -> None:
item_id = str(item["id"])
async with self.semaphore:
heartbeat = asyncio.create_task(self._heartbeat(item_id))
try:
await self.executor.run_item(item)
finally:
heartbeat.cancel()
try:
await heartbeat
except asyncio.CancelledError:
pass
self._running.discard(item_id)
self._wake.set()
async def _heartbeat(self, item_id: str) -> None:
while True:
await asyncio.sleep(min(30, max(5, self.lease_seconds // 3)))
if not await self.store.renew_lease(
item_id,
owner=self.owner,
lease_seconds=self.lease_seconds,
):
return
async def _loop(self) -> None:
while True:
try:
for item in await self.store.list_queued_items(limit=20):
item_id = str(item["id"])
if item_id in self._running:
continue
claimed = await self.store.claim_item(
item_id,
owner=self.owner,
lease_seconds=self.lease_seconds,
)
if claimed is None:
continue
self._running.add(item_id)
asyncio.create_task(self._run(claimed))
except Exception:
logger.exception("Skill knowledge dispatcher tick failed")
try:
await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds)
except TimeoutError:
pass
self._wake.clear()
@@ -0,0 +1,403 @@
"""Background executor for one skill x target matrix item."""
from __future__ import annotations
import asyncio
import json
import logging
from pathlib import Path
from typing import Any
import httpx
from deerflow.assistant_knowledge import export_package_from_skill_ir
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
from deerflow.skill_knowledge.graph_writer import parameterized_statements, plan_from_ir
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.types import SkillCategory
logger = logging.getLogger(__name__)
def _skill_directory(app_config: Any, skill_name: str) -> Path:
storage = get_or_new_skill_storage(app_config=app_config)
root = storage.get_skills_root_path().resolve()
for category in SkillCategory:
direct = root / category.value / skill_name
if (direct / "SKILL.md").is_file():
return direct
for category in SkillCategory:
category_root = root / category.value
if not category_root.is_dir():
continue
for skill_md in category_root.rglob("SKILL.md"):
if skill_md.is_symlink():
continue
parsed = parse_skill_file(skill_md, category=category, relative_path=skill_md.parent.relative_to(category_root))
if parsed and parsed.name == skill_name:
return skill_md.parent
raise FileNotFoundError(f"Skill '{skill_name}' not found under running skill roots")
def _graph_projection(ir: dict[str, Any]) -> str:
rows = []
entities = {str(item.get("local_id")): item for item in ir.get("entities", [])}
for relation in ir.get("relations", []):
if float(relation.get("confidence") or 0) < 0.75 or not relation.get("evidence"):
continue
source = entities.get(str(relation.get("source_local_id")), {})
target = entities.get(str(relation.get("target_local_id")), {})
rows.append(
json.dumps(
{
"source": source.get("name"),
"source_type": source.get("type"),
"predicate": relation.get("predicate"),
"target": target.get("name"),
"target_type": target.get("type"),
"description": relation.get("description"),
"confidence": relation.get("confidence"),
"evidence": relation.get("evidence"),
},
ensure_ascii=False,
)
)
return "\n".join(rows)
class SkillKnowledgeJobExecutor:
def __init__(self, app: Any) -> None:
self.app = app
self.store = app.state.skill_knowledge_store
async def _cancel_if_requested(self, item: dict[str, Any]) -> bool:
job = await self.store.get_job(str(item["job_id"]))
if not (job or {}).get("cancel_requested"):
return False
await self.store.update_item(
str(item["id"]),
status="canceled",
phase="completed",
progress=100,
)
return True
async def _assert_source_unchanged(self, item: dict[str, Any], expected_digest: str) -> None:
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
current = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
current = await enrich_converted_documents(current)
if current.source_digest == expected_digest:
return
await self.store.mark_skill_stale(str(item["skill_name"]))
raise RuntimeError("技能知识文件在归纳期间发生变化,旧版本保持激活,请重试")
async def _write_direct_graph(
self,
item: dict[str, Any],
snapshot_id: str,
ir: dict[str, Any],
) -> dict[str, Any] | None:
config = self.app.state.config.llmwiki.weknora.graph
if not config.direct_write_enabled:
return None
if not config.neo4j_http_url or not config.username or not config.password:
raise RuntimeError("Neo4j direct writer 已开启但后端连接配置不完整")
plan = plan_from_ir(
ir,
target_type=str(item["target_type"]),
target_id=str(item["target_id"]),
binding_id=str(item["binding_id"]),
snapshot_id=snapshot_id,
job_id=str(item["job_id"]),
)
statements = parameterized_statements(plan)
if not statements:
return {"nodes": 0, "edges": 0, "statement_count": 0}
endpoint = f"{config.neo4j_http_url.rstrip('/')}/db/{config.database}/tx/commit"
async with httpx.AsyncClient(
auth=(config.username, config.password),
timeout=config.timeout_seconds,
) as client:
response = await client.post(endpoint, json={"statements": statements})
response.raise_for_status()
payload = response.json()
errors = payload.get("errors") or []
if errors:
raise RuntimeError(f"Neo4j transaction failed: {str(errors[0].get('code') or 'unknown')}")
return {
"nodes": len(plan.nodes),
"edges": len(plan.edges),
"statement_count": len(statements),
"scope": plan.scope,
}
async def run_item(self, item: dict[str, Any]) -> None:
item_id = str(item["id"])
try:
await self.store.update_item(item_id, status="running", phase="scanning", progress=5)
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
if await self._cancel_if_requested(item):
return
await self.store.update_item(item_id, phase="extracting", progress=30)
skill_record = await self.app.state.skill_store.get_any(str(item["skill_name"]))
ir = await asyncio.to_thread(
build_skill_ir,
scan,
display_name=(skill_record or {}).get("name_zh"),
description=str((skill_record or {}).get("description") or ""),
)
snapshot = await self.store.get_or_create_snapshot(
skill_name=str(item["skill_name"]),
source_digest=scan.source_digest,
extractor_version=str(ir["extractor_version"]),
prompt_hash=str(ir["prompt_hash"]),
model_name=str(ir["model_name"]),
manifest=scan.manifest,
ir=ir,
counts=scan.counts,
)
snapshot_id = str(snapshot["id"])
await self.store.update_item(item_id, phase="awaiting_review", progress=45, snapshot_id=snapshot_id)
job = await self.store.get_job(str(item["job_id"]))
review_mode = str((job or {}).get("review_mode") or "auto_high_confidence")
review_objects: list[tuple[str, dict]] = []
if review_mode == "required":
review_objects = [("entity", obj) for obj in ir.get("entities", [])] + [("relation", obj) for obj in ir.get("relations", [])]
elif review_mode == "auto_high_confidence":
review_objects = [(kind, obj) for kind, rows in (("entity", ir.get("entities", [])), ("relation", ir.get("relations", []))) for obj in rows if float(obj.get("confidence") or 0) < 0.9]
if review_objects:
existing = await self.store.list_reviews(item_id=item_id, status=None)
if not existing:
await self.store.create_reviews(
job_id=str(item["job_id"]),
item_id=item_id,
binding_id=str(item["binding_id"]),
snapshot_id=snapshot_id,
objects=review_objects,
)
await self.store.update_item(item_id, status="awaiting_review", phase="awaiting_review", progress=50)
return
await self._project_and_activate(item, snapshot_id, ir, scan)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
logger.exception("Skill knowledge item failed: item=%s", item_id)
await self.store.update_item(
item_id,
status="failed",
phase="completed",
progress=100,
error_code=type(exc).__name__,
error_message=str(exc)[:2000],
)
async def resume_reviewed_item(self, item_id: str) -> None:
if await self.store.pending_review_count(item_id):
return
jobs = await self.store.list_jobs(limit=100)
item = None
for job in jobs:
detail = await self.store.get_job(str(job["id"]))
item = next((row for row in (detail or {}).get("items", []) if row["id"] == item_id), None)
if item:
break
if not item or not item.get("snapshot_id"):
return
snapshot = await self.store.get_snapshot(str(item["snapshot_id"]))
ir = await self.store.effective_ir(str(item["snapshot_id"]), item_id)
if not snapshot or not ir:
return
try:
await self.store.update_item(item_id, status="running", phase="merging", progress=55)
await self._project_and_activate(item, str(item["snapshot_id"]), ir, None)
except Exception as exc: # noqa: BLE001
logger.exception("Reviewed skill knowledge item failed: item=%s", item_id)
await self.store.update_item(
item_id,
status="failed",
phase="completed",
progress=100,
error_code=type(exc).__name__,
error_message=str(exc)[:2000],
)
async def _project_and_activate(self, item: dict[str, Any], snapshot_id: str, ir: dict[str, Any], scan: Any | None) -> None:
if await self._cancel_if_requested(item):
return
expected_digest = str(ir.get("skill", {}).get("source_digest") or "")
if scan is None:
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
target_type = str(item["target_type"])
artifacts: list[dict[str, Any]] = []
binding_status = "synced"
page = ir["wiki"]["skill_page"]
if target_type == "assistant":
package = export_package_from_skill_ir(ir)
await self.app.state.assistant_knowledge_store.import_package(
package=package,
source_type="skill",
source_key=str(item["skill_name"]),
source_name=str(ir["skill"].get("display_name") or item["skill_name"]),
trigger="skill_direct",
created_by=(await self.store.get_job(str(item["job_id"]))).get("created_by"),
skill_name=str(item["skill_name"]),
)
artifacts.append({"kind": "assistant_wiki_revision", "slug": page["slug"], "content": page["content"], "business_key": f"assistant:wiki:{page['slug']}"})
elif str(item["target_mode"]) == "readonly_sidecar":
artifacts.append({"kind": "sidecar_wiki", "slug": page["slug"], "content": page["content"], "business_key": f"sidecar:{item['target_id']}:{page['slug']}"})
artifacts.append({"kind": "sidecar_graph", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "content": _graph_projection(ir)})
binding_status = "pending_remote_write"
else:
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
if mapping is None:
raise ValueError("Target WeKnora mapping does not exist")
runtime = get_resolved_llmwiki_runtime(self.app.state.config)
if not runtime.weknora_enabled:
raise RuntimeError("普通知识库服务未配置")
client = build_weknora_client(runtime)
remote_kb_id = str(mapping["weknora_id"])
if scan is not None:
for file in scan.files:
source_path = Path(scan.source_root) / file.path
if file.text:
remote = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/source/{file.path}", content=file.text)
else:
remote = await client.upload_document(
remote_kb_id,
filename=f"{item['skill_name']}-{source_path.name}",
content=await asyncio.to_thread(source_path.read_bytes),
)
remote_id = str(remote.get("id") or remote.get("knowledge_id") or "") or None
artifacts.append(
{
"kind": "raw_source",
"slug": file.path,
"remote_id": remote_id,
"business_key": f"weknora:document:{remote_id}" if remote_id else None,
"remote_refs": [{"provider": "weknora", "remote_kind": "document", "remote_id": remote_id}] if remote_id else [],
}
)
target_mode = str(item["target_mode"])
if target_mode == "wiki":
remote_page: dict[str, Any]
try:
await client.get_wiki_page(remote_kb_id, page["slug"])
except WeKnoraError as exc:
if exc.status_code != 404:
raise
remote_page = await client.create_wiki_page(
remote_kb_id,
{
"slug": page["slug"],
"title": page["title"],
"content": page["content"],
"summary": str(ir["skill"].get("description") or ""),
"page_type": "skill",
},
)
else:
remote_page = await client.update_wiki_page(
remote_kb_id,
page["slug"],
{
"title": page["title"],
"content": page["content"],
"summary": str(ir["skill"].get("description") or ""),
},
)
remote_page_id = str(remote_page.get("id") or remote_page.get("page_id") or page["slug"])
artifacts.append(
{
"kind": "wiki_page",
"slug": page["slug"],
"content": page["content"],
"remote_id": remote_page_id,
"business_key": f"weknora:wiki_page:{remote_page_id}",
"remote_refs": [
{
"provider": "weknora",
"remote_kind": "wiki_page",
"remote_id": remote_page_id,
"slug": page["slug"],
}
],
}
)
else:
document_content = page["content"]
artifact_kind = "summary_document"
if target_mode == "faq":
artifact_kind = "faq_projection"
document_content = f"问题:{page['title']} 能解决什么问题,如何使用?\n\n回答:\n{page['content']}"
elif target_mode == "conversation_archive":
artifact_kind = "conversation_archive"
document_content = f"# 技能归纳会话归档:{page['title']}\n\n以下内容由技能知识归纳任务生成。\n\n{page['content']}"
remote_document = await client.create_manual_document(
remote_kb_id,
title=f"skills/{item['skill_name']}/{target_mode}.md",
content=document_content,
)
remote_document_id = str(remote_document.get("id") or remote_document.get("knowledge_id") or "") or None
artifacts.append(
{
"kind": artifact_kind,
"slug": f"skills/{item['skill_name']}/{target_mode}.md",
"content": document_content,
"remote_id": remote_document_id,
"business_key": (f"weknora:document:{remote_document_id}" if remote_document_id else None),
"remote_refs": (
[
{
"provider": "weknora",
"remote_kind": "document",
"remote_id": remote_document_id,
}
]
if remote_document_id
else []
),
}
)
projection = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/graph-projection.jsonl", content=_graph_projection(ir))
projection_id = str(projection.get("id") or projection.get("knowledge_id") or "") or None
artifacts.append({"kind": "graph_projection", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "remote_id": projection_id})
try:
graph_audit = await self._write_direct_graph(item, snapshot_id, ir)
if graph_audit is not None:
artifacts.append(
{
"kind": "neo4j_direct_write_audit",
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
"content": json.dumps(graph_audit, ensure_ascii=False),
"metadata": graph_audit,
}
)
except Exception as exc: # noqa: BLE001
logger.exception("Optional Neo4j direct graph write failed: item=%s", item["id"])
binding_status = "partial_failed"
artifacts.append(
{
"kind": "neo4j_direct_write_audit",
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
"content": json.dumps(
{"status": "failed", "error": type(exc).__name__},
ensure_ascii=False,
),
"metadata": {
"status": "failed",
"error_type": type(exc).__name__,
},
}
)
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
@@ -556,6 +556,14 @@ llmwiki:
web_base_url: "http://119.254.86.251:8088"
admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX"
# 高风险能力,默认关闭。连接参数仅允许后端配置;生产环境请使用环境变量。
graph:
direct_write_enabled: false
neo4j_http_url: ""
username: ""
password: ""
database: "neo4j"
timeout_seconds: 30
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
# 且服务启动时只启动定时器,不会立刻全量向量化。
@@ -0,0 +1,5 @@
"""Assistant knowledge package helpers."""
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
__all__ = ["export_package_from_skill_ir"]
@@ -0,0 +1,22 @@
"""Normalize heterogeneous sources into the assistant import-package shape."""
from __future__ import annotations
from typing import Any
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
skill = ir.get("skill") or {}
return {
"source": {
"type": "skill",
"skill_name": skill.get("name"),
"source_digest": skill.get("source_digest"),
},
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
"documents": ir.get("files") or [],
"chunks": [],
"entities": ir.get("entities") or [],
"relations": ir.get("relations") or [],
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
}
@@ -44,6 +44,7 @@ class WeKnoraConnectionConfig(BaseModel):
repr=False,
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@@ -52,6 +53,23 @@ class WeKnoraConnectionConfig(BaseModel):
return normalize_service_url(value)
class WeKnoraDirectGraphConfig(BaseModel):
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
direct_write_enabled: bool = False
neo4j_http_url: str = ""
username: str = ""
password: str = Field(default="", repr=False)
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
timeout_seconds: float = Field(default=30, gt=0, le=300)
model_config = ConfigDict(extra="ignore")
@field_validator("neo4j_http_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
@@ -156,6 +156,18 @@ class PromptPrefixSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class SkillKnowledgeAutoSyncSettings(BaseModel):
"""Watcher/scheduler policy for bindings that explicitly opt in."""
watch_enabled: bool = True
schedule_enabled: bool = True
schedule_cron: str = Field(default="0 */6 * * *", min_length=5, max_length=128)
debounce_seconds: int = Field(default=120, ge=10, le=86_400)
write_remote_enabled: bool = True
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
model_config = ConfigDict(extra="ignore")
class AppearanceRouteParam(BaseModel):
"""Runtime query-param descriptor for top-bar shortcut buttons.
@@ -501,6 +513,11 @@ class SystemSettings(BaseModel):
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
knowledge_base_product_version: Literal["v1", "v2"] = Field(
default="v1",
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
)
model_config = ConfigDict(extra="ignore")
@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,143 @@
"""ORM model for the single global DeerFlow assistant knowledge base."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,601 @@
"""Repository for the single global assistant knowledge base."""
from __future__ import annotations
import difflib
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
return data
class AssistantKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_base(self) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
existing = await self.get_base()
if existing:
return existing
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
async with self._sf() as session:
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def _upsert_source(
self,
session: AsyncSession,
*,
base_id: str,
source_type: str,
source_key: str,
source_name: str,
metadata: dict,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
) -> AssistantKnowledgeSourceRow:
stmt = select(AssistantKnowledgeSourceRow).where(
AssistantKnowledgeSourceRow.base_id == base_id,
AssistantKnowledgeSourceRow.source_type == source_type,
AssistantKnowledgeSourceRow.source_key == source_key,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = AssistantKnowledgeSourceRow(
id=str(uuid4()),
base_id=base_id,
source_type=source_type,
source_key=source_key,
source_name=source_name,
skill_name=skill_name,
knowledge_base_mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
status="active",
metadata_json=metadata,
created_at=_now(),
updated_at=_now(),
)
session.add(row)
await session.flush()
else:
row.source_name = source_name
row.status = "active"
row.metadata_json = metadata
row.updated_at = _now()
return row
async def _publish_page(
self,
session: AsyncSession,
*,
base_id: str,
slug: str,
title: str,
page_type: str,
content: str,
summary: str,
search_terms: list[str],
source_id: str,
import_job_id: str,
created_by: str | None,
change_type: str,
) -> AssistantKnowledgeWikiPageRow:
now = _now()
page = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(
AssistantKnowledgeWikiPageRow.base_id == base_id,
AssistantKnowledgeWikiPageRow.slug == slug,
)
)
).scalar_one_or_none()
if page is None:
page = AssistantKnowledgeWikiPageRow(
id=str(uuid4()),
base_id=base_id,
slug=slug,
title=title,
page_type=page_type,
content_markdown="",
summary="",
search_text="",
status="active",
updated_at=now,
)
session.add(page)
await session.flush()
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
revision = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=content,
summary=summary,
source_id=source_id,
import_job_id=import_job_id,
change_type=change_type,
status="published",
created_by=created_by,
created_at=now,
published_at=now,
)
session.add(revision)
page.title = title
page.page_type = page_type
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, *search_terms])
page.current_revision_id = revision.id
page.status = "active"
page.updated_at = now
return page
async def import_package(
self,
*,
package: dict[str, Any],
source_type: str,
source_key: str,
source_name: str,
trigger: str,
created_by: str | None,
skill_name: str | None = None,
mapping_id: str | None = None,
upload_batch_id: str | None = None,
) -> dict[str, Any]:
base = await self.initialize(created_by=created_by)
now = _now()
async with self._sf() as session:
source = await self._upsert_source(
session,
base_id=base["id"],
source_type=source_type,
source_key=source_key,
source_name=source_name,
metadata=package.get("source") or {},
skill_name=skill_name,
mapping_id=mapping_id,
upload_batch_id=upload_batch_id,
)
# A re-import replaces this source's prior contributions while retaining
# canonical entities/relations that are still supported by other sources.
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
job = AssistantKnowledgeImportJobRow(
id=str(uuid4()),
base_id=base["id"],
source_id=source.id,
trigger=trigger,
status="running",
phase="aligning_entities",
counts_json={},
created_by=created_by,
created_at=now,
)
session.add(job)
await session.flush()
source.last_import_job_id = job.id
entity_map: dict[str, str] = {}
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
raw_entities = list(package.get("entities") or [])
for raw in raw_entities:
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
normalized = normalize_name(display_name)
if not normalized:
continue
stmt = select(AssistantKnowledgeEntityRow).where(
AssistantKnowledgeEntityRow.base_id == base["id"],
AssistantKnowledgeEntityRow.entity_type == entity_type,
AssistantKnowledgeEntityRow.normalized_name == normalized,
)
entity = (await session.execute(stmt)).scalar_one_or_none()
if entity is None:
entity = AssistantKnowledgeEntityRow(
id=str(uuid4()),
base_id=base["id"],
entity_type=entity_type,
normalized_name=normalized,
display_name=display_name,
aliases_json=raw.get("aliases") or [],
attributes_json=raw.get("attributes") or {},
status="active",
)
session.add(entity)
await session.flush()
raw_id = str(raw.get("id") or raw.get("local_id") or display_name)
entity_map[raw_id] = entity.id
entity_rows[entity.id] = (entity, raw)
session.add(
AssistantKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=entity.id,
source_id=source.id,
import_job_id=job.id,
source_wiki_slug=raw.get("source_wiki_slug"),
source_chunk_id=raw.get("source_chunk_id"),
source_skill_name=skill_name,
evidence_json=raw.get("evidence") or [],
confidence=float(raw.get("confidence") or 1.0),
)
)
raw_relations = list(package.get("relations") or [])
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
for raw in raw_relations:
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
target_entity_id = entity_map.get(str(raw.get("target_entity_id") or raw.get("target_local_id") or raw.get("target")))
confidence = float(raw.get("confidence") or 1.0)
if not source_entity_id or not target_entity_id or confidence < 0.75:
continue
predicate = normalize_predicate(str(raw.get("predicate") or "related_to"))
stmt = select(AssistantKnowledgeRelationRow).where(
AssistantKnowledgeRelationRow.base_id == base["id"],
AssistantKnowledgeRelationRow.source_entity_id == source_entity_id,
AssistantKnowledgeRelationRow.predicate == predicate,
AssistantKnowledgeRelationRow.target_entity_id == target_entity_id,
)
relation = (await session.execute(stmt)).scalar_one_or_none()
if relation is None:
relation = AssistantKnowledgeRelationRow(
id=str(uuid4()),
base_id=base["id"],
source_entity_id=source_entity_id,
predicate=predicate,
target_entity_id=target_entity_id,
description=str(raw.get("description") or ""),
status="active",
)
session.add(relation)
await session.flush()
relation_rows[relation.id] = (relation, raw)
session.add(
AssistantKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=relation.id,
source_id=source.id,
import_job_id=job.id,
source_wiki_slug=raw.get("source_wiki_slug"),
graph_edge_id=raw.get("graph_edge_id"),
evidence_json=raw.get("evidence") or [],
confidence=confidence,
)
)
for entity, raw in entity_rows.values():
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
entity.aliases_json = aliases
content = "\n".join(
[
f"# {entity.display_name}",
"",
f"- 类型:{entity.entity_type}",
f"- 别名:{'、'.join(aliases) if aliases else '无'}",
f"- 本次来源:{source_name}",
"",
"## 属性",
"",
"```json",
str(entity.attributes_json or {}),
"```",
]
)
entity_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"entities/{entity.entity_type}/{entity.id}",
title=entity.display_name,
page_type="entity",
content=content,
summary=f"{entity.entity_type} 实体,来源:{source_name}",
search_terms=aliases,
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="entity_alignment",
)
entity.wiki_page_id = entity_page.id
for relation, raw in relation_rows.values():
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
target_name_display = target_entity.display_name if target_entity else relation.target_entity_id
content = f"# {source_name_display} —{relation.predicate}→ {target_name_display}\n\n- 本次来源:{source_name}\n- 置信度:{float(raw.get('confidence') or 1.0):.2f}\n\n{relation.description}"
relation_page = await self._publish_page(
session,
base_id=base["id"],
slug=f"relations/{relation.id}",
title=f"{source_name_display} {relation.predicate} {target_name_display}",
page_type="relation",
content=content,
summary=relation.description,
search_terms=[source_name_display, target_name_display, relation.predicate],
source_id=source.id,
import_job_id=job.id,
created_by=created_by,
change_type="relation_alignment",
)
relation.wiki_page_id = relation_page.id
pages = list(package.get("wiki_pages") or [])
if not pages and package.get("wiki"):
wiki = package["wiki"]
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
pages = [page] if page else []
for raw in pages:
if not isinstance(raw, dict):
continue
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
title = str(raw.get("title") or source_name)
content = str(raw.get("content") or raw.get("content_markdown") or "")
summary = str(raw.get("summary") or "")
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
page = (await session.execute(stmt)).scalar_one_or_none()
if page is None:
page = AssistantKnowledgeWikiPageRow(
id=str(uuid4()),
base_id=base["id"],
slug=slug,
title=title,
page_type=str(raw.get("page_type") or "source"),
content_markdown="",
summary="",
search_text="",
status="active",
updated_at=now,
)
session.add(page)
await session.flush()
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
revision = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=content,
summary=summary,
source_id=source.id,
import_job_id=job.id,
change_type="skill_sync" if source_type == "skill" else "import",
status="published",
created_by=created_by,
created_at=now,
published_at=now,
)
session.add(revision)
page.title = title
page.content_markdown = content
page.summary = summary
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
page.current_revision_id = revision.id
page.updated_at = now
counts = {
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
"documents": len(package.get("documents") or []),
"chunks": len(package.get("chunks") or []),
"entities": len(raw_entities),
"relations": len(raw_relations),
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
"graph_edges": len((package.get("graph") or {}).get("edges") or []),
}
job.status = "completed"
job.phase = "completed"
job.counts_json = counts
job.completed_at = now
await self._retire_orphans(session, base["id"])
await session.flush()
result = _dict(job)
await session.commit()
return result
async def _retire_orphans(self, session: AsyncSession, base_id: str) -> None:
entities = (await session.execute(select(AssistantKnowledgeEntityRow).where(AssistantKnowledgeEntityRow.base_id == base_id))).scalars()
for entity in entities:
contribution = (await session.execute(select(AssistantKnowledgeEntityContributionRow.id).where(AssistantKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (await session.execute(select(AssistantKnowledgeRelationRow).where(AssistantKnowledgeRelationRow.base_id == base_id))).scalars()
for relation in relations:
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
if query:
needle = f"%{query.strip()}%"
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
if page_type:
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_page(self, slug: str) -> dict[str, Any] | None:
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
if page is None:
return []
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_revision(self, revision_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
return _dict(row) if row else None
async def compare_revisions(self, left_id: str, right_id: str) -> dict[str, Any] | None:
left, right = await self.get_revision(left_id), await self.get_revision(right_id)
if not left or not right or left["page_id"] != right["page_id"]:
return None
diff = "\n".join(
difflib.unified_diff(
str(left["content_markdown"]).splitlines(),
str(right["content_markdown"]).splitlines(),
fromfile=f"revision-{left['revision_no']}",
tofile=f"revision-{right['revision_no']}",
lineterm="",
)
)
return {"left": left, "right": right, "diff": diff}
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
now = _now()
async with self._sf() as session:
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
if page is None or source is None or source.page_id != page.id:
return None
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one()) + 1
row = AssistantKnowledgeWikiRevisionRow(
id=str(uuid4()),
page_id=page.id,
revision_no=revision_no,
content_markdown=source.content_markdown,
summary=source.summary,
source_id=source.source_id,
import_job_id=source.import_job_id,
change_type="rollback",
status="published",
created_by=actor,
created_at=now,
published_at=now,
)
session.add(row)
page.current_revision_id = row.id
page.content_markdown = row.content_markdown
page.summary = row.summary
page.search_text = "\n".join([page.title, row.summary, row.content_markdown])
page.updated_at = now
await session.flush()
data = _dict(row)
await session.commit()
return data
async def list_sources(self) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeSourceRow, source_id)
return _dict(row) if row and row.status != "deleted" else None
async def get_entity(self, entity_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeEntityRow, entity_id)
if row is None or row.status != "active":
return None
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.entity_id == entity_id))).scalars()
result = _dict(row)
result["contributions"] = [_dict(value) for value in contributions]
return result
async def get_relation(self, relation_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeRelationRow, relation_id)
if row is None or row.status != "active":
return None
contributions = (await session.execute(select(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.relation_id == relation_id))).scalars()
result = _dict(row)
result["contributions"] = [_dict(value) for value in contributions]
return result
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(AssistantKnowledgeImportJobRow, job_id)
return _dict(row) if row else None
async def delete_source(self, source_id: str) -> bool:
async with self._sf() as session:
source = await session.get(AssistantKnowledgeSourceRow, source_id)
if source is None:
return False
source.status = "deleted"
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
pages = (
await session.execute(
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
)
).scalars()
for page in pages:
fallback = (
await session.execute(
select(AssistantKnowledgeWikiRevisionRow)
.where(
AssistantKnowledgeWikiRevisionRow.page_id == page.id,
AssistantKnowledgeWikiRevisionRow.source_id != source_id,
AssistantKnowledgeWikiRevisionRow.status == "published",
)
.order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
.limit(1)
)
).scalar_one_or_none()
if fallback is None:
page.status = "deleted"
page.current_revision_id = None
else:
page.current_revision_id = fallback.id
page.content_markdown = fallback.content_markdown
page.summary = fallback.summary
page.search_text = "\n".join([page.title, fallback.summary, fallback.content_markdown])
await session.execute(AssistantKnowledgeWikiRevisionRow.__table__.update().where(AssistantKnowledgeWikiRevisionRow.source_id == source_id).values(status="deleted"))
await self._retire_orphans(session, source.base_id)
await session.commit()
return True
def make_assistant_knowledge_store(session_factory):
return AssistantKnowledgeRepository(session_factory) if session_factory is not None else None
@@ -0,0 +1,73 @@
"""Add skill distillation and assistant knowledge tables.
Revision ID: 20260901_01
Revises: 20260825_01
Create Date: 2026-09-01
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
revision = "20260901_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
TABLES = [
SkillKnowledgeBindingRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncJobRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeArtifactRow,
SkillKnowledgeReviewRow,
SkillKnowledgeEntityRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeRelationContributionRow,
AssistantKnowledgeBaseRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeRelationContributionRow,
]
def upgrade() -> None:
bind = op.get_bind()
for model in TABLES:
model.__table__.create(bind, checkfirst=True)
def downgrade() -> None:
bind = op.get_bind()
for model in reversed(TABLES):
model.__table__.drop(bind, checkfirst=True)
@@ -35,6 +35,17 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -86,6 +97,18 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.sensitive_words.model import SensitiveWordRow
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_buttons.model import TaskButtonRow
@@ -107,6 +130,15 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
"AssistantKnowledgeEntityRow",
"AssistantKnowledgeEntityContributionRow",
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
@@ -164,6 +196,16 @@ __all__ = [
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
"SkillKnowledgeArtifactRow",
"SkillKnowledgeBindingRow",
"SkillKnowledgeEntityContributionRow",
"SkillKnowledgeEntityRow",
"SkillKnowledgeRelationContributionRow",
"SkillKnowledgeRelationRow",
"SkillKnowledgeReviewRow",
"SkillKnowledgeSnapshotRow",
"SkillKnowledgeSyncItemRow",
"SkillKnowledgeSyncJobRow",
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
@@ -0,0 +1,15 @@
from deerflow.persistence.skill_knowledge.model import ( # noqa: F401
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]
@@ -0,0 +1,203 @@
"""ORM rows for skill-to-knowledge distillation and review."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class SkillKnowledgeBindingRow(Base):
__tablename__ = "skill_knowledge_bindings"
__table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
target_id: Mapped[str] = mapped_column(String(128), index=True)
target_name: Mapped[str] = mapped_column(String(255), default="")
remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeSnapshotRow(Base):
__tablename__ = "skill_knowledge_snapshots"
__table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
source_digest: Mapped[str] = mapped_column(String(80), index=True)
extractor_version: Mapped[str] = mapped_column(String(64))
prompt_hash: Mapped[str] = mapped_column(String(80))
model_name: Mapped[str] = mapped_column(String(128))
manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="ready")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeSyncJobRow(Base):
__tablename__ = "skill_knowledge_sync_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
trigger: Mapped[str] = mapped_column(String(32), default="manual")
target_count: Mapped[int] = mapped_column(Integer, default=0)
targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
total_count: Mapped[int] = mapped_column(Integer, default=0)
success_count: Mapped[int] = mapped_column(Integer, default=0)
partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
failed_count: Mapped[int] = mapped_column(Integer, default=0)
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeSyncItemRow(Base):
__tablename__ = "skill_knowledge_sync_items"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
target_type: Mapped[str] = mapped_column(String(32))
target_mode: Mapped[str] = mapped_column(String(32))
target_id: Mapped[str] = mapped_column(String(128))
skill_name: Mapped[str] = mapped_column(String(255), index=True)
snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="scanning")
progress: Mapped[int] = mapped_column(Integer, default=0)
attempts: Mapped[int] = mapped_column(Integer, default=0)
active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeArtifactRow(Base):
__tablename__ = "skill_knowledge_artifacts"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
kind: Mapped[str] = mapped_column(String(48), index=True)
slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeReviewRow(Base):
__tablename__ = "skill_knowledge_reviews"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
item_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
object_type: Mapped[str] = mapped_column(String(32), index=True)
object_key: Mapped[str] = mapped_column(String(128))
original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeEntityRow(Base):
__tablename__ = "skill_knowledge_entities"
__table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512))
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeEntityContributionRow(Base):
__tablename__ = "skill_knowledge_entity_contributions"
__table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_entity_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class SkillKnowledgeRelationRow(Base):
__tablename__ = "skill_knowledge_relations"
__table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeRelationContributionRow(Base):
__tablename__ = "skill_knowledge_relation_contributions"
__table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_relation_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,758 @@
"""SQL repository for skill knowledge jobs, snapshots and contributions."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
list_json_fields = {
"aliases_json",
"evidence_json",
"manifest_json",
"refs_json",
"targets_json",
}
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
return data
class SkillKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
entities = (
await session.execute(
select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == target_type,
SkillKnowledgeEntityRow.target_id == target_id,
)
)
).scalars()
for entity in entities:
contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (
await session.execute(
select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == target_type,
SkillKnowledgeRelationRow.target_id == target_id,
)
)
).scalars()
for relation in relations:
contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def create_job(
self,
*,
skill_names: list[str],
targets: list[dict[str, Any]],
created_by: str | None,
force: bool = False,
trigger: str = "manual",
review_mode: str = "auto_high_confidence",
) -> dict[str, Any]:
names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
normalized_targets: list[dict[str, Any]] = []
seen_targets: set[tuple[str, str]] = set()
for target in targets:
target_type = str(target.get("target_type") or "").strip()
target_id = str(target.get("target_id") or "").strip()
if not target_type or not target_id or (target_type, target_id) in seen_targets:
continue
seen_targets.add((target_type, target_id))
normalized_targets.append(
{
"target_type": target_type,
"target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
"target_id": target_id,
"target_name": str(target.get("target_name") or target_id),
"remote_business_key": target.get("remote_business_key"),
"readonly": bool(target.get("readonly", False)),
}
)
job_id = str(uuid4())
now = _now()
async with self._sf() as session:
job = SkillKnowledgeSyncJobRow(
id=job_id,
trigger=trigger,
target_count=len(normalized_targets),
targets_json=normalized_targets,
status="queued",
total_count=0,
review_mode=review_mode,
created_by=created_by,
created_at=now,
)
session.add(job)
for skill_name in names:
for target in normalized_targets:
stmt = select(SkillKnowledgeBindingRow).where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.target_type == target["target_type"],
SkillKnowledgeBindingRow.target_id == target["target_id"],
)
binding = (await session.execute(stmt)).scalar_one_or_none()
if binding is None:
binding = SkillKnowledgeBindingRow(
id=str(uuid4()),
skill_name=skill_name,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
target_name=target["target_name"],
remote_business_key=target.get("remote_business_key"),
status="queued",
created_by=created_by,
created_at=now,
updated_at=now,
)
session.add(binding)
else:
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
continue
binding.target_mode = target["target_mode"]
binding.target_name = target["target_name"]
binding.remote_business_key = target.get("remote_business_key")
binding.status = "queued"
binding.active = True
binding.updated_at = now
item = SkillKnowledgeSyncItemRow(
id=str(uuid4()),
job_id=job_id,
binding_id=binding.id,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
skill_name=skill_name,
status="queued",
phase="scanning",
progress=0,
active_dedupe_key=binding.id,
created_at=now,
updated_at=now,
)
session.add(item)
binding.last_job_id = job_id
binding.last_item_id = item.id
job.total_count += 1
if job.total_count == 0:
job.status = "completed"
job.completed_at = now
await session.flush()
result = _dict(job)
result["force"] = force
await session.commit()
return result
async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeBindingRow)
if skill_names:
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
if target_id:
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
if not include_detached:
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
return _dict(row) if row else None
async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
if active_only:
stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def has_active_item(self, binding_id: str) -> bool:
async with self._sf() as session:
value = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
return value is not None
async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
if row is None or not row.active:
return None
row.auto_sync_enabled = enabled
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return None
items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
result = _dict(job)
result["items"] = [_dict(item) for item in items]
return result
async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
now = _now()
stmt = (
select(SkillKnowledgeSyncItemRow)
.where(
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
)
)
.order_by(SkillKnowledgeSyncItemRow.created_at)
.limit(limit)
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def claim_item(
self,
item_id: str,
*,
owner: str,
lease_seconds: int = 900,
) -> dict[str, Any] | None:
"""Atomically claim a queued item, or reclaim one whose worker lease expired."""
now = _now()
stmt = (
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
),
)
.values(
status="running",
started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
attempts=SkillKnowledgeSyncItemRow.attempts + 1,
lease_owner=owner,
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
async with self._sf() as session:
result = await session.execute(stmt)
if result.rowcount != 1:
await session.rollback()
return None
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
SkillKnowledgeSyncItemRow.status == "running",
SkillKnowledgeSyncItemRow.lease_owner == owner,
)
.values(
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
await session.commit()
return result.rowcount == 1
async def update_item(
self,
item_id: str,
*,
status: str | None = None,
phase: str | None = None,
progress: int | None = None,
snapshot_id: str | None = None,
error_code: str | None = None,
error_message: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
if status is not None:
row.status = status
if status == "running" and row.started_at is None:
row.started_at = _now()
row.attempts += 1
if status in {"completed", "partial_failed", "failed", "canceled"}:
row.completed_at = _now()
row.active_dedupe_key = None
elif status == "queued":
row.completed_at = None
if status != "running":
row.lease_owner = None
row.lease_until = None
if phase is not None:
row.phase = phase
if progress is not None:
row.progress = max(0, min(100, progress))
if snapshot_id is not None:
row.snapshot_id = snapshot_id
row.last_error_code = error_code
row.last_error_message = error_message
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def refresh_job(self, job_id: str) -> None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return
rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
counts = {str(status): int(count) for status, count in rows}
job.success_count = counts.get("completed", 0)
job.partial_failed_count = counts.get("partial_failed", 0)
job.failed_count = counts.get("failed", 0)
active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
if counts.get("running", 0) or counts.get("awaiting_review", 0):
job.status = "running"
job.started_at = job.started_at or _now()
elif active == 0 and sum(counts.values()) >= job.total_count:
if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
job.status = "failed"
elif job.failed_count or job.partial_failed_count:
job.status = "partial_failed"
else:
job.status = "completed"
job.completed_at = _now()
await session.commit()
async def cancel_job(self, job_id: str) -> bool:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return False
job.cancel_requested = True
await session.execute(
SkillKnowledgeSyncItemRow.__table__.update()
.where(
SkillKnowledgeSyncItemRow.job_id == job_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
)
.values(
status="canceled",
phase="completed",
completed_at=_now(),
progress=100,
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
)
)
job.status = "canceled"
job.completed_at = _now()
await session.commit()
return True
async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
SkillKnowledgeSyncItemRow.id != row.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
return None
row.status = "queued"
row.phase = "scanning"
row.progress = 0
row.completed_at = None
row.active_dedupe_key = row.binding_id
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
stmt = select(SkillKnowledgeSnapshotRow).where(
SkillKnowledgeSnapshotRow.skill_name == skill_name,
SkillKnowledgeSnapshotRow.source_digest == source_digest,
SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
SkillKnowledgeSnapshotRow.model_name == model_name,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = SkillKnowledgeSnapshotRow(
id=str(uuid4()),
skill_name=skill_name,
source_digest=source_digest,
extractor_version=extractor_version,
prompt_hash=prompt_hash,
model_name=model_name,
manifest_json=manifest,
ir_json=ir,
counts_json=counts,
status="ready",
created_at=_now(),
)
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
return _dict(row) if row else None
async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
rows = [
SkillKnowledgeReviewRow(
id=str(uuid4()),
job_id=job_id,
item_id=item_id,
binding_id=binding_id,
snapshot_id=snapshot_id,
object_type=kind,
object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
original_json=obj,
status="pending",
created_at=_now(),
)
for kind, obj in objects
]
async with self._sf() as session:
session.add_all(rows)
await session.commit()
return len(rows)
async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeReviewRow)
if job_id:
stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
if item_id:
stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
if status:
stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
final_status = "approved" if action == "approve" else "rejected"
affected_items: set[str] = set()
async with self._sf() as session:
rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
for row in rows:
row.status = final_status
row.edited_json = (edits or {}).get(row.id)
row.reason = reason
row.reviewed_by = actor
row.reviewed_at = _now()
affected_items.add(row.item_id)
await session.commit()
return sorted(affected_items)
async def pending_review_count(self, item_id: str) -> int:
stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
snapshot = await self.get_snapshot(snapshot_id)
if snapshot is None:
return None
ir = dict(snapshot.get("ir") or {})
reviews = await self.list_reviews(item_id=item_id, status=None)
if not reviews:
return ir
by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
for key in ("entities", "relations"):
kind = key[:-1]
accepted = []
for obj in ir.get(key, []):
review = by_key.get((kind, str(obj.get("local_id"))))
if review and review.get("status") == "rejected":
continue
accepted.append((review or {}).get("edited") or obj)
ir[key] = accepted
return ir
async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
now = _now()
async with self._sf() as session:
item = await session.get(SkillKnowledgeSyncItemRow, item_id)
if item is None:
raise KeyError(item_id)
binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
if binding is None:
raise KeyError(item.binding_id)
if not binding.active:
raise RuntimeError("Binding was detached before staged activation")
await session.execute(
SkillKnowledgeArtifactRow.__table__.update()
.where(
SkillKnowledgeArtifactRow.binding_id == binding.id,
SkillKnowledgeArtifactRow.status == "active",
)
.values(status="retired", updated_at=now)
)
for artifact in artifacts:
session.add(
SkillKnowledgeArtifactRow(
id=str(uuid4()),
binding_id=binding.id,
snapshot_id=snapshot_id,
kind=str(artifact.get("kind") or "wiki_page"),
slug=artifact.get("slug"),
content_text=artifact.get("content"),
remote_id=artifact.get("remote_id"),
business_key=artifact.get("business_key"),
remote_refs_json=artifact.get("remote_refs") or [],
status="active",
metadata_json=artifact.get("metadata") or {},
created_at=now,
updated_at=now,
)
)
# Retire contributions from the old active version only after all staged artifacts are ready.
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
entity_ids: dict[str, str] = {}
for entity in ir.get("entities", []):
normalized = normalize_name(str(entity.get("name") or ""))
if not normalized:
continue
entity_type = str(entity.get("type") or "concept")
stmt = select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == binding.target_type,
SkillKnowledgeEntityRow.target_id == binding.target_id,
SkillKnowledgeEntityRow.entity_type == entity_type,
SkillKnowledgeEntityRow.normalized_name == normalized,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeEntityRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
entity_type=entity_type,
normalized_name=normalized,
display_name=str(entity.get("name") or normalized),
aliases_json=entity.get("aliases") or [],
attributes_json=entity.get("attributes") or {},
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
entity_ids[str(entity.get("local_id"))] = canonical.id
session.add(
SkillKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_entity_id=str(entity.get("local_id")),
evidence_json=entity.get("evidence") or [],
attributes_json=entity.get("attributes") or {},
confidence=float(entity.get("confidence") or 0),
)
)
for relation in ir.get("relations", []):
source_id = entity_ids.get(str(relation.get("source_local_id")))
target_id = entity_ids.get(str(relation.get("target_local_id")))
confidence = float(relation.get("confidence") or 0)
if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
continue
predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
stmt = select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == binding.target_type,
SkillKnowledgeRelationRow.target_id == binding.target_id,
SkillKnowledgeRelationRow.source_entity_id == source_id,
SkillKnowledgeRelationRow.predicate == predicate,
SkillKnowledgeRelationRow.target_entity_id == target_id,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeRelationRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
source_entity_id=source_id,
predicate=predicate,
target_entity_id=target_id,
description=str(relation.get("description") or ""),
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
session.add(
SkillKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_relation_id=str(relation.get("local_id")),
evidence_json=relation.get("evidence") or [],
confidence=confidence,
)
)
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
binding.current_snapshot_id = snapshot_id
binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
binding.synced_digest = binding.source_digest
binding.status = status
binding.last_synced_at = now
binding.last_error_code = None
binding.last_error_message = None
binding.updated_at = now
item.snapshot_id = snapshot_id
item.status = "completed" if status in {"synced", "pending_remote_write"} else status
item.phase = "completed"
item.progress = 100
item.completed_at = now
item.active_dedupe_key = None
item.lease_owner = None
item.lease_until = None
item.updated_at = now
await session.flush()
result = _dict(binding)
await session.commit()
await self.refresh_job(item.job_id)
return result
async def detach_binding(
self,
binding_id: str,
*,
delete_contributions: bool,
cleanup_failed: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
binding = await session.get(SkillKnowledgeBindingRow, binding_id)
if binding is None:
return None
binding.active = False
binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
binding.updated_at = _now()
if delete_contributions:
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
await session.flush()
data = _dict(binding)
await session.commit()
return data
async def mark_skill_stale(self, skill_name: str) -> int:
async with self._sf() as session:
result = await session.execute(
SkillKnowledgeBindingRow.__table__.update()
.where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.active.is_(True),
SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
)
.values(status="stale", updated_at=_now())
)
await session.commit()
return int(result.rowcount or 0)
def make_skill_knowledge_store(session_factory):
return SkillKnowledgeRepository(session_factory) if session_factory is not None else None
@@ -0,0 +1,10 @@
"""Skill knowledge distillation primitives.
The package is deliberately independent from ``app.*``. Gateway composition
injects persistence and remote adapters at runtime.
"""
from deerflow.skill_knowledge.extractor import build_skill_ir
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
@@ -0,0 +1,152 @@
"""IR construction and validation.
V1 intentionally starts with deterministic, evidence-backed extraction. The
schema is compatible with a future model-backed extractor, while never letting
unvalidated model output flow directly into persistence or graph writers.
"""
from __future__ import annotations
import hashlib
import re
from typing import Any
from deerflow.skill_knowledge.scanner import ScanResult
EXTRACTOR_VERSION = "skill-knowledge-v1"
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
PREDICATE_PATTERNS = {
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
}
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
line_start = text.count("\n", 0, start) + 1
line_end = text.count("\n", 0, end) + 1
quote = " ".join(text[start:end].split())[:240]
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
entities: list[dict[str, Any]] = [
{
"local_id": "ent_skill",
"type": "skill",
"name": display_name or scan.skill_name,
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
"description": description,
"attributes": {},
"evidence": [],
"confidence": 1.0,
}
]
relations: list[dict[str, Any]] = []
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
for file in scan.files:
if not file.text:
continue
for predicate, pattern in PREDICATE_PATTERNS.items():
for match in pattern.finditer(file.text):
name = match.group(1).strip(" `\"'“”()[]")
if not name or len(name) > 80:
continue
key = ("concept", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "concept",
"name": name,
"aliases": [],
"description": "从技能业务说明中抽取的知识对象",
"attributes": {},
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": predicate,
"target_local_id": entity_id,
"description": " ".join(match.group(0).split())[:240],
"relation_source": "explicit_text",
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
for ref in file.references:
name = ref["target_path"]
key = ("file", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "file",
"name": name,
"aliases": [],
"description": "技能文档引用的资料文件",
"attributes": {},
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": "references",
"target_local_id": entity_id,
"description": f"{file.path} 引用 {name}",
"relation_source": "deterministic_structure",
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
title = display_name or scan.skill_name
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
content = "\n".join(
[
f"# {title}",
"",
description or "由技能知识文件安全归纳生成。",
"",
"## 知识资料",
"",
*(summaries or ["- 暂无可归纳知识文件"]),
"",
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
]
)
return {
"extractor_version": EXTRACTOR_VERSION,
"prompt_hash": PROMPT_HASH,
"model_name": "deterministic-v1",
"skill": {
"name": scan.skill_name,
"display_name": title,
"description": description,
"source_root": scan.source_root,
"source_digest": scan.source_digest,
},
"files": scan.manifest,
"entities": entities,
"relations": relations,
"wiki": {
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
"pages": [],
},
"safety": {"blocked_files": scan.blocked_files},
}
@@ -0,0 +1,115 @@
"""Guarded graph-write plans.
No caller can submit Cypher. The writer accepts only normalized node/edge
records and executes fixed parameterized templates when explicitly enabled.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from typing import Any, Protocol
class GraphTransaction(Protocol):
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
@dataclass(frozen=True, slots=True)
class GraphWritePlan:
scope: str
nodes: tuple[dict[str, Any], ...]
edges: tuple[dict[str, Any], ...]
NODE_QUERY = """
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
SET n.display_name = $display_name, n.entity_type = $entity_type,
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
n.confidence = $confidence, n.updated_at = datetime()
""".strip()
EDGE_QUERY = """
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
SET r.predicate = $predicate, r.binding_id = $binding_id,
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
r.evidence = $evidence, r.updated_at = datetime()
""".strip()
def _key(*parts: str) -> str:
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
def plan_from_ir(
ir: dict[str, Any],
*,
target_type: str,
target_id: str,
binding_id: str,
snapshot_id: str,
job_id: str,
) -> GraphWritePlan:
"""Build a stable write plan; callers cannot supply executable Cypher."""
scope = f"{target_type}:{target_id}"
local_keys: dict[str, str] = {}
nodes: list[dict[str, Any]] = []
for entity in ir.get("entities") or []:
entity_type = str(entity.get("type") or "concept")
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
if not normalized_name:
continue
entity_key = _key(entity_type, normalized_name)
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
nodes.append(
{
"entity_key": entity_key,
"display_name": str(entity.get("name") or normalized_name),
"entity_type": entity_type,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": float(entity.get("confidence") or 1.0),
}
)
edges: list[dict[str, Any]] = []
for relation in ir.get("relations") or []:
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
evidence = list(relation.get("evidence") or [])
confidence = float(relation.get("confidence") or 0)
predicate = str(relation.get("predicate") or "related_to")
if not source_key or not target_key or confidence < 0.75 or not evidence:
continue
edges.append(
{
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
"source_key": source_key,
"target_key": target_key,
"predicate": predicate,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": confidence,
"evidence": evidence,
}
)
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
"""Return Neo4j transactional HTTP statements using only fixed templates."""
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
return statements
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
if not enabled:
return {"nodes": 0, "edges": 0}
for node in plan.nodes:
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
for edge in plan.edges:
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
@@ -0,0 +1,31 @@
"""Portable entity/relation normalization used by both knowledge products."""
from __future__ import annotations
import re
import unicodedata
def normalize_name(value: str) -> str:
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
text = text.strip("`'\"“”‘’《》〈〉[](){}")
text = re.sub(r"\s+", " ", text)
return text
def normalize_predicate(value: str) -> str:
normalized = normalize_name(value).replace(" ", "_")
aliases = {
"依赖": "depends_on",
"需要": "depends_on",
"requires": "depends_on",
"调用": "uses",
"使用": "uses",
"uses": "uses",
"生成": "produces",
"输出": "produces",
"读取": "consumes",
"输入": "consumes",
"引用": "references",
}
return aliases.get(normalized, normalized or "related_to")
@@ -0,0 +1,351 @@
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
from __future__ import annotations
import csv
import hashlib
import json
import re
import zipfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
CODE_SUFFIXES = {
".py",
".pyc",
".pyo",
".ts",
".tsx",
".js",
".jsx",
".mjs",
".cjs",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".psm1",
".java",
".kt",
".kts",
".go",
".rs",
".c",
".cc",
".cpp",
".h",
".hpp",
".cs",
".php",
".rb",
".pl",
".lua",
".swift",
".scala",
".sql",
".vue",
".svelte",
}
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
EXCLUDED_FILENAMES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"bun.lockb",
"poetry.lock",
"uv.lock",
"pipfile.lock",
"cargo.lock",
"composer.lock",
}
SENSITIVE_NAME_RE = re.compile(
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
re.IGNORECASE,
)
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
MAX_TEXT_BYTES = 2 * 1024 * 1024
MAX_ARCHIVE_FILES = 1_000
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
@dataclass(slots=True)
class ScannedFile:
path: str
kind: str
sha256: str
size: int
text: str = ""
title: str = ""
references: list[dict[str, Any]] = field(default_factory=list)
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
def manifest(self) -> dict[str, Any]:
return {
"path": self.path,
"kind": self.kind,
"sha256": self.sha256,
"size": self.size,
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
"title": self.title,
"references": self.references,
"safety": self.safety,
}
@dataclass(slots=True)
class ScanResult:
skill_name: str
source_root: str
source_digest: str
files: list[ScannedFile]
blocked_files: list[dict[str, str]]
skipped_code_count: int
skipped_other_count: int
@property
def manifest(self) -> list[dict[str, Any]]:
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
@property
def counts(self) -> dict[str, int]:
return {
"knowledge_files": len(self.files),
"blocked_files": len(self.blocked_files),
"skipped_code_files": self.skipped_code_count,
"skipped_other_files": self.skipped_other_count,
}
def _safe_relative(path: Path, root: Path) -> str:
resolved = path.resolve()
resolved_root = root.resolve()
try:
relative = resolved.relative_to(resolved_root)
except ValueError as exc:
raise ValueError("Skill file escapes its source root") from exc
return relative.as_posix()
def _strip_markdown_code(text: str) -> str:
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
def _read_text(path: Path, suffix: str) -> str:
raw = path.read_bytes()
if len(raw) > MAX_TEXT_BYTES:
raise ValueError("text file exceeds 2 MiB parsing limit")
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
return _strip_markdown_code(text)
if suffix in {".csv", ".tsv"}:
delimiter = "\t" if suffix == ".tsv" else ","
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
if suffix == ".json":
try:
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
except json.JSONDecodeError:
return text
return text
def _title(text: str, fallback: str) -> str:
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
return match.group(1).strip() if match else fallback
def _kind(suffix: str) -> str:
return {
".md": "markdown",
".mdx": "markdown",
".txt": "text",
".csv": "table",
".tsv": "table",
".xlsx": "spreadsheet",
".xls": "spreadsheet",
".pdf": "pdf",
".docx": "office",
".pptx": "office",
".png": "image",
".jpg": "image",
".jpeg": "image",
".webp": "image",
".gif": "image",
".bmp": "image",
".zip": "archive",
}.get(suffix, "structured_data")
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
with zipfile.ZipFile(path) as archive:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_FILES:
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
total_size = sum(member.file_size for member in members)
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
for member in members:
member_path = Path(member.filename.replace("\\", "/"))
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
unix_mode = member.external_attr >> 16
if member.is_dir():
continue
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
continue
suffix = member_path.suffix.lower()
lower_name = member_path.name.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
skipped_other += 1
continue
raw = archive.read(member)
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
continue
files.append(
ScannedFile(
path=virtual_path,
kind=_kind(suffix),
sha256=hashlib.sha256(raw).hexdigest(),
size=len(raw),
text=text,
title=_title(text, member_path.stem),
)
)
return files, blocked, skipped_code, skipped_other
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
"""Scan one already-resolved skill directory without following symlinks."""
root = skill_root.resolve()
if not root.is_dir():
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
if path.is_symlink() or not path.is_file():
continue
relative = _safe_relative(path, root)
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
continue
lower_name = path.name.lower()
suffix = path.suffix.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
skipped_other += 1
continue
# Never upload an archive wholesale: it may contain excluded code or a
# blocked secret. Only independently validated knowledge members enter
# the manifest and digest.
if suffix == ".zip":
try:
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
files.extend(nested)
blocked.extend(nested_blocked)
skipped_code += nested_code
skipped_other += nested_other
except (OSError, zipfile.BadZipFile):
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
continue
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
text = ""
warnings: list[str] = []
if suffix in TEXT_SUFFIXES:
try:
text = _read_text(path, suffix)
except ValueError as exc:
warnings.append(str(exc))
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
continue
references = []
if suffix in {".md", ".mdx"}:
for target in MARKDOWN_LINK_RE.findall(text):
normalized = target.replace("\\", "/").lstrip("./")
if normalized:
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
files.append(
ScannedFile(
path=relative,
kind=_kind(suffix),
sha256=digest,
size=len(raw),
text=text,
title=_title(text, path.stem),
references=references,
safety={"status": "allowed", "warnings": warnings},
)
)
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return ScanResult(
skill_name=skill_name,
source_root=str(root),
source_digest=source_digest,
files=files,
blocked_files=blocked,
skipped_code_count=skipped_code,
skipped_other_count=skipped_other,
)
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
allowed: list[ScannedFile] = []
root = Path(scan.source_root)
for item in scan.files:
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
text = await extract_file_to_markdown(root / item.path)
if text:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
continue
item.text = text
item.title = _title(text, item.title)
else:
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
allowed.append(item)
scan.files = allowed
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return scan
@@ -167,6 +167,24 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
return None
async def extract_file_to_markdown(file_path: Path) -> str | None:
"""Extract a supported document to Markdown without creating a sidecar file.
Knowledge-ingest scanners use this variant because running a read-only
distillation task must never add generated ``.md`` files to a live skill.
Conversion always runs off the event loop; Office/PDF parsers can be CPU
intensive even for deceptively small inputs.
"""
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
try:
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
except Exception as exc:
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
return None
# Regex for bold-only lines that look like section headings.
# Targets SEC filing structural headings that pymupdf4llm renders as **bold**
# rather than # Markdown headings (because they use same font size as body text,
@@ -0,0 +1,96 @@
import pytest
import pytest_asyncio
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
)
from deerflow.persistence.base import Base
@pytest_asyncio.fixture
async def repository():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
yield AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
await engine.dispose()
@pytest.mark.asyncio
async def test_single_base_entity_alignment_search_and_revision(repository: AssistantKnowledgeRepository) -> None:
first = await repository.initialize(created_by="admin")
second = await repository.initialize(created_by="someone-else")
assert first["id"] == second["id"]
package = {
"source": {"type": "skill"},
"wiki_pages": [{"slug": "skills/demo/overview", "title": "示例知识", "content": "张三负责测试。"}],
"entities": [{"local_id": "e1", "type": "person", "name": "张三", "aliases": ["Zhang San"]}],
"relations": [],
"documents": [],
"chunks": [],
"graph": {"nodes": [], "edges": []},
}
await repository.import_package(
package=package,
source_type="skill",
source_key="demo",
source_name="示例技能",
trigger="skill_direct",
created_by="admin",
skill_name="demo",
)
await repository.import_package(
package=package,
source_type="weknora",
source_key="kb-1",
source_name="普通库",
trigger="manual_from_weknora",
created_by="admin",
)
results = await repository.list_pages(query="张三")
alias_results = await repository.list_pages(query="Zhang San")
revisions = await repository.list_revisions("skills/demo/overview")
assert {row["page_type"] for row in results} == {"source", "entity"}
assert [row["page_type"] for row in alias_results] == ["entity"]
assert len(revisions) == 2
@pytest.mark.asyncio
async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
repository: AssistantKnowledgeRepository,
) -> None:
package = {
"source": {},
"wiki_pages": [],
"entities": [{"local_id": "e1", "type": "concept", "name": "共享实体"}],
"relations": [],
}
for source_key in ("one", "two"):
await repository.import_package(
package=package,
source_type="skill",
source_key=source_key,
source_name=source_key,
trigger="skill_direct",
created_by="admin",
skill_name=source_key,
)
sources = await repository.list_sources()
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "one"))
async with repository._sf() as session:
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow))).scalars().all()
assert entity.status == "active"
assert len(contributions) == 1
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "two"))
async with repository._sf() as session:
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
assert entity.status == "deleted"
@@ -0,0 +1,62 @@
import pytest
from deerflow.skill_knowledge.graph_writer import (
EDGE_QUERY,
NODE_QUERY,
execute_plan,
parameterized_statements,
plan_from_ir,
)
class RecordingTransaction:
def __init__(self) -> None:
self.calls: list[tuple[str, dict]] = []
async def run(self, query: str, parameters: dict) -> None:
self.calls.append((query, parameters))
def _plan():
return plan_from_ir(
{
"entities": [
{"local_id": "a", "type": "person", "name": "张三", "confidence": 1},
{"local_id": "b", "type": "system", "name": "知识库", "confidence": 1},
],
"relations": [
{
"source_local_id": "a",
"target_local_id": "b",
"predicate": "uses",
"confidence": 0.9,
"evidence": [{"file": "SKILL.md", "line": 2}],
}
],
},
target_type="weknora",
target_id="kb-1",
binding_id="binding-1",
snapshot_id="snapshot-1",
job_id="job-1",
)
def test_direct_graph_plan_uses_only_fixed_parameterized_templates() -> None:
statements = parameterized_statements(_plan())
assert len(statements) == 3
assert {row["statement"] for row in statements} == {NODE_QUERY, EDGE_QUERY}
assert all("parameters" in row for row in statements)
assert all("张三" not in row["statement"] for row in statements)
@pytest.mark.asyncio
async def test_direct_graph_writer_obeys_explicit_enable_switch() -> None:
transaction = RecordingTransaction()
disabled = await execute_plan(transaction, _plan(), enabled=False)
assert disabled == {"nodes": 0, "edges": 0}
assert transaction.calls == []
enabled = await execute_plan(transaction, _plan(), enabled=True)
assert enabled == {"nodes": 2, "edges": 1}
assert len(transaction.calls) == 3
@@ -0,0 +1,95 @@
import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.base import Base
from deerflow.persistence.skill_knowledge import SkillKnowledgeRepository
@pytest_asyncio.fixture
async def repository():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
yield SkillKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
await engine.dispose()
@pytest.mark.asyncio
async def test_matrix_job_creates_one_item_and_binding_per_pair(repository: SkillKnowledgeRepository) -> None:
job = await repository.create_job(
skill_names=["a", "b", "c"],
targets=[
{"target_type": "weknora", "target_mode": "wiki", "target_id": "kb-1", "target_name": "一"},
{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"},
],
created_by="admin",
)
detail = await repository.get_job(job["id"])
bindings = await repository.list_bindings()
assert detail is not None
assert detail["total_count"] == 6
assert len(detail["items"]) == 6
assert len(bindings) == 6
@pytest.mark.asyncio
async def test_detach_delete_only_removes_binding_contributions(repository: SkillKnowledgeRepository) -> None:
job = await repository.create_job(
skill_names=["a", "b"],
targets=[{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"}],
created_by="admin",
review_mode="off",
)
detail = await repository.get_job(job["id"])
assert detail is not None
binding = (await repository.list_bindings(skill_names=["a"]))[0]
detached = await repository.detach_binding(binding["id"], delete_contributions=True)
assert detached and detached["status"] == "detached_deleted"
assert len(await repository.list_bindings()) == 1
@pytest.mark.asyncio
async def test_item_claim_is_atomic_and_records_worker_lease(repository: SkillKnowledgeRepository) -> None:
job = await repository.create_job(
skill_names=["a"],
targets=[
{
"target_type": "assistant",
"target_mode": "assistant",
"target_id": "base",
"target_name": "知识梳理",
}
],
created_by="admin",
)
detail = await repository.get_job(job["id"])
item_id = detail["items"][0]["id"]
first = await repository.claim_item(item_id, owner="worker-a")
second = await repository.claim_item(item_id, owner="worker-b")
assert first and first["lease_owner"] == "worker-a"
assert first["attempts"] == 1
assert second is None
@pytest.mark.asyncio
async def test_binding_allows_only_one_active_item(repository: SkillKnowledgeRepository) -> None:
target = {
"target_type": "assistant",
"target_mode": "assistant",
"target_id": "base",
"target_name": "知识梳理",
}
first = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
duplicate = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
assert first["total_count"] == 1
assert duplicate["total_count"] == 0
assert duplicate["status"] == "completed"
detail = await repository.get_job(first["id"])
await repository.update_item(detail["items"][0]["id"], status="failed")
retry = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
assert retry["total_count"] == 1
@@ -0,0 +1,52 @@
from pathlib import Path
from zipfile import ZipFile
from deerflow.skill_knowledge import build_skill_ir, scan_skill
def test_scanner_excludes_code_blocks_source_and_sensitive_files(tmp_path: Path) -> None:
(tmp_path / "SKILL.md").write_text(
"# 示例\n本技能使用 客户画像数据。\n```python\nPASSWORD = 'must-not-enter-ir'\n```\n",
encoding="utf-8",
)
(tmp_path / "worker.py").write_text("# uses hidden system\nSECRET='never-read'", encoding="utf-8")
(tmp_path / ".env").write_text("TOKEN=super-secret-value", encoding="utf-8")
scan = scan_skill("example", tmp_path)
ir = build_skill_ir(scan)
assert scan.skipped_code_count == 1
assert scan.blocked_files == [{"path": ".env", "reason": "疑似密钥或敏感配置文件"}]
assert "must-not-enter-ir" not in str(ir)
assert "never-read" not in str(ir)
assert any(relation["predicate"] == "uses" for relation in ir["relations"])
def test_code_only_change_does_not_change_digest(tmp_path: Path) -> None:
(tmp_path / "SKILL.md").write_text("# Stable knowledge", encoding="utf-8")
code = tmp_path / "worker.py"
code.write_text("print(1)", encoding="utf-8")
before = scan_skill("example", tmp_path).source_digest
code.write_text("print(2)", encoding="utf-8")
after = scan_skill("example", tmp_path).source_digest
assert before == after
def test_zip_is_inspected_and_only_safe_knowledge_members_are_read(tmp_path: Path) -> None:
archive = tmp_path / "materials.zip"
with ZipFile(archive, "w") as output:
output.writestr("guide.md", "# 操作手册\n仅提炼这段知识。")
output.writestr("worker.py", "SECRET = 'code-must-not-be-read'")
output.writestr("../escape.md", "path traversal")
scan = scan_skill("example", tmp_path)
assert [file.path for file in scan.files] == ["materials.zip!/guide.md"]
assert scan.skipped_code_count == 1
assert scan.blocked_files == [
{
"path": "materials.zip!/../escape.md",
"reason": "压缩包包含路径穿越或符号链接",
}
]
assert "code-must-not-be-read" not in str(scan.manifest)