feat: add skill and assistant knowledge bases
This commit is contained in:
1 parent
dd0ee282df
commit
fbd8dec48f
45 files changed
+5244
-3
No files matched your search
@@ -0,0 +1,7 @@
|
||||
# 技能知识归纳与“知识梳理”助手知识库
|
||||
|
||||
管理员在「技能管理 → 技能知识归纳」中选择一个或多个技能、一个或多个目标,系统按“技能 × 目标”创建后台任务。界面提供总体进度、单项失败重试、实体/关系审核、绑定重归纳、自动同步和两种解绑动作。技能列表通过一次 bindings 查询展示归纳目标,避免逐技能请求。
|
||||
|
||||
扫描仅处理知识资料。代码、脚本、锁文件、密钥文件以及 Markdown 代码块不会进入上传、摘要、提示词或知识指纹;ZIP 只读取逐项验证通过的知识成员。自动监听只比较该知识指纹,所以纯代码改动不会产生 stale 状态。
|
||||
|
||||
知识库页面新增「普通知识库 / 助手知识库」并行入口。管理员可将系统默认入口设为 v1(普通 WeKnora)或 v2(全局“知识梳理”)。助手库无需 WeKnora 即可初始化,所有用户可只读查看和搜索 Wiki;管理员可上传本地文件、从普通库导入、重新导入或删除来源、查看版本并回滚。来源删除只移除对应贡献,共享实体与关系继续保留。
|
||||
@@ -0,0 +1,55 @@
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { BookOpen, Database } from "lucide-react";
|
||||
import { useLocation, useNavigate } from "react-router-dom";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { getStoredAuth } from "@/core/auth";
|
||||
import {
|
||||
loadKnowledgeBaseProductVersion,
|
||||
type KnowledgeBaseProductVersion,
|
||||
updateKnowledgeBaseProductVersion,
|
||||
} from "@/core/system-settings/api";
|
||||
|
||||
export const KNOWLEDGE_PRODUCT_KEY = ["system-settings", "knowledge-base-product-version"] as const;
|
||||
|
||||
export function useKnowledgeBaseProductVersion() {
|
||||
return useQuery({ queryKey: KNOWLEDGE_PRODUCT_KEY, queryFn: loadKnowledgeBaseProductVersion, staleTime: 60_000 });
|
||||
}
|
||||
|
||||
export function KnowledgeProductNav() {
|
||||
const location = useLocation();
|
||||
const navigate = useNavigate();
|
||||
const client = useQueryClient();
|
||||
const version = useKnowledgeBaseProductVersion();
|
||||
const isAdmin = getStoredAuth()?.system_role === "admin";
|
||||
const save = useMutation({
|
||||
mutationFn: updateKnowledgeBaseProductVersion,
|
||||
onSuccess: (value) => {
|
||||
client.setQueryData(KNOWLEDGE_PRODUCT_KEY, value);
|
||||
toast.success(`默认知识库体验已切换为 ${value}`);
|
||||
},
|
||||
});
|
||||
const assistant = location.pathname.includes("/knowledge/assistant");
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 border-b bg-card px-6 py-2">
|
||||
<Button size="sm" variant={assistant ? "ghost" : "secondary"} onClick={() => navigate("/page/workspace/knowledge")}>
|
||||
<BookOpen />普通知识库
|
||||
</Button>
|
||||
<Button size="sm" variant={assistant ? "secondary" : "ghost"} onClick={() => navigate("/page/workspace/knowledge/assistant")}>
|
||||
<Database />助手知识库
|
||||
</Button>
|
||||
{isAdmin && (
|
||||
<div className="ml-auto flex items-center gap-2 text-xs text-muted-foreground">
|
||||
默认产品版本
|
||||
<Select value={version.data ?? "v1"} onValueChange={(value) => save.mutate(value as KnowledgeBaseProductVersion)}>
|
||||
<SelectTrigger className="h-8 w-40"><SelectValue /></SelectTrigger>
|
||||
<SelectContent><SelectItem value="v1">v1 普通知识库</SelectItem><SelectItem value="v2">v2 助手知识库</SelectItem></SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -93,6 +93,7 @@ import {
|
||||
useViewMode,
|
||||
} from "@/core/system-settings/hooks";
|
||||
import type { Skill } from "@/core/skills/type";
|
||||
import type { SkillKnowledgeBinding } from "@/strategy-components/api/skill-knowledge";
|
||||
import { formatDateTime } from "@/core/utils/datetime";
|
||||
import { env } from "@/env";
|
||||
import { desensitize } from "@/lib/desensitize";
|
||||
@@ -601,6 +602,7 @@ export function SkillRow({
|
||||
isAdmin,
|
||||
layout = "row",
|
||||
onPublisherClick,
|
||||
knowledgeBindings = [],
|
||||
}: {
|
||||
skill: Skill;
|
||||
currentUserId: string | null;
|
||||
@@ -610,6 +612,8 @@ export function SkillRow({
|
||||
/** Click the 发布者 line to filter the list to that publisher. Receives the
|
||||
* owner_user_id + its desensitized display label. */
|
||||
onPublisherClick?: (publisher: { id: string; label: string }) => void;
|
||||
/** Admin list can inject its single batched binding query here (never N+1). */
|
||||
knowledgeBindings?: SkillKnowledgeBinding[];
|
||||
}) {
|
||||
const isCard = layout === "card";
|
||||
const { t } = useI18n();
|
||||
@@ -858,6 +862,20 @@ export function SkillRow({
|
||||
常驻
|
||||
</Badge>
|
||||
)}
|
||||
{knowledgeBindings.slice(0, 3).map((binding) => (
|
||||
<Badge
|
||||
key={binding.id}
|
||||
variant="outline"
|
||||
className="border-indigo-500/30 text-indigo-700 dark:text-indigo-300 text-xs"
|
||||
title={`${binding.target_name} · ${binding.status}${binding.remote_business_key ? ` · ${binding.remote_business_key}` : ""}`}
|
||||
>
|
||||
知识库:{binding.target_name || binding.target_id}
|
||||
{binding.status === "stale" ? "(待更新)" : ""}
|
||||
</Badge>
|
||||
))}
|
||||
{knowledgeBindings.length > 3 && (
|
||||
<Badge variant="secondary" className="text-xs">+{knowledgeBindings.length - 3}</Badge>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { CheckCircle2, Database, Loader2, RefreshCw, Sparkles } from "lucide-react";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { ScrollArea } from "@/components/ui/scroll-area";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { useAllSkillsAdmin } from "@/core/skills/hooks";
|
||||
import {
|
||||
useAssistantKnowledgeActions,
|
||||
useAssistantKnowledgeBase,
|
||||
} from "@/strategy-components/api/assistant-knowledge";
|
||||
import { useLlmWikiKnowledgeBases, useLlmWikiRuntime } from "@/strategy-components/api/llmwiki";
|
||||
import {
|
||||
type SkillKnowledgeTarget,
|
||||
useSkillKnowledgeActions,
|
||||
useSkillKnowledgeAutoSyncSettings,
|
||||
useSkillKnowledgeBindings,
|
||||
useSkillKnowledgeJobs,
|
||||
useSkillKnowledgeReviews,
|
||||
} from "@/strategy-components/api/skill-knowledge";
|
||||
|
||||
const PHASE_LABELS: Record<string, string> = {
|
||||
scanning: "扫描知识文件",
|
||||
parsing: "解析资料",
|
||||
extracting: "抽取实体关系",
|
||||
awaiting_review: "等待审核",
|
||||
merging: "合并实体关系",
|
||||
writing_wiki: "写入 Wiki",
|
||||
activating: "激活新版本",
|
||||
completed: "完成",
|
||||
};
|
||||
|
||||
export function SkillKnowledgePanel() {
|
||||
const { skills, isLoading } = useAllSkillsAdmin(true);
|
||||
const liveSkills = useMemo(() => skills.filter((skill) => (skill.state ?? "active") === "active"), [skills]);
|
||||
const [selectedSkills, setSelectedSkills] = useState<string[]>([]);
|
||||
const [selectedTargetKeys, setSelectedTargetKeys] = useState<string[]>([]);
|
||||
const [reviewMode, setReviewMode] = useState<"required" | "auto_high_confidence" | "off">("auto_high_confidence");
|
||||
const runtime = useLlmWikiRuntime();
|
||||
const knowledgeBases = useLlmWikiKnowledgeBases("all", runtime.data?.provider === "weknora");
|
||||
const assistantBase = useAssistantKnowledgeBase();
|
||||
const assistantActions = useAssistantKnowledgeActions();
|
||||
const bindings = useSkillKnowledgeBindings();
|
||||
const jobs = useSkillKnowledgeJobs();
|
||||
const reviews = useSkillKnowledgeReviews();
|
||||
const actions = useSkillKnowledgeActions();
|
||||
const autoSettings = useSkillKnowledgeAutoSyncSettings();
|
||||
const [scheduleCron, setScheduleCron] = useState("0 */6 * * *");
|
||||
const [debounceSeconds, setDebounceSeconds] = useState(120);
|
||||
|
||||
useEffect(() => {
|
||||
if (!autoSettings.data) return;
|
||||
setScheduleCron(autoSettings.data.schedule_cron);
|
||||
setDebounceSeconds(autoSettings.data.debounce_seconds);
|
||||
}, [autoSettings.data]);
|
||||
|
||||
const targets = useMemo<SkillKnowledgeTarget[]>(() => {
|
||||
const rows: SkillKnowledgeTarget[] = (knowledgeBases.data ?? []).map((kb) => ({
|
||||
target_type: "weknora",
|
||||
target_mode: kb.can_write ? kb.type === "faq" ? "faq" : kb.type === "wiki" ? "wiki" : "document" : "readonly_sidecar",
|
||||
target_id: kb.id,
|
||||
target_name: kb.name,
|
||||
remote_business_key: `weknora:kb:${kb.id}`,
|
||||
readonly: !kb.can_write,
|
||||
}));
|
||||
if (assistantBase.data) {
|
||||
rows.unshift({
|
||||
target_type: "assistant",
|
||||
target_mode: "assistant",
|
||||
target_id: assistantBase.data.id,
|
||||
target_name: assistantBase.data.name,
|
||||
remote_business_key: `assistant:base:${assistantBase.data.id}`,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}, [assistantBase.data, knowledgeBases.data]);
|
||||
|
||||
const toggle = (values: string[], value: string, setter: (next: string[]) => void) =>
|
||||
setter(values.includes(value) ? values.filter((item) => item !== value) : [...values, value]);
|
||||
|
||||
const submit = async () => {
|
||||
const selectedTargets = targets.filter((target) =>
|
||||
selectedTargetKeys.includes(`${target.target_type}:${target.target_id}`),
|
||||
);
|
||||
if (!selectedSkills.length || !selectedTargets.length) {
|
||||
toast.error("请至少选择一个技能和一个目标知识库");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const job = await actions.createJob.mutateAsync({
|
||||
skill_names: selectedSkills,
|
||||
targets: selectedTargets,
|
||||
review_mode: reviewMode,
|
||||
});
|
||||
toast.success(`已创建 ${job.total_count} 个归纳项`);
|
||||
setSelectedSkills([]);
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="grid size-full min-h-0 grid-cols-[minmax(320px,0.9fr)_minmax(420px,1.1fr)] gap-4 overflow-hidden p-6">
|
||||
<section className="flex min-h-0 flex-col rounded-lg border bg-card">
|
||||
<div className="border-b p-4">
|
||||
<h2 className="font-semibold">批量归纳</h2>
|
||||
<p className="mt-1 text-xs text-muted-foreground">按“技能 × 目标知识库”生成独立后台任务;代码与脚本不会被读取。</p>
|
||||
</div>
|
||||
<ScrollArea className="min-h-0 flex-1">
|
||||
<div className="space-y-5 p-4">
|
||||
<div>
|
||||
<div className="mb-2 flex items-center justify-between">
|
||||
<span className="text-sm font-medium">1. 选择技能({selectedSkills.length})</span>
|
||||
<Button variant="ghost" size="sm" onClick={() => setSelectedSkills(selectedSkills.length === liveSkills.length ? [] : liveSkills.map((skill) => skill.name))}>全选/清空</Button>
|
||||
</div>
|
||||
<div className="max-h-60 space-y-1 overflow-y-auto rounded-md border p-2">
|
||||
{isLoading ? <Loader2 className="m-4 size-5 animate-spin" /> : liveSkills.map((skill) => (
|
||||
<label key={skill.name} className="flex cursor-pointer items-center gap-2 rounded px-2 py-1.5 hover:bg-muted">
|
||||
<Checkbox checked={selectedSkills.includes(skill.name)} onCheckedChange={() => toggle(selectedSkills, skill.name, setSelectedSkills)} />
|
||||
<span className="truncate text-sm">{skill.name_zh || skill.name}</span>
|
||||
<span className="ml-auto truncate text-xs text-muted-foreground">{skill.name}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="mb-2 flex items-center justify-between">
|
||||
<span className="text-sm font-medium">2. 选择目标({selectedTargetKeys.length})</span>
|
||||
{!assistantBase.data && (
|
||||
<Button size="sm" variant="outline" disabled={assistantActions.initialize.isPending} onClick={() => assistantActions.initialize.mutate("知识梳理")}>创建知识梳理库</Button>
|
||||
)}
|
||||
</div>
|
||||
<div className="space-y-1 rounded-md border p-2">
|
||||
{targets.map((target) => {
|
||||
const key = `${target.target_type}:${target.target_id}`;
|
||||
return (
|
||||
<label key={key} className="flex cursor-pointer items-center gap-2 rounded px-2 py-2 hover:bg-muted">
|
||||
<Checkbox checked={selectedTargetKeys.includes(key)} onCheckedChange={() => toggle(selectedTargetKeys, key, setSelectedTargetKeys)} />
|
||||
<Database className="size-4 text-muted-foreground" />
|
||||
<span className="text-sm">{target.target_name}</span>
|
||||
<Badge variant="outline" className="ml-auto text-[10px]">{target.target_type === "assistant" ? "助手库" : target.target_mode === "readonly_sidecar" ? "只读·本地投影" : target.target_mode}</Badge>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
{!targets.length && <p className="p-3 text-xs text-muted-foreground">未配置普通知识库;可先创建全局“知识梳理”助手库。</p>}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-sm font-medium">审核策略</span>
|
||||
<Select value={reviewMode} onValueChange={(value) => setReviewMode(value as typeof reviewMode)}>
|
||||
<SelectTrigger className="w-52"><SelectValue /></SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="auto_high_confidence">高置信自动通过</SelectItem>
|
||||
<SelectItem value="required">全部逐条审核</SelectItem>
|
||||
<SelectItem value="off">不阻塞写入</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<Button className="w-full" disabled={actions.createJob.isPending} onClick={() => void submit()}>
|
||||
{actions.createJob.isPending ? <Loader2 className="animate-spin" /> : <Sparkles />}
|
||||
开始归纳({selectedSkills.length * selectedTargetKeys.length} 项)
|
||||
</Button>
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</section>
|
||||
|
||||
<section className="flex min-h-0 flex-col gap-4 overflow-hidden">
|
||||
<div className="flex min-h-0 flex-1 flex-col rounded-lg border bg-card">
|
||||
<div className="flex items-center justify-between border-b p-4">
|
||||
<div><h2 className="font-semibold">归纳任务</h2><p className="text-xs text-muted-foreground">后台执行、失败可单项重试</p></div>
|
||||
<Button size="icon" variant="ghost" onClick={() => void jobs.refetch()}><RefreshCw className="size-4" /></Button>
|
||||
</div>
|
||||
<ScrollArea className="min-h-0 flex-1">
|
||||
<div className="space-y-3 p-4">
|
||||
{(jobs.data ?? []).map((job) => {
|
||||
const done = job.success_count + job.partial_failed_count + job.failed_count;
|
||||
const progress = job.total_count ? Math.round((done / job.total_count) * 100) : 0;
|
||||
return (
|
||||
<div key={job.id} className="rounded-md border p-3">
|
||||
<div className="flex items-center gap-2 text-sm"><Badge variant={job.status === "completed" ? "default" : "secondary"}>{job.status}</Badge><span>{done}/{job.total_count} 项</span><span className="ml-auto text-xs text-muted-foreground">{new Date(job.created_at).toLocaleString()}</span></div>
|
||||
<Progress className="my-2" value={progress} />
|
||||
{job.items?.map((item) => (
|
||||
<div key={item.id} className="mt-2 flex items-center gap-2 border-t pt-2 text-xs">
|
||||
<span className="max-w-36 truncate font-medium">{item.skill_name}</span>
|
||||
<span className="text-muted-foreground">{PHASE_LABELS[item.phase] || item.phase}</span>
|
||||
<span className="ml-auto">{item.progress}%</span>
|
||||
{item.status === "failed" && <Button size="sm" variant="outline" onClick={() => actions.retry.mutate(item.id)}>重试</Button>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{!jobs.data?.length && <p className="py-10 text-center text-sm text-muted-foreground">暂无归纳任务</p>}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</div>
|
||||
{(reviews.data?.length ?? 0) > 0 && (
|
||||
<div className="max-h-64 overflow-auto rounded-lg border bg-card p-4">
|
||||
<div className="mb-3 flex items-center"><h2 className="font-semibold">待审核({reviews.data!.length})</h2><Button size="sm" className="ml-auto" onClick={() => actions.review.mutate({ review_ids: reviews.data!.map((row) => row.id), action: "approve" })}><CheckCircle2 />批量批准</Button></div>
|
||||
{reviews.data!.map((review) => <div key={review.id} className="mb-2 flex items-center gap-2 rounded border p-2 text-xs"><Badge variant="outline">{review.object_type}</Badge><code className="truncate">{String(review.original.name || review.original.predicate || review.object_key)}</code><Button size="sm" variant="ghost" className="ml-auto" onClick={() => actions.review.mutate({ review_ids: [review.id], action: "approve" })}>批准</Button><Button size="sm" variant="ghost" onClick={() => actions.review.mutate({ review_ids: [review.id], action: "reject" })}>拒绝</Button></div>)}
|
||||
</div>
|
||||
)}
|
||||
<div className="max-h-80 overflow-auto rounded-lg border bg-card p-3 text-xs">
|
||||
<div className="mb-3 flex flex-wrap items-center gap-3 border-b pb-3">
|
||||
<span className="font-medium">自动同步</span>
|
||||
{autoSettings.data && <>
|
||||
<label className="flex items-center gap-1"><Checkbox checked={autoSettings.data.watch_enabled} onCheckedChange={(checked) => actions.updateAutoSyncSettings.mutate({ watch_enabled: checked === true })} />文件监听</label>
|
||||
<label className="flex items-center gap-1"><Checkbox checked={autoSettings.data.schedule_enabled} onCheckedChange={(checked) => actions.updateAutoSyncSettings.mutate({ schedule_enabled: checked === true })} />定时同步</label>
|
||||
<label className="flex items-center gap-1"><Checkbox checked={autoSettings.data.write_remote_enabled} onCheckedChange={(checked) => actions.updateAutoSyncSettings.mutate({ write_remote_enabled: checked === true })} />自动写入目标</label>
|
||||
<Input className="h-8 w-32" value={scheduleCron} onChange={(event) => setScheduleCron(event.target.value)} aria-label="自动同步 cron" />
|
||||
<Input className="h-8 w-24" type="number" min={10} value={debounceSeconds} onChange={(event) => setDebounceSeconds(Number(event.target.value))} aria-label="防抖秒数" />
|
||||
<Button size="sm" variant="outline" onClick={() => actions.updateAutoSyncSettings.mutate({ schedule_cron: scheduleCron, debounce_seconds: debounceSeconds }, { onSuccess: () => toast.success("自动同步设置已保存"), onError: (error) => toast.error(error.message) })}>保存</Button>
|
||||
</>}
|
||||
</div>
|
||||
<div className="mb-2 text-muted-foreground">当前有效绑定:{bindings.data?.length ?? 0};同名同类型实体按来源贡献合并。</div>
|
||||
<div className="space-y-2">{bindings.data?.map((binding) => <div key={binding.id} className="flex flex-wrap items-center gap-2 rounded border p-2"><span className="max-w-32 truncate font-medium">{binding.skill_name}</span><span className="max-w-40 truncate text-muted-foreground">→ {binding.target_name}</span><Badge variant="outline">{binding.status}</Badge><code className="max-w-40 truncate text-[10px] text-muted-foreground">{binding.remote_business_key}</code><label className="ml-auto flex items-center gap-1"><Checkbox checked={binding.auto_sync_enabled} onCheckedChange={(checked) => actions.setAutoSync.mutate({ id: binding.id, enabled: checked === true })} />自动</label><Button size="sm" variant="outline" disabled={actions.resync.isPending} onClick={() => actions.resync.mutate(binding.id)}>重归纳</Button><Button size="sm" variant="ghost" onClick={() => { if (window.confirm("解除关联并保留目标知识内容?")) actions.detach.mutate({ id: binding.id, deleteContributions: false }); }}>解绑并保留</Button><Button size="sm" variant="destructive" onClick={() => { if (window.confirm("解除关联并删除此技能贡献?共享知识仍会保留其他来源。")) actions.detach.mutate({ id: binding.id, deleteContributions: true }); }}>解绑并删除贡献</Button></div>)}</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -8,6 +8,31 @@ export interface PromptPrefixSettings {
|
||||
prompt_prefix: string;
|
||||
}
|
||||
|
||||
export type KnowledgeBaseProductVersion = "v1" | "v2";
|
||||
|
||||
export async function loadKnowledgeBaseProductVersion(): Promise<KnowledgeBaseProductVersion> {
|
||||
const response = await apiFetch(
|
||||
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
|
||||
);
|
||||
if (!response.ok) throw new Error(`Failed to load knowledge base product version: ${response.status}`);
|
||||
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
|
||||
}
|
||||
|
||||
export async function updateKnowledgeBaseProductVersion(
|
||||
version: KnowledgeBaseProductVersion,
|
||||
): Promise<KnowledgeBaseProductVersion> {
|
||||
const response = await apiFetch(
|
||||
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
|
||||
{
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ version }),
|
||||
},
|
||||
);
|
||||
if (!response.ok) throw new Error(`Failed to update knowledge base product version: ${response.status}`);
|
||||
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
|
||||
}
|
||||
|
||||
export interface PluginDomainAgentMappingDto {
|
||||
id: string;
|
||||
domain: string;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import {
|
||||
ChevronLeftIcon,
|
||||
ChevronRightIcon,
|
||||
DatabaseZap,
|
||||
Replace,
|
||||
SearchIcon,
|
||||
SparklesIcon,
|
||||
@@ -18,9 +19,11 @@ import {
|
||||
WorkspaceHeader,
|
||||
} from "@/components/workspace/workspace-container";
|
||||
import { SkillRow } from "@/components/workspace/settings/skill-settings-page";
|
||||
import { SkillKnowledgePanel } from "@/components/workspace/skills/SkillKnowledgePanel";
|
||||
import { getStoredAuth } from "@/core/auth";
|
||||
import { useAllSkillsAdmin } from "@/core/skills/hooks";
|
||||
import { desensitize } from "@/lib/desensitize";
|
||||
import { useSkillKnowledgeBindings } from "@/strategy-components/api/skill-knowledge";
|
||||
import { SkillAddressesPanel } from "./SkillAddressesPage";
|
||||
|
||||
const PAGE_SIZE = 10;
|
||||
@@ -29,6 +32,7 @@ export default function AdminSkillsPage() {
|
||||
const [search, setSearch] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
const { skills, isLoading, error } = useAllSkillsAdmin(true);
|
||||
const knowledgeBindings = useSkillKnowledgeBindings();
|
||||
|
||||
const storedAuth = useMemo(() => getStoredAuth(), []);
|
||||
const currentUserId = storedAuth?.user_id ?? null;
|
||||
@@ -71,6 +75,10 @@ export default function AdminSkillsPage() {
|
||||
<Replace />
|
||||
技能地址管理
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="knowledge">
|
||||
<DatabaseZap />
|
||||
技能知识归纳
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
</div>
|
||||
</div>
|
||||
@@ -126,6 +134,9 @@ export default function AdminSkillsPage() {
|
||||
skill={skill}
|
||||
currentUserId={currentUserId}
|
||||
isAdmin
|
||||
knowledgeBindings={(knowledgeBindings.data ?? []).filter(
|
||||
(binding) => binding.skill_name === skill.name,
|
||||
)}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
@@ -164,6 +175,12 @@ export default function AdminSkillsPage() {
|
||||
>
|
||||
<SkillAddressesPanel />
|
||||
</TabsContent>
|
||||
<TabsContent
|
||||
value="knowledge"
|
||||
className="mt-0 min-h-0 flex-1 overflow-hidden"
|
||||
>
|
||||
<SkillKnowledgePanel />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</WorkspaceBody>
|
||||
</WorkspaceContainer>
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { Database, FileUp, History, Loader2, RefreshCw, Search, ShieldCheck, Trash2 } from "lucide-react";
|
||||
import { useMemo, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { ScrollArea } from "@/components/ui/scroll-area";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
|
||||
import { KnowledgeProductNav } from "@/components/workspace/knowledge-product-nav";
|
||||
import { getStoredAuth } from "@/core/auth";
|
||||
import {
|
||||
listAssistantWikiRevisions,
|
||||
type AssistantWikiPage,
|
||||
type AssistantWikiRevision,
|
||||
useAssistantImportJobs,
|
||||
useAssistantKnowledgeActions,
|
||||
useAssistantKnowledgeBase,
|
||||
useAssistantKnowledgeSources,
|
||||
useAssistantWikiPages,
|
||||
} from "@/strategy-components/api/assistant-knowledge";
|
||||
|
||||
export default function AssistantKnowledgePage() {
|
||||
const isAdmin = getStoredAuth()?.system_role === "admin";
|
||||
const [query, setQuery] = useState("");
|
||||
const [selected, setSelected] = useState<AssistantWikiPage | null>(null);
|
||||
const [revisions, setRevisions] = useState<AssistantWikiRevision[]>([]);
|
||||
const fileRef = useRef<HTMLInputElement>(null);
|
||||
const base = useAssistantKnowledgeBase();
|
||||
const pages = useAssistantWikiPages(query);
|
||||
const sources = useAssistantKnowledgeSources();
|
||||
const jobs = useAssistantImportJobs();
|
||||
const actions = useAssistantKnowledgeActions();
|
||||
const activePage = selected ?? pages.data?.[0] ?? null;
|
||||
|
||||
const counts = useMemo(() => ({ pages: pages.data?.length ?? 0, sources: sources.data?.length ?? 0, jobs: jobs.data?.length ?? 0 }), [pages.data, sources.data, jobs.data]);
|
||||
|
||||
if (base.isLoading) return <div className="flex size-full flex-col"><KnowledgeProductNav /><div className="flex flex-1 items-center justify-center"><Loader2 className="animate-spin" /></div></div>;
|
||||
if (!base.data) {
|
||||
return <div className="flex size-full flex-col"><KnowledgeProductNav /><div className="flex flex-1 items-center justify-center p-8"><div className="max-w-lg rounded-xl border bg-card p-8 text-center"><Database className="mx-auto mb-4 size-10 text-primary" /><h1 className="text-xl font-semibold">创建“知识梳理”助手知识库</h1><p className="mt-2 text-sm text-muted-foreground">无需配置普通知识库服务。创建后所有用户可查看 Wiki 与搜索,只有管理员可导入和维护。</p>{isAdmin ? <Button className="mt-5" disabled={actions.initialize.isPending} onClick={() => actions.initialize.mutate("知识梳理")}>创建知识梳理库</Button> : <p className="mt-5 text-sm">请联系管理员初始化。</p>}</div></div></div>;
|
||||
}
|
||||
|
||||
const uploadFiles = async (files: FileList | null) => {
|
||||
if (!files?.length) return;
|
||||
try {
|
||||
await actions.upload.mutateAsync(Array.from(files));
|
||||
toast.success("文件已安全扫描并导入");
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : String(error));
|
||||
} finally {
|
||||
if (fileRef.current) fileRef.current.value = "";
|
||||
}
|
||||
};
|
||||
|
||||
const openHistory = async () => {
|
||||
if (!activePage) return;
|
||||
setRevisions(await listAssistantWikiRevisions(activePage.slug));
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex size-full min-h-0 flex-col bg-background">
|
||||
<KnowledgeProductNav />
|
||||
<header className="flex items-center gap-4 border-b px-6 py-4"><div><h1 className="text-xl font-semibold">{base.data.name}</h1><p className="text-xs text-muted-foreground">全局助手知识库 · 普通用户只读</p></div><Badge variant="outline"><ShieldCheck className="mr-1 size-3" />实体对齐与版本管理</Badge><div className="ml-auto flex gap-2">{isAdmin && <><input ref={fileRef} className="hidden" type="file" multiple onChange={(event) => void uploadFiles(event.target.files)} /><Button variant="outline" onClick={() => fileRef.current?.click()}><FileUp />导入本地文件</Button></>}<Button variant="outline" disabled={!activePage} onClick={() => void openHistory()}><History />页面版本</Button></div></header>
|
||||
<Tabs defaultValue="wiki" className="flex min-h-0 flex-1 flex-col">
|
||||
<div className="flex items-center gap-4 border-b px-6"><TabsList className="my-2"><TabsTrigger value="wiki">Wiki({counts.pages})</TabsTrigger><TabsTrigger value="sources">来源({counts.sources})</TabsTrigger><TabsTrigger value="jobs">导入记录({counts.jobs})</TabsTrigger><TabsTrigger value="revisions">版本({revisions.length})</TabsTrigger></TabsList><div className="relative ml-auto w-72"><Search className="absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" /><Input value={query} onChange={(event) => setQuery(event.target.value)} placeholder="搜索标题、实体别名和正文" className="pl-8" /></div></div>
|
||||
<TabsContent value="wiki" className="mt-0 grid min-h-0 flex-1 grid-cols-[300px_1fr]"><ScrollArea className="border-r"><div className="space-y-1 p-3">{pages.data?.map((page) => <button key={page.id} type="button" onClick={() => setSelected(page)} className={`w-full rounded-md px-3 py-2 text-left ${activePage?.id === page.id ? "bg-primary/10 text-primary" : "hover:bg-muted"}`}><div className="truncate text-sm font-medium">{page.title}</div><div className="mt-1 flex items-center gap-2 text-[11px] text-muted-foreground"><span>{page.page_type}</span><span>{new Date(page.updated_at).toLocaleString()}</span></div></button>)}</div></ScrollArea><ScrollArea><article className="mx-auto max-w-5xl p-8">{activePage ? <><h1 className="mb-1 text-2xl font-semibold">{activePage.title}</h1><p className="mb-6 text-xs text-muted-foreground">{activePage.slug}</p><MarkdownContent content={activePage.content_markdown} isLoading={false} rehypePlugins={[]} /></> : <p className="text-sm text-muted-foreground">暂无 Wiki 页面。</p>}</article></ScrollArea></TabsContent>
|
||||
<TabsContent value="sources" className="mt-0 min-h-0 flex-1 overflow-auto p-6"><div className="grid gap-3 md:grid-cols-2 xl:grid-cols-3">{sources.data?.map((source) => <div key={source.id} className="rounded-lg border bg-card p-4"><div className="flex items-center gap-2"><Database className="size-4" /><span className="font-medium">{source.source_name}</span><Badge variant="outline" className="ml-auto">{source.source_type}</Badge></div><p className="mt-3 text-xs text-muted-foreground">业务键:{source.source_key}</p>{isAdmin && <div className="mt-4 flex justify-end gap-2"><Button size="sm" variant="outline" disabled={source.source_type === "local_file" || actions.reimportSource.isPending} onClick={() => actions.reimportSource.mutate(source.id, { onSuccess: () => toast.success("已发起重新导入"), onError: (error) => toast.error(error.message) })}><RefreshCw />重新导入</Button><Button size="sm" variant="destructive" disabled={actions.deleteSource.isPending} onClick={() => { if (window.confirm(`确认删除来源“${source.source_name}”的贡献?仍被其他来源引用的知识会保留。`)) actions.deleteSource.mutate(source.id, { onSuccess: () => toast.success("来源贡献已删除"), onError: (error) => toast.error(error.message) }); }}><Trash2 />删除来源</Button></div>}</div>)}</div></TabsContent>
|
||||
<TabsContent value="jobs" className="mt-0 min-h-0 flex-1 overflow-auto p-6"><div className="space-y-3">{jobs.data?.map((job) => <div key={job.id} className="flex items-center gap-3 rounded-lg border bg-card p-4"><Badge>{job.status}</Badge><span className="text-sm">{job.trigger}</span><span className="text-xs text-muted-foreground">{Object.entries(job.counts || {}).map(([key, value]) => `${key} ${value}`).join(" · ")}</span><span className="ml-auto text-xs">{new Date(job.created_at).toLocaleString()}</span></div>)}</div></TabsContent>
|
||||
<TabsContent value="revisions" className="mt-0 min-h-0 flex-1 overflow-auto p-6"><div className="space-y-3">{revisions.map((revision) => <div key={revision.id} className="flex items-center gap-3 rounded-lg border bg-card p-4"><Badge variant="outline">v{revision.revision_no}</Badge><span className="text-sm">{revision.change_type}</span><span className="text-xs text-muted-foreground">{revision.summary}</span><span className="ml-auto text-xs">{new Date(revision.created_at).toLocaleString()}</span>{isAdmin && activePage && revision.id !== activePage.current_revision_id && <Button size="sm" variant="outline" onClick={() => actions.rollback.mutate({ slug: activePage.slug, revisionId: revision.id })}>回滚到此版本</Button>}</div>)}{!revisions.length && <p className="text-sm text-muted-foreground">请先在 Wiki 页选择页面并点击“页面版本”。</p>}</div></TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { useState } from "react";
|
||||
import { Navigate, useParams, useSearchParams } from "react-router-dom";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { KnowledgeProductNav, useKnowledgeBaseProductVersion } from "@/components/workspace/knowledge-product-nav";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
@@ -70,6 +71,7 @@ export default function LlmWikiKnowledgePage({
|
||||
const { noteId = "" } = useParams();
|
||||
const [searchParams] = useSearchParams();
|
||||
const runtime = useLlmWikiRuntime();
|
||||
const productVersion = useKnowledgeBaseProductVersion();
|
||||
const wikiSlug = searchParams.get("wikiSlug")?.trim() || "";
|
||||
|
||||
if (mode === "detail" && noteId && wikiSlug) {
|
||||
@@ -80,11 +82,15 @@ export default function LlmWikiKnowledgePage({
|
||||
return <RuntimeLoading />;
|
||||
}
|
||||
|
||||
if (mode === "list" && productVersion.data === "v2") {
|
||||
return <Navigate to="/page/workspace/knowledge/assistant" replace />;
|
||||
}
|
||||
|
||||
if (runtime.isError || runtime.data?.provider !== "weknora") {
|
||||
if (mode === "graph") {
|
||||
return <KnowledgeGraphPage />;
|
||||
}
|
||||
return <LegacyKnowledgePage />;
|
||||
return mode === "list" ? <div className="flex size-full min-h-0 flex-col"><KnowledgeProductNav /><div className="min-h-0 flex-1"><LegacyKnowledgePage /></div></div> : <LegacyKnowledgePage />;
|
||||
}
|
||||
|
||||
if (mode === "graph") {
|
||||
@@ -95,5 +101,5 @@ export default function LlmWikiKnowledgePage({
|
||||
return <WeKnoraKnowledgeDetailPage />;
|
||||
}
|
||||
|
||||
return <WeKnoraKnowledgeWorkspace />;
|
||||
return <div className="flex size-full min-h-0 flex-col"><KnowledgeProductNav /><div className="min-h-0 flex-1"><WeKnoraKnowledgeWorkspace /></div></div>;
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import AdminSkillsPage from './AdminSkillsPage';
|
||||
import ScheduledTaskRunDetailPage from './ScheduledTaskRunDetailPage';
|
||||
import ScheduledTasksPage from './ScheduledTasksPage';
|
||||
import LlmWikiKnowledgePage from './LlmWikiKnowledgePage';
|
||||
import AssistantKnowledgePage from './AssistantKnowledgePage';
|
||||
import WeKnoraWikiManagementPage from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage';
|
||||
import WeKnoraWikiPageView from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView';
|
||||
import MarkdownDocxEditorPage from './MarkdownDocxEditorPage';
|
||||
@@ -689,6 +690,14 @@ export default function WorkspaceRoutes({
|
||||
</WorkspaceLayout>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="knowledge/assistant"
|
||||
element={
|
||||
<WorkspaceLayout {...layoutProps}>
|
||||
<AssistantKnowledgePage />
|
||||
</WorkspaceLayout>
|
||||
}
|
||||
/>
|
||||
<Route
|
||||
path="knowledge/:noteId/wiki/view"
|
||||
element={
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
import { apiFetch } from "@/core/api/fetch-client";
|
||||
import { getBackendBaseURL } from "@/core/config";
|
||||
|
||||
const BASE = () => `${getBackendBaseURL()}/api/assistant-knowledge`;
|
||||
|
||||
export interface AssistantKnowledgeBase {
|
||||
id: string;
|
||||
name: string;
|
||||
status: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface AssistantWikiPage {
|
||||
id: string;
|
||||
slug: string;
|
||||
title: string;
|
||||
page_type: string;
|
||||
content_markdown: string;
|
||||
summary: string;
|
||||
current_revision_id?: string | null;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface AssistantWikiRevision {
|
||||
id: string;
|
||||
page_id: string;
|
||||
revision_no: number;
|
||||
summary: string;
|
||||
content_markdown: string;
|
||||
change_type: string;
|
||||
created_by?: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface AssistantKnowledgeSource {
|
||||
id: string;
|
||||
source_type: "weknora" | "skill" | "local_file";
|
||||
source_name: string;
|
||||
source_key: string;
|
||||
status: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface AssistantImportJob {
|
||||
id: string;
|
||||
source_id: string;
|
||||
trigger: string;
|
||||
status: string;
|
||||
phase: string;
|
||||
counts: Record<string, number>;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
async function readJson<T>(response: Response): Promise<T> {
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => null);
|
||||
const detail = data && typeof data === "object" && "detail" in data
|
||||
? (data as { detail: unknown }).detail
|
||||
: `请求失败 (${response.status})`;
|
||||
throw new Error(typeof detail === "string" ? detail : JSON.stringify(detail));
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
export async function getAssistantKnowledgeBase() {
|
||||
const data = await readJson<{ base: AssistantKnowledgeBase | null }>(await apiFetch(`${BASE()}/base`));
|
||||
return data.base;
|
||||
}
|
||||
|
||||
export async function initializeAssistantKnowledgeBase(name = "知识梳理") {
|
||||
const data = await readJson<{ base: AssistantKnowledgeBase }>(
|
||||
await apiFetch(`${BASE()}/base/initialize`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ name }),
|
||||
}),
|
||||
);
|
||||
return data.base;
|
||||
}
|
||||
|
||||
export async function listAssistantWikiPages(q = "") {
|
||||
const params = new URLSearchParams();
|
||||
if (q.trim()) params.set("q", q.trim());
|
||||
return (await readJson<{ pages: AssistantWikiPage[] }>(await apiFetch(`${BASE()}/wiki-pages?${params}`))).pages;
|
||||
}
|
||||
|
||||
export async function listAssistantKnowledgeSources() {
|
||||
return (await readJson<{ sources: AssistantKnowledgeSource[] }>(await apiFetch(`${BASE()}/sources`))).sources;
|
||||
}
|
||||
|
||||
export async function listAssistantImportJobs() {
|
||||
return (await readJson<{ jobs: AssistantImportJob[] }>(await apiFetch(`${BASE()}/import-jobs`))).jobs;
|
||||
}
|
||||
|
||||
export async function listAssistantWikiRevisions(slug: string) {
|
||||
return (await readJson<{ revisions: AssistantWikiRevision[] }>(
|
||||
await apiFetch(`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions`),
|
||||
)).revisions;
|
||||
}
|
||||
|
||||
export async function rollbackAssistantWikiRevision(slug: string, revisionId: string) {
|
||||
return readJson<AssistantWikiRevision>(
|
||||
await apiFetch(
|
||||
`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions/${encodeURIComponent(revisionId)}/rollback`,
|
||||
{ method: "POST" },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function uploadAssistantKnowledgeFiles(files: File[]) {
|
||||
const form = new FormData();
|
||||
files.forEach((file) => form.append("files", file));
|
||||
return readJson<{ batch_id: string; jobs: AssistantImportJob[] }>(
|
||||
await apiFetch(`${BASE()}/imports/files`, { method: "POST", body: form }),
|
||||
);
|
||||
}
|
||||
|
||||
export async function exportLlmWikiToAssistant(mappingId: string) {
|
||||
return readJson<AssistantImportJob>(
|
||||
await apiFetch(
|
||||
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-to-assistant`,
|
||||
{ method: "POST" },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function reimportAssistantKnowledgeSource(sourceId: string) {
|
||||
return readJson<AssistantImportJob>(
|
||||
await apiFetch(`${BASE()}/sources/${encodeURIComponent(sourceId)}/reimport`, {
|
||||
method: "POST",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function deleteAssistantKnowledgeSource(sourceId: string) {
|
||||
const response = await apiFetch(`${BASE()}/sources/${encodeURIComponent(sourceId)}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
if (!response.ok) await readJson(response);
|
||||
}
|
||||
|
||||
const KEY = "assistant-knowledge";
|
||||
|
||||
export function useAssistantKnowledgeBase() {
|
||||
return useQuery({ queryKey: [KEY, "base"], queryFn: getAssistantKnowledgeBase });
|
||||
}
|
||||
|
||||
export function useAssistantWikiPages(query = "") {
|
||||
return useQuery({ queryKey: [KEY, "pages", query], queryFn: () => listAssistantWikiPages(query) });
|
||||
}
|
||||
|
||||
export function useAssistantKnowledgeSources() {
|
||||
return useQuery({ queryKey: [KEY, "sources"], queryFn: listAssistantKnowledgeSources });
|
||||
}
|
||||
|
||||
export function useAssistantImportJobs() {
|
||||
return useQuery({ queryKey: [KEY, "jobs"], queryFn: listAssistantImportJobs });
|
||||
}
|
||||
|
||||
export function useAssistantKnowledgeActions() {
|
||||
const client = useQueryClient();
|
||||
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
|
||||
return {
|
||||
initialize: useMutation({ mutationFn: initializeAssistantKnowledgeBase, onSuccess: invalidate }),
|
||||
upload: useMutation({ mutationFn: uploadAssistantKnowledgeFiles, onSuccess: invalidate }),
|
||||
reimportSource: useMutation({ mutationFn: reimportAssistantKnowledgeSource, onSuccess: invalidate }),
|
||||
deleteSource: useMutation({ mutationFn: deleteAssistantKnowledgeSource, onSuccess: invalidate }),
|
||||
rollback: useMutation({
|
||||
mutationFn: ({ slug, revisionId }: { slug: string; revisionId: string }) =>
|
||||
rollbackAssistantWikiRevision(slug, revisionId),
|
||||
onSuccess: invalidate,
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
import { apiFetch } from "@/core/api/fetch-client";
|
||||
import { getBackendBaseURL } from "@/core/config";
|
||||
|
||||
const BASE = () => `${getBackendBaseURL()}/api/skill-knowledge`;
|
||||
|
||||
export type SkillKnowledgeTargetType = "weknora" | "assistant";
|
||||
export type SkillKnowledgeTargetMode =
|
||||
| "wiki"
|
||||
| "document"
|
||||
| "faq"
|
||||
| "conversation_archive"
|
||||
| "readonly_sidecar"
|
||||
| "assistant";
|
||||
|
||||
export interface SkillKnowledgeTarget {
|
||||
target_type: SkillKnowledgeTargetType;
|
||||
target_mode: SkillKnowledgeTargetMode;
|
||||
target_id: string;
|
||||
target_name: string;
|
||||
remote_business_key?: string | null;
|
||||
readonly?: boolean;
|
||||
}
|
||||
|
||||
export interface SkillKnowledgeBinding {
|
||||
id: string;
|
||||
skill_name: string;
|
||||
target_type: SkillKnowledgeTargetType;
|
||||
target_mode: SkillKnowledgeTargetMode;
|
||||
target_id: string;
|
||||
target_name: string;
|
||||
remote_business_key?: string | null;
|
||||
remote_refs?: Array<{
|
||||
provider: string;
|
||||
remote_kind: string;
|
||||
remote_id: string;
|
||||
slug?: string;
|
||||
}>;
|
||||
status: string;
|
||||
active: boolean;
|
||||
auto_sync_enabled: boolean;
|
||||
synced_digest?: string | null;
|
||||
last_synced_at?: string | null;
|
||||
last_error_message?: string | null;
|
||||
}
|
||||
|
||||
export interface SkillKnowledgeSyncItem {
|
||||
id: string;
|
||||
job_id: string;
|
||||
binding_id: string;
|
||||
skill_name: string;
|
||||
target_type: SkillKnowledgeTargetType;
|
||||
target_mode: SkillKnowledgeTargetMode;
|
||||
target_id: string;
|
||||
status: string;
|
||||
phase: string;
|
||||
progress: number;
|
||||
attempts: number;
|
||||
last_error_message?: string | null;
|
||||
}
|
||||
|
||||
export interface SkillKnowledgeJob {
|
||||
id: string;
|
||||
status: string;
|
||||
total_count: number;
|
||||
success_count: number;
|
||||
partial_failed_count: number;
|
||||
failed_count: number;
|
||||
review_mode: "required" | "auto_high_confidence" | "off";
|
||||
created_at: string;
|
||||
completed_at?: string | null;
|
||||
items?: SkillKnowledgeSyncItem[];
|
||||
}
|
||||
|
||||
export interface SkillKnowledgeReview {
|
||||
id: string;
|
||||
job_id: string;
|
||||
item_id: string;
|
||||
binding_id: string;
|
||||
object_type: "entity" | "relation" | "wiki" | "alignment";
|
||||
object_key: string;
|
||||
original: Record<string, unknown>;
|
||||
edited?: Record<string, unknown> | null;
|
||||
status: string;
|
||||
reason?: string | null;
|
||||
}
|
||||
|
||||
export interface SkillKnowledgeAutoSyncSettings {
|
||||
watch_enabled: boolean;
|
||||
schedule_enabled: boolean;
|
||||
schedule_cron: string;
|
||||
debounce_seconds: number;
|
||||
write_remote_enabled: boolean;
|
||||
review_mode: "required" | "auto_high_confidence" | "off";
|
||||
}
|
||||
|
||||
async function readJson<T>(response: Response): Promise<T> {
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => null);
|
||||
const detail =
|
||||
data && typeof data === "object" && "detail" in data
|
||||
? JSON.stringify((data as { detail: unknown }).detail)
|
||||
: `请求失败 (${response.status})`;
|
||||
throw new Error(detail.replace(/^"|"$/g, ""));
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
export async function listSkillKnowledgeBindings(
|
||||
skillNames?: string[],
|
||||
): Promise<SkillKnowledgeBinding[]> {
|
||||
const params = new URLSearchParams();
|
||||
for (const name of skillNames ?? []) params.append("skill_names", name);
|
||||
const response = await apiFetch(`${BASE()}/bindings?${params.toString()}`);
|
||||
return (await readJson<{ bindings: SkillKnowledgeBinding[] }>(response)).bindings;
|
||||
}
|
||||
|
||||
export async function createSkillKnowledgeJob(input: {
|
||||
skill_names: string[];
|
||||
targets: SkillKnowledgeTarget[];
|
||||
force?: boolean;
|
||||
review_mode?: "required" | "auto_high_confidence" | "off";
|
||||
}): Promise<SkillKnowledgeJob> {
|
||||
return readJson(
|
||||
await apiFetch(`${BASE()}/sync-jobs`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(input),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function listSkillKnowledgeJobs(): Promise<SkillKnowledgeJob[]> {
|
||||
const response = await apiFetch(`${BASE()}/sync-jobs`);
|
||||
return (await readJson<{ jobs: SkillKnowledgeJob[] }>(response)).jobs;
|
||||
}
|
||||
|
||||
export async function getSkillKnowledgeJob(id: string): Promise<SkillKnowledgeJob> {
|
||||
return readJson(await apiFetch(`${BASE()}/sync-jobs/${encodeURIComponent(id)}`));
|
||||
}
|
||||
|
||||
export async function listSkillKnowledgeReviews(): Promise<SkillKnowledgeReview[]> {
|
||||
const response = await apiFetch(`${BASE()}/reviews?status=pending`);
|
||||
return (await readJson<{ reviews: SkillKnowledgeReview[] }>(response)).reviews;
|
||||
}
|
||||
|
||||
export async function reviewSkillKnowledge(input: {
|
||||
review_ids: string[];
|
||||
action: "approve" | "reject";
|
||||
edits?: Record<string, Record<string, unknown>>;
|
||||
reason?: string;
|
||||
}) {
|
||||
return readJson<{ updated: number; resumed_item_ids: string[] }>(
|
||||
await apiFetch(`${BASE()}/reviews/actions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(input),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function retrySkillKnowledgeItem(itemId: string) {
|
||||
return readJson<SkillKnowledgeSyncItem>(
|
||||
await apiFetch(`${BASE()}/sync-items/${encodeURIComponent(itemId)}/retry`, {
|
||||
method: "POST",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function detachSkillKnowledgeBinding(
|
||||
bindingId: string,
|
||||
deleteContributions: boolean,
|
||||
) {
|
||||
return readJson<SkillKnowledgeBinding>(
|
||||
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/detach`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ delete_contributions: deleteContributions }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function resyncSkillKnowledgeBinding(bindingId: string) {
|
||||
return readJson<SkillKnowledgeJob>(
|
||||
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/resync`, {
|
||||
method: "POST",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function setSkillKnowledgeBindingAutoSync(
|
||||
bindingId: string,
|
||||
enabled: boolean,
|
||||
) {
|
||||
return readJson<SkillKnowledgeBinding>(
|
||||
await apiFetch(
|
||||
`${BASE()}/bindings/${encodeURIComponent(bindingId)}/${enabled ? "enable" : "disable"}-auto-sync`,
|
||||
{ method: "POST" },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function getSkillKnowledgeAutoSyncSettings() {
|
||||
return readJson<SkillKnowledgeAutoSyncSettings>(
|
||||
await apiFetch(`${BASE()}/auto-sync-settings`),
|
||||
);
|
||||
}
|
||||
|
||||
export async function updateSkillKnowledgeAutoSyncSettings(
|
||||
input: Partial<SkillKnowledgeAutoSyncSettings>,
|
||||
) {
|
||||
return readJson<SkillKnowledgeAutoSyncSettings>(
|
||||
await apiFetch(`${BASE()}/auto-sync-settings`, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(input),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const KEY = "skill-knowledge";
|
||||
|
||||
export function useSkillKnowledgeBindings(skillNames?: string[]) {
|
||||
return useQuery({
|
||||
queryKey: [KEY, "bindings", ...(skillNames ?? [])],
|
||||
queryFn: () => listSkillKnowledgeBindings(skillNames),
|
||||
refetchOnWindowFocus: false,
|
||||
});
|
||||
}
|
||||
|
||||
export function useSkillKnowledgeJobs() {
|
||||
return useQuery({
|
||||
queryKey: [KEY, "jobs"],
|
||||
queryFn: listSkillKnowledgeJobs,
|
||||
refetchInterval: (query) =>
|
||||
(query.state.data ?? []).some((job) => ["queued", "running"].includes(job.status))
|
||||
? 1500
|
||||
: false,
|
||||
});
|
||||
}
|
||||
|
||||
export function useSkillKnowledgeReviews() {
|
||||
return useQuery({ queryKey: [KEY, "reviews"], queryFn: listSkillKnowledgeReviews });
|
||||
}
|
||||
|
||||
export function useSkillKnowledgeAutoSyncSettings() {
|
||||
return useQuery({
|
||||
queryKey: [KEY, "auto-sync-settings"],
|
||||
queryFn: getSkillKnowledgeAutoSyncSettings,
|
||||
});
|
||||
}
|
||||
|
||||
export function useSkillKnowledgeActions() {
|
||||
const client = useQueryClient();
|
||||
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
|
||||
return {
|
||||
createJob: useMutation({ mutationFn: createSkillKnowledgeJob, onSuccess: invalidate }),
|
||||
review: useMutation({ mutationFn: reviewSkillKnowledge, onSuccess: invalidate }),
|
||||
retry: useMutation({ mutationFn: retrySkillKnowledgeItem, onSuccess: invalidate }),
|
||||
resync: useMutation({ mutationFn: resyncSkillKnowledgeBinding, onSuccess: invalidate }),
|
||||
setAutoSync: useMutation({
|
||||
mutationFn: ({ id, enabled }: { id: string; enabled: boolean }) =>
|
||||
setSkillKnowledgeBindingAutoSync(id, enabled),
|
||||
onSuccess: invalidate,
|
||||
}),
|
||||
updateAutoSyncSettings: useMutation({
|
||||
mutationFn: updateSkillKnowledgeAutoSyncSettings,
|
||||
onSuccess: invalidate,
|
||||
}),
|
||||
detach: useMutation({
|
||||
mutationFn: ({ id, deleteContributions }: { id: string; deleteContributions: boolean }) =>
|
||||
detachSkillKnowledgeBinding(id, deleteContributions),
|
||||
onSuccess: invalidate,
|
||||
}),
|
||||
};
|
||||
}
|
||||
+21
-1
@@ -1,4 +1,4 @@
|
||||
import { AlertCircle, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
|
||||
import { AlertCircle, Database, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Navigate, useNavigate, useParams, useSearchParams } from "react-router-dom";
|
||||
import { toast } from "sonner";
|
||||
@@ -14,6 +14,7 @@ import { Input } from "@/components/ui/input";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { MarkdownCodeEditor } from "@/components/wiki/MarkdownCodeEditor";
|
||||
import { getBackendBaseURL } from "@/core/config";
|
||||
import { getStoredAuth } from "@/core/auth";
|
||||
import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
|
||||
import {
|
||||
createLlmWikiWeKnoraEmbedSession,
|
||||
@@ -23,6 +24,7 @@ import {
|
||||
useLlmWikiWikiPage,
|
||||
useLlmWikiRuntime,
|
||||
} from "@/strategy-components/api/llmwiki";
|
||||
import { exportLlmWikiToAssistant } from "@/strategy-components/api/assistant-knowledge";
|
||||
|
||||
const LIST_PATH = "/page/workspace/knowledge";
|
||||
|
||||
@@ -104,6 +106,7 @@ export function WeKnoraKnowledgeDetailPage() {
|
||||
const [frameSrc, setFrameSrc] = useState("");
|
||||
const [frameVersion, setFrameVersion] = useState(0);
|
||||
const [frameError, setFrameError] = useState<string | null>(null);
|
||||
const [assistantImporting, setAssistantImporting] = useState(false);
|
||||
const runtime = useLlmWikiRuntime();
|
||||
const details = useLlmWikiKnowledgeBase(knowledgeBaseId || null, !!knowledgeBaseId);
|
||||
|
||||
@@ -255,6 +258,23 @@ export function WeKnoraKnowledgeDetailPage() {
|
||||
)}
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
{getStoredAuth()?.system_role === "admin" && (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
disabled={assistantImporting}
|
||||
onClick={() => {
|
||||
setAssistantImporting(true);
|
||||
void exportLlmWikiToAssistant(knowledgeBaseId)
|
||||
.then(() => toast.success("已导入到助手知识库“知识梳理”"))
|
||||
.catch((error: unknown) => toast.error(error instanceof Error ? error.message : String(error)))
|
||||
.finally(() => setAssistantImporting(false));
|
||||
}}
|
||||
>
|
||||
{assistantImporting ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Database className="mr-2 size-4" />}
|
||||
导入到助手知识库
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
|
||||
@@ -1619,3 +1619,27 @@ write, log the exception, complete the rewrite with
|
||||
`versionSnapshotFailed: true`, and do not emit a user-facing failure or restore
|
||||
the original. The sandbox uses that signal only to enable its ordinary manual
|
||||
save fallback.
|
||||
|
||||
## Skill/assistant knowledge invariants
|
||||
|
||||
Skill distillation is implemented in `deerflow.skill_knowledge` and
|
||||
`deerflow.persistence.skill_knowledge`; the single native assistant base is in
|
||||
`deerflow.assistant_knowledge` and
|
||||
`deerflow.persistence.assistant_knowledge`. Gateway composition, remote adapters,
|
||||
the lease dispatcher, watcher/scheduler, and APIs remain in `app.gateway` so the
|
||||
harness-to-app import firewall is preserved.
|
||||
|
||||
Never execute a skill or read/upload/summarize its code and script files.
|
||||
Knowledge fingerprints exclude those files and Markdown code fences. All
|
||||
resyncs must stage and validate the knowledge digest before activation; the old
|
||||
snapshot remains authoritative on failure. A binding may have only one active
|
||||
item (`active_dedupe_key` plus atomic lease claims). Detach/delete operates on
|
||||
source contributions, not canonical rows, and shared entities/relations are
|
||||
retired only when their final contribution is gone.
|
||||
|
||||
There is exactly one global assistant base (`知识梳理`). Ordinary users have
|
||||
read/search access and administrators own all mutations. WeKnora direct graph
|
||||
writes require `llmwiki.weknora.graph.direct_write_enabled`, backend-only
|
||||
credentials, and the fixed parameterized templates in
|
||||
`deerflow.skill_knowledge.graph_writer`; never accept Cypher from an API, model,
|
||||
skill, or frontend.
|
||||
@@ -953,3 +953,30 @@ The final rewrite commit uses a deliberately short sibling temporary filename,
|
||||
which keeps long Windows artifact paths below the path-length limit. It also
|
||||
recreates a missing `outputs` parent before retrying, so transient
|
||||
sandbox-directory recreation cannot discard a completed rewrite.
|
||||
|
||||
## Skill knowledge distillation and assistant knowledge
|
||||
|
||||
The Gateway now exposes an administrator-only skill distillation pipeline at
|
||||
`/api/skill-knowledge`. A matrix job can send up to 200 skills to multiple
|
||||
WeKnora or global assistant-knowledge targets. The scanner never reads or
|
||||
uploads code/script files, strips Markdown code fences, blocks credential-like
|
||||
files and content, safely inspects ZIP members, and converts supported
|
||||
Office/PDF/spreadsheet files off the event loop. Jobs use database claims and
|
||||
renewable leases, stage snapshots before activation, retain the prior active
|
||||
version on failure, support per-item retry and entity/relation review, and keep
|
||||
per-binding contribution records for safe detach/delete.
|
||||
|
||||
Bindings may opt into debounced file watching and cron synchronization through
|
||||
`/api/skill-knowledge/auto-sync-settings`. Only non-code knowledge digests can
|
||||
mark a binding stale. WeKnora Wiki, document, FAQ, conversation-archive and
|
||||
read-only sidecar targets have separate projections. Optional Neo4j direct
|
||||
writes are disabled by default under `llmwiki.weknora.graph`; when enabled they
|
||||
use only fixed parameterized templates and Neo4j's transactional HTTP endpoint.
|
||||
|
||||
`/api/assistant-knowledge` owns exactly one global DeerFlow-native base named
|
||||
`知识梳理`. Reads and search are available to authenticated users; initialization,
|
||||
local-file import, WeKnora export/import, source reimport/delete, and revision
|
||||
rollback are admin-only. Imports align canonical entities and relations while
|
||||
retaining source contributions, publish source/entity/relation Wiki revisions,
|
||||
and remove canonical knowledge only after its final contribution disappears.
|
||||
The database schema is introduced by Alembic revision `20260901_01`.
|
||||
@@ -32,6 +32,7 @@ from app.gateway.routers import (
|
||||
admin_users,
|
||||
agents,
|
||||
artifacts,
|
||||
assistant_knowledge,
|
||||
assistants_compat,
|
||||
auth,
|
||||
browser_context,
|
||||
@@ -85,6 +86,7 @@ from app.gateway.routers import (
|
||||
scheduled_tasks,
|
||||
sensitive_words,
|
||||
skill_addresses,
|
||||
skill_knowledge,
|
||||
skills,
|
||||
suggestions,
|
||||
system_settings,
|
||||
@@ -1505,6 +1507,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# Skills API is mounted at /api/skills
|
||||
app.include_router(skills.router)
|
||||
# Admin skill distillation matrix + global assistant knowledge base.
|
||||
app.include_router(skill_knowledge.router)
|
||||
app.include_router(assistant_knowledge.router)
|
||||
app.include_router(assistant_knowledge.export_router)
|
||||
|
||||
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
|
||||
app.include_router(artifacts.router)
|
||||
|
||||
@@ -315,6 +315,33 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
|
||||
# Skill distillation and the single global assistant knowledge base.
|
||||
# Both repositories live in ``deerflow.*``; only this composition layer
|
||||
# wires them to Gateway executors and remote adapters.
|
||||
from deerflow.persistence.assistant_knowledge import make_assistant_knowledge_store
|
||||
from deerflow.persistence.skill_knowledge import make_skill_knowledge_store
|
||||
|
||||
app.state.assistant_knowledge_store = make_assistant_knowledge_store(sf)
|
||||
app.state.skill_knowledge_store = make_skill_knowledge_store(sf)
|
||||
if app.state.skill_knowledge_store is not None and app.state.assistant_knowledge_store is not None:
|
||||
from app.gateway.skill_knowledge_job_dispatcher import SkillKnowledgeJobDispatcher
|
||||
from app.gateway.skill_knowledge_job_executor import SkillKnowledgeJobExecutor
|
||||
|
||||
app.state.skill_knowledge_executor = SkillKnowledgeJobExecutor(app)
|
||||
app.state.skill_knowledge_dispatcher = SkillKnowledgeJobDispatcher(
|
||||
app.state.skill_knowledge_store,
|
||||
app.state.skill_knowledge_executor,
|
||||
)
|
||||
app.state.skill_knowledge_dispatcher.start()
|
||||
from app.gateway.skill_knowledge_auto_sync import SkillKnowledgeAutoSyncService
|
||||
|
||||
app.state.skill_knowledge_auto_sync = SkillKnowledgeAutoSyncService(app)
|
||||
app.state.skill_knowledge_auto_sync.start()
|
||||
else:
|
||||
app.state.skill_knowledge_executor = None
|
||||
app.state.skill_knowledge_dispatcher = None
|
||||
app.state.skill_knowledge_auto_sync = None
|
||||
|
||||
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
|
||||
# migrations and status inspection work while feature flags are off;
|
||||
# network/search services only exist when explicitly enabled.
|
||||
@@ -583,6 +610,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
|
||||
if dr_dispatcher is not None:
|
||||
await dr_dispatcher.stop()
|
||||
skill_knowledge_auto_sync = getattr(app.state, "skill_knowledge_auto_sync", None)
|
||||
if skill_knowledge_auto_sync is not None:
|
||||
await skill_knowledge_auto_sync.close()
|
||||
skill_knowledge_dispatcher = getattr(app.state, "skill_knowledge_dispatcher", None)
|
||||
if skill_knowledge_dispatcher is not None:
|
||||
await skill_knowledge_dispatcher.close()
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
if queue is not None:
|
||||
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Read-mostly global assistant knowledge API and admin import actions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.assistant_knowledge import export_package_from_skill_ir
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
|
||||
|
||||
router = APIRouter(prefix="/api/assistant-knowledge", tags=["assistant-knowledge"])
|
||||
export_router = APIRouter(tags=["assistant-knowledge"])
|
||||
|
||||
|
||||
async def _actor(request: Request, *, admin: bool = False) -> str | None:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return None
|
||||
if admin and getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="该操作仅管理员可用")
|
||||
return str(user.id)
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
store = getattr(request.app.state, "assistant_knowledge_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="助手知识库存储不可用")
|
||||
return store
|
||||
|
||||
|
||||
class InitializeRequest(BaseModel):
|
||||
name: str = Field(default="知识梳理", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class SkillImportRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
force: bool = False
|
||||
|
||||
|
||||
@router.get("/base")
|
||||
async def get_base(request: Request) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"base": await _store(request).get_base()}
|
||||
|
||||
|
||||
@router.post("/base/initialize", status_code=201)
|
||||
async def initialize(request: Request, body: InitializeRequest) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
return {"base": await _store(request).initialize(created_by=actor, name=body.name)}
|
||||
|
||||
|
||||
@router.get("/wiki-pages")
|
||||
async def list_wiki_pages(request: Request, q: str | None = Query(default=None), page_type: str | None = Query(default=None), limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
pages = await _store(request).list_pages(query=q, page_type=page_type, limit=limit)
|
||||
return {"pages": pages, "total": len(pages)}
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions/compare")
|
||||
async def compare_revisions(request: Request, slug: str, left: str = Query(...), right: str = Query(...)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
result = await _store(request).compare_revisions(left, right)
|
||||
if result is None:
|
||||
raise HTTPException(status_code=404, detail="版本不存在或不属于同一页面")
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions")
|
||||
async def list_revisions(request: Request, slug: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"revisions": await _store(request).list_revisions(slug)}
|
||||
|
||||
|
||||
@router.post("/wiki-pages/{slug:path}/revisions/{revision_id}/rollback")
|
||||
async def rollback_revision(request: Request, slug: str, revision_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
row = await _store(request).rollback(slug=slug, revision_id=revision_id, actor=actor)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="页面或版本不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}")
|
||||
async def get_wiki_page(request: Request, slug: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_page(slug)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Wiki 页面不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
async def search(request: Request, q: str = Query(..., min_length=1), page_type: str | None = Query(default=None)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
rows = await _store(request).list_pages(query=q, page_type=page_type, limit=100)
|
||||
return {"results": rows, "total": len(rows)}
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_sources(request: Request) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"sources": await _store(request).list_sources()}
|
||||
|
||||
|
||||
@router.get("/entities/{entity_id}")
|
||||
async def get_entity(request: Request, entity_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_entity(entity_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="实体不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/relations/{relation_id}")
|
||||
async def get_relation(request: Request, relation_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_relation(relation_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="关系不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.delete("/sources/{source_id}", status_code=204)
|
||||
async def delete_source(request: Request, source_id: str) -> None:
|
||||
await _actor(request, admin=True)
|
||||
if not await _store(request).delete_source(source_id):
|
||||
raise HTTPException(status_code=404, detail="来源不存在")
|
||||
|
||||
|
||||
@router.get("/import-jobs")
|
||||
async def list_import_jobs(request: Request, limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"jobs": await _store(request).list_jobs(limit)}
|
||||
|
||||
|
||||
@router.get("/import-jobs/{job_id}")
|
||||
async def get_import_job(request: Request, job_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="导入任务不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/imports/skills", status_code=202)
|
||||
async def import_skills(request: Request, body: SkillImportRequest) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
base = await _store(request).initialize(created_by=actor)
|
||||
target = {"target_type": "assistant", "target_mode": "assistant", "target_id": base["id"], "target_name": base["name"]}
|
||||
job = await request.app.state.skill_knowledge_store.create_job(
|
||||
skill_names=body.skill_names,
|
||||
targets=[target],
|
||||
created_by=actor,
|
||||
force=body.force,
|
||||
review_mode="auto_high_confidence",
|
||||
trigger="skill_direct",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/imports/files", status_code=201)
|
||||
async def import_files(request: Request, files: list[UploadFile] = File(...)) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
if not files or len(files) > 50:
|
||||
raise HTTPException(status_code=422, detail="一次可导入 1-50 个文件")
|
||||
batch_id = str(uuid4())
|
||||
jobs: list[dict[str, Any]] = []
|
||||
with tempfile.TemporaryDirectory(prefix="assistant-knowledge-") as tmp:
|
||||
root = Path(tmp)
|
||||
for upload in files:
|
||||
filename = Path(upload.filename or "upload.bin").name
|
||||
content = await upload.read()
|
||||
if len(content) > 50 * 1024 * 1024:
|
||||
raise HTTPException(status_code=413, detail=f"文件过大:{filename}")
|
||||
path = root / filename
|
||||
await asyncio.to_thread(path.write_bytes, content)
|
||||
scan = await asyncio.to_thread(scan_skill, f"local-{batch_id}", root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
if scan.blocked_files:
|
||||
raise HTTPException(status_code=422, detail={"message": "文件命中敏感信息阻断规则", "blocked_files": scan.blocked_files})
|
||||
ir = await asyncio.to_thread(build_skill_ir, scan, display_name="本地文件导入", description="管理员本地文件导入")
|
||||
package = export_package_from_skill_ir(ir)
|
||||
job = await _store(request).import_package(
|
||||
package=package,
|
||||
source_type="local_file",
|
||||
source_key=batch_id,
|
||||
source_name="、".join(Path(file.filename or "upload").name for file in files)[:255],
|
||||
trigger="manual_file_upload",
|
||||
created_by=actor,
|
||||
upload_batch_id=batch_id,
|
||||
)
|
||||
jobs.append(job)
|
||||
return {"batch_id": batch_id, "jobs": jobs}
|
||||
|
||||
|
||||
async def _export_weknora_package(client: Any, remote_id: str, source: dict[str, Any]) -> dict[str, Any]:
|
||||
pages: list[dict[str, Any]] = []
|
||||
page_no = 1
|
||||
while True:
|
||||
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("pages") or []
|
||||
pages.extend(batch)
|
||||
if len(pages) >= int(page.get("total") or len(pages)) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
documents: list[dict[str, Any]] = []
|
||||
chunks: list[dict[str, Any]] = []
|
||||
page_no = 1
|
||||
while True:
|
||||
page = await client.list_documents(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("items") or []
|
||||
documents.extend(batch)
|
||||
for document in batch:
|
||||
document_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not document_id:
|
||||
continue
|
||||
chunk_page = 1
|
||||
while True:
|
||||
values = await client.list_chunks(document_id, page=chunk_page, page_size=100)
|
||||
chunk_batch = values.get("items") or []
|
||||
chunks.extend([{**row, "source_document_id": document_id} for row in chunk_batch])
|
||||
if len(chunk_batch) < 100:
|
||||
break
|
||||
chunk_page += 1
|
||||
if len(documents) >= int(page.get("total") or len(documents)) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
try:
|
||||
graph = await client.get_wiki_graph(remote_id, limit=500)
|
||||
except WeKnoraError:
|
||||
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
|
||||
entities = [{"id": row.get("id"), "name": row.get("title") or row.get("name"), "type": row.get("type") or "concept", "confidence": 1.0} for row in graph.get("nodes") or [] if row.get("id") and (row.get("title") or row.get("name"))]
|
||||
relations = [
|
||||
{"source_entity_id": row.get("source"), "target_entity_id": row.get("target"), "predicate": row.get("predicate") or "references", "confidence": 1.0, "evidence": [{"source": "weknora_graph"}]} for row in graph.get("edges") or []
|
||||
]
|
||||
return {"source": source, "wiki_pages": pages, "documents": documents, "chunks": chunks, "entities": entities, "relations": relations, "graph": graph}
|
||||
|
||||
|
||||
@export_router.post("/api/llmwiki/knowledge-bases/{mapping_id}/export-to-assistant", status_code=202)
|
||||
async def export_to_assistant(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=False, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="普通知识库不存在")
|
||||
client = build_weknora_client(runtime)
|
||||
package = await _export_weknora_package(
|
||||
client,
|
||||
str(mapping["weknora_id"]),
|
||||
{
|
||||
"type": "weknora",
|
||||
"knowledge_base_id": mapping_id,
|
||||
"remote_knowledge_base_id": mapping["weknora_id"],
|
||||
"knowledge_base_name": mapping["name"],
|
||||
},
|
||||
)
|
||||
return await _store(request).import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=mapping_id,
|
||||
source_name=str(mapping["name"]),
|
||||
trigger="manual_from_weknora",
|
||||
created_by=actor,
|
||||
mapping_id=mapping_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/reimport", status_code=202)
|
||||
async def reimport_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
store = _store(request)
|
||||
source = await store.get_source(source_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail="来源不存在")
|
||||
if source["source_type"] == "skill":
|
||||
base = await store.initialize(created_by=actor)
|
||||
job = await request.app.state.skill_knowledge_store.create_job(
|
||||
skill_names=[str(source.get("skill_name") or source["source_key"])],
|
||||
targets=[
|
||||
{
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": base["id"],
|
||||
"target_name": base["name"],
|
||||
}
|
||||
],
|
||||
created_by=actor,
|
||||
force=True,
|
||||
review_mode="auto_high_confidence",
|
||||
trigger="assistant_source_reimport",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
if source["source_type"] != "weknora":
|
||||
raise HTTPException(status_code=409, detail="本地文件来源未保留原文件,请重新上传")
|
||||
mapping_id = str(source.get("knowledge_base_mapping_id") or source["source_key"])
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
mapping_id,
|
||||
actor or "system",
|
||||
write=False,
|
||||
is_admin=True,
|
||||
)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="来源普通知识库不存在")
|
||||
package = await _export_weknora_package(
|
||||
build_weknora_client(runtime),
|
||||
str(mapping["weknora_id"]),
|
||||
{
|
||||
"type": "weknora",
|
||||
"knowledge_base_id": mapping_id,
|
||||
"remote_knowledge_base_id": mapping["weknora_id"],
|
||||
"knowledge_base_name": mapping["name"],
|
||||
},
|
||||
)
|
||||
return await store.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=mapping_id,
|
||||
source_name=str(mapping["name"]),
|
||||
trigger="source_reimport",
|
||||
created_by=actor,
|
||||
mapping_id=mapping_id,
|
||||
)
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Administrator APIs for skill knowledge bindings and matrix jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
|
||||
from croniter import croniter
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.skill_knowledge_job_executor import _skill_directory
|
||||
from deerflow.config.system_settings import (
|
||||
SkillKnowledgeAutoSyncSettings,
|
||||
load_system_settings,
|
||||
save_system_settings,
|
||||
)
|
||||
from deerflow.integrations.weknora.runtime import (
|
||||
build_weknora_client,
|
||||
get_resolved_llmwiki_runtime,
|
||||
)
|
||||
from deerflow.skill_knowledge import scan_skill
|
||||
|
||||
router = APIRouter(prefix="/api/skill-knowledge", tags=["skill-knowledge"])
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> str | None:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return None
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="技能知识归纳仅管理员可操作")
|
||||
return str(user.id)
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
value = getattr(request.app.state, "skill_knowledge_store", None)
|
||||
if value is None:
|
||||
raise HTTPException(status_code=503, detail="技能知识归纳存储不可用")
|
||||
return value
|
||||
|
||||
|
||||
class SyncTarget(BaseModel):
|
||||
target_type: Literal["weknora", "assistant"]
|
||||
target_mode: Literal["wiki", "document", "faq", "conversation_archive", "readonly_sidecar", "assistant"]
|
||||
target_id: str = Field(min_length=1, max_length=128)
|
||||
target_name: str = Field(default="", max_length=255)
|
||||
remote_business_key: str | None = Field(default=None, max_length=512)
|
||||
readonly: bool = False
|
||||
|
||||
|
||||
class CreateSyncJobRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
targets: list[SyncTarget] = Field(min_length=1, max_length=50)
|
||||
force: bool = False
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
|
||||
|
||||
@field_validator("skill_names")
|
||||
@classmethod
|
||||
def validate_names(cls, values: list[str]) -> list[str]:
|
||||
result = list(dict.fromkeys(value.strip() for value in values if value.strip()))
|
||||
if not result:
|
||||
raise ValueError("At least one skill is required")
|
||||
return result
|
||||
|
||||
|
||||
class ReviewRequest(BaseModel):
|
||||
review_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
action: Literal["approve", "reject"]
|
||||
edits: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
reason: str | None = Field(default=None, max_length=1024)
|
||||
|
||||
|
||||
class DetachRequest(BaseModel):
|
||||
delete_contributions: bool
|
||||
|
||||
|
||||
class AutoSyncSettingsUpdate(BaseModel):
|
||||
watch_enabled: bool | None = None
|
||||
schedule_enabled: bool | None = None
|
||||
schedule_cron: str | None = Field(default=None, min_length=5, max_length=128)
|
||||
debounce_seconds: int | None = Field(default=None, ge=10, le=86_400)
|
||||
write_remote_enabled: bool | None = None
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] | None = None
|
||||
|
||||
@field_validator("schedule_cron")
|
||||
@classmethod
|
||||
def validate_cron(cls, value: str | None) -> str | None:
|
||||
if value is not None and not croniter.is_valid(value):
|
||||
raise ValueError("无效的 cron 表达式")
|
||||
return value
|
||||
|
||||
|
||||
@router.get("/bindings")
|
||||
async def list_bindings(
|
||||
request: Request,
|
||||
skill_names: list[str] | None = Query(default=None),
|
||||
target_id: str | None = Query(default=None),
|
||||
include_detached: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
rows = await _store(request).list_bindings(skill_names=skill_names, target_id=target_id, include_detached=include_detached)
|
||||
return {"bindings": rows}
|
||||
|
||||
|
||||
@router.get("/source-status")
|
||||
async def source_status(request: Request, skill_names: list[str] = Query(...)) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
bindings = await _store(request).list_bindings(skill_names=skill_names)
|
||||
by_skill: dict[str, dict[str, Any]] = {}
|
||||
for name in list(dict.fromkeys(skill_names)):
|
||||
try:
|
||||
root = _skill_directory(request.app.state.config, name)
|
||||
result = await asyncio.to_thread(scan_skill, name, root)
|
||||
by_skill[name] = {"source_digest": result.source_digest, "counts": result.counts, "blocked_files": result.blocked_files}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
by_skill[name] = {"error": str(exc)}
|
||||
for binding in bindings:
|
||||
status = by_skill.get(binding["skill_name"])
|
||||
if status and status.get("source_digest") and status["source_digest"] != binding.get("synced_digest"):
|
||||
binding["computed_status"] = "stale"
|
||||
return {"skills": by_skill, "bindings": bindings}
|
||||
|
||||
|
||||
@router.get("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
|
||||
async def get_auto_sync_settings(request: Request) -> SkillKnowledgeAutoSyncSettings:
|
||||
await _require_admin(request)
|
||||
return load_system_settings().skill_knowledge_auto_sync
|
||||
|
||||
|
||||
@router.put("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
|
||||
async def update_auto_sync_settings(request: Request, body: AutoSyncSettingsUpdate) -> SkillKnowledgeAutoSyncSettings:
|
||||
await _require_admin(request)
|
||||
current = load_system_settings()
|
||||
updated = current.skill_knowledge_auto_sync.model_copy(update=body.model_dump(exclude_unset=True))
|
||||
saved = save_system_settings(current.model_copy(update={"skill_knowledge_auto_sync": updated}))
|
||||
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
|
||||
if service is not None:
|
||||
service.nudge()
|
||||
return saved.skill_knowledge_auto_sync
|
||||
|
||||
|
||||
async def _set_binding_auto_sync(request: Request, binding_id: str, *, enabled: bool) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).set_binding_auto_sync(binding_id, enabled=enabled)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
|
||||
if service is not None:
|
||||
service.nudge()
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/enable-auto-sync")
|
||||
async def enable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
return await _set_binding_auto_sync(request, binding_id, enabled=True)
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/disable-auto-sync")
|
||||
async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
return await _set_binding_auto_sync(request, binding_id, enabled=False)
|
||||
|
||||
|
||||
@router.post("/sync-jobs", status_code=202)
|
||||
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
skill_store = request.app.state.skill_store
|
||||
missing = [name for name in body.skill_names if await skill_store.get_any(name) is None]
|
||||
if missing:
|
||||
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
|
||||
targets: list[dict[str, Any]] = []
|
||||
for target in body.targets:
|
||||
data = target.model_dump()
|
||||
if target.target_type == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(target.target_id, actor or "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail=f"普通知识库不存在:{target.target_id}")
|
||||
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
|
||||
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
|
||||
else:
|
||||
base = await request.app.state.assistant_knowledge_store.get_base()
|
||||
if base is None:
|
||||
base = await request.app.state.assistant_knowledge_store.initialize(created_by=actor)
|
||||
if target.target_id not in {base["id"], "global"}:
|
||||
raise HTTPException(status_code=404, detail="助手知识库仅允许选择全局知识梳理库")
|
||||
data["target_id"] = base["id"]
|
||||
data["target_name"] = base["name"]
|
||||
data["remote_business_key"] = f"assistant:base:{base['id']}"
|
||||
targets.append(data)
|
||||
job = await _store(request).create_job(
|
||||
skill_names=body.skill_names,
|
||||
targets=targets,
|
||||
created_by=actor,
|
||||
force=body.force,
|
||||
review_mode=body.review_mode,
|
||||
)
|
||||
dispatcher = getattr(request.app.state, "skill_knowledge_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/sync-jobs")
|
||||
async def list_sync_jobs(request: Request, limit: int = Query(default=50, ge=1, le=200)) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
return {"jobs": await _store(request).list_jobs(limit=limit)}
|
||||
|
||||
|
||||
@router.get("/sync-jobs/{job_id}")
|
||||
async def get_sync_job(request: Request, job_id: str) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/sync-jobs/{job_id}/cancel")
|
||||
async def cancel_sync_job(request: Request, job_id: str) -> dict[str, bool]:
|
||||
await _require_admin(request)
|
||||
if not await _store(request).cancel_job(job_id):
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
return {"canceled": True}
|
||||
|
||||
|
||||
@router.post("/sync-items/{item_id}/retry", status_code=202)
|
||||
async def retry_sync_item(request: Request, item_id: str) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).requeue_item(item_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="归纳任务项不存在")
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/resync", status_code=202)
|
||||
async def resync_binding(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
binding = await _store(request).get_binding(binding_id)
|
||||
if binding is None or not binding.get("active"):
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
target = {
|
||||
"target_type": binding["target_type"],
|
||||
"target_mode": binding["target_mode"],
|
||||
"target_id": binding["target_id"],
|
||||
"target_name": binding["target_name"],
|
||||
"remote_business_key": binding.get("remote_business_key"),
|
||||
}
|
||||
job = await _store(request).create_job(
|
||||
skill_names=[binding["skill_name"]],
|
||||
targets=[target],
|
||||
created_by=actor,
|
||||
force=True,
|
||||
review_mode="auto_high_confidence",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/detach")
|
||||
async def detach_binding(request: Request, binding_id: str, body: DetachRequest) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
store = _store(request)
|
||||
binding = await store.get_binding(binding_id)
|
||||
if binding is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
if await store.has_active_item(binding_id):
|
||||
raise HTTPException(status_code=409, detail="该绑定仍有运行中的归纳项,请先取消任务")
|
||||
cleanup_failed = False
|
||||
if body.delete_contributions:
|
||||
try:
|
||||
if binding["target_type"] == "assistant":
|
||||
sources = await request.app.state.assistant_knowledge_store.list_sources()
|
||||
for source in sources:
|
||||
if source.get("source_type") == "skill" and source.get("source_key") == binding["skill_name"]:
|
||||
await request.app.state.assistant_knowledge_store.delete_source(source["id"])
|
||||
elif binding["target_type"] == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
str(binding["target_id"]),
|
||||
"system",
|
||||
write=True,
|
||||
is_admin=True,
|
||||
)
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if mapping is None or not runtime.weknora_enabled:
|
||||
raise RuntimeError("远端普通知识库当前不可写")
|
||||
client = build_weknora_client(runtime)
|
||||
remote_kb_id = str(mapping["weknora_id"])
|
||||
for artifact in await store.list_artifacts(binding_id):
|
||||
if artifact["kind"] == "wiki_page" and artifact.get("slug"):
|
||||
await client.delete_wiki_page(remote_kb_id, artifact["slug"])
|
||||
elif artifact.get("remote_id") and artifact["kind"] in {
|
||||
"raw_source",
|
||||
"summary_document",
|
||||
"faq_projection",
|
||||
"conversation_archive",
|
||||
"graph_projection",
|
||||
}:
|
||||
await client.delete_document(str(artifact["remote_id"]))
|
||||
except Exception: # noqa: BLE001
|
||||
cleanup_failed = True
|
||||
row = await store.detach_binding(
|
||||
binding_id,
|
||||
delete_contributions=body.delete_contributions,
|
||||
cleanup_failed=cleanup_failed,
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/reviews")
|
||||
async def list_reviews(
|
||||
request: Request,
|
||||
job_id: str | None = Query(default=None),
|
||||
item_id: str | None = Query(default=None),
|
||||
status: str | None = Query(default="pending"),
|
||||
) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
return {"reviews": await _store(request).list_reviews(job_id=job_id, item_id=item_id, status=status)}
|
||||
|
||||
|
||||
@router.post("/reviews/actions")
|
||||
async def review_actions(request: Request, body: ReviewRequest) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
item_ids = await _store(request).review(body.review_ids, action=body.action, actor=actor, edits=body.edits, reason=body.reason)
|
||||
resumed: list[str] = []
|
||||
executor = request.app.state.skill_knowledge_executor
|
||||
for item_id in item_ids:
|
||||
if await _store(request).pending_review_count(item_id) == 0:
|
||||
asyncio.create_task(executor.resume_reviewed_item(item_id))
|
||||
resumed.append(item_id)
|
||||
return {"updated": len(body.review_ids), "resumed_item_ids": resumed}
|
||||
@@ -44,6 +44,14 @@ class PromptPrefixResponse(BaseModel):
|
||||
prompt_prefix: str = Field(..., description="Prefix text prepended to the first user message")
|
||||
|
||||
|
||||
class KnowledgeBaseProductVersionResponse(BaseModel):
|
||||
version: Literal["v1", "v2"] = "v1"
|
||||
|
||||
|
||||
class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
|
||||
version: Literal["v1", "v2"]
|
||||
|
||||
|
||||
class PromptPrefixUpdateRequest(BaseModel):
|
||||
enabled: bool | None = None
|
||||
switch_label: str | None = None
|
||||
@@ -167,6 +175,21 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
|
||||
return _to_response(saved.prompt_prefix)
|
||||
|
||||
|
||||
@router.get("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
|
||||
async def get_knowledge_base_product_version() -> KnowledgeBaseProductVersionResponse:
|
||||
return KnowledgeBaseProductVersionResponse(version=load_system_settings().knowledge_base_product_version)
|
||||
|
||||
|
||||
@router.put("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
|
||||
async def update_knowledge_base_product_version(
|
||||
body: KnowledgeBaseProductVersionUpdateRequest,
|
||||
request: Request,
|
||||
) -> KnowledgeBaseProductVersionResponse:
|
||||
await _require_admin(request)
|
||||
saved = save_system_settings(load_system_settings().model_copy(update={"knowledge_base_product_version": body.version}))
|
||||
return KnowledgeBaseProductVersionResponse(version=saved.knowledge_base_product_version)
|
||||
|
||||
|
||||
@router.get("/plugin-management", response_model=PluginManagementResponse)
|
||||
async def get_plugin_management() -> PluginManagementResponse:
|
||||
"""Return the iframe plugin domain -> agent mappings."""
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Debounced file watcher and cron scheduler for opted-in knowledge bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
from app.gateway.skill_knowledge_job_executor import _skill_directory
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.skill_knowledge import scan_skill
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillKnowledgeAutoSyncService:
|
||||
def __init__(self, app: Any, *, poll_seconds: float = 10.0) -> None:
|
||||
self.app = app
|
||||
self.store = app.state.skill_knowledge_store
|
||||
self.poll_seconds = poll_seconds
|
||||
self._task: asyncio.Task | None = None
|
||||
self._wake = asyncio.Event()
|
||||
self._observed: dict[str, tuple[str, datetime]] = {}
|
||||
self._schedule_expression = ""
|
||||
self._next_schedule: datetime | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _queue_skill(
|
||||
self,
|
||||
skill_name: str,
|
||||
bindings: list[dict[str, Any]],
|
||||
*,
|
||||
trigger: str,
|
||||
review_mode: str,
|
||||
) -> None:
|
||||
targets = [
|
||||
{
|
||||
"target_type": binding["target_type"],
|
||||
"target_mode": binding["target_mode"],
|
||||
"target_id": binding["target_id"],
|
||||
"target_name": binding["target_name"],
|
||||
"remote_business_key": binding.get("remote_business_key"),
|
||||
}
|
||||
for binding in bindings
|
||||
]
|
||||
job = await self.store.create_job(
|
||||
skill_names=[skill_name],
|
||||
targets=targets,
|
||||
created_by=None,
|
||||
trigger=trigger,
|
||||
review_mode=review_mode,
|
||||
)
|
||||
if job["total_count"]:
|
||||
self.app.state.skill_knowledge_dispatcher.nudge()
|
||||
|
||||
async def _watch(
|
||||
self,
|
||||
bindings_by_skill: dict[str, list[dict[str, Any]]],
|
||||
*,
|
||||
debounce_seconds: int,
|
||||
write_remote_enabled: bool,
|
||||
review_mode: str,
|
||||
) -> None:
|
||||
now = datetime.now(UTC)
|
||||
for skill_name, bindings in bindings_by_skill.items():
|
||||
try:
|
||||
root = _skill_directory(self.app.state.config, skill_name)
|
||||
scan = await asyncio.to_thread(scan_skill, skill_name, root)
|
||||
except Exception:
|
||||
logger.exception("Failed to fingerprint skill %s", skill_name)
|
||||
continue
|
||||
synced = {str(binding.get("synced_digest") or "") for binding in bindings}
|
||||
if synced == {scan.source_digest}:
|
||||
self._observed.pop(skill_name, None)
|
||||
continue
|
||||
await self.store.mark_skill_stale(skill_name)
|
||||
previous = self._observed.get(skill_name)
|
||||
if previous is None or previous[0] != scan.source_digest:
|
||||
self._observed[skill_name] = (scan.source_digest, now)
|
||||
continue
|
||||
if (now - previous[1]).total_seconds() < debounce_seconds:
|
||||
continue
|
||||
if write_remote_enabled:
|
||||
current = await self.store.list_bindings(skill_names=[skill_name])
|
||||
stale = [binding for binding in current if binding.get("auto_sync_enabled") and binding.get("status") == "stale"]
|
||||
if stale:
|
||||
await self._queue_skill(
|
||||
skill_name,
|
||||
stale,
|
||||
trigger="watcher",
|
||||
review_mode=review_mode,
|
||||
)
|
||||
self._observed.pop(skill_name, None)
|
||||
|
||||
def _schedule_due(self, expression: str, now: datetime) -> bool:
|
||||
if expression != self._schedule_expression or self._next_schedule is None:
|
||||
self._schedule_expression = expression
|
||||
self._next_schedule = croniter(expression, now).get_next(datetime)
|
||||
return False
|
||||
if now < self._next_schedule:
|
||||
return False
|
||||
self._next_schedule = croniter(expression, now).get_next(datetime)
|
||||
return True
|
||||
|
||||
async def _tick(self) -> None:
|
||||
settings = (await asyncio.to_thread(load_system_settings)).skill_knowledge_auto_sync
|
||||
bindings = [row for row in await self.store.list_bindings() if row.get("auto_sync_enabled")]
|
||||
by_skill: dict[str, list[dict[str, Any]]] = {}
|
||||
for binding in bindings:
|
||||
by_skill.setdefault(str(binding["skill_name"]), []).append(binding)
|
||||
if settings.watch_enabled:
|
||||
await self._watch(
|
||||
by_skill,
|
||||
debounce_seconds=settings.debounce_seconds,
|
||||
write_remote_enabled=settings.write_remote_enabled,
|
||||
review_mode=settings.review_mode,
|
||||
)
|
||||
if settings.schedule_enabled and settings.write_remote_enabled and by_skill and self._schedule_due(settings.schedule_cron, datetime.now(UTC)):
|
||||
for skill_name, skill_bindings in by_skill.items():
|
||||
await self._queue_skill(
|
||||
skill_name,
|
||||
skill_bindings,
|
||||
trigger="scheduled",
|
||||
review_mode=settings.review_mode,
|
||||
)
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick()
|
||||
except Exception:
|
||||
logger.exception("Skill knowledge automatic sync tick failed")
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=self.poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Small persistent-queue dispatcher for skill knowledge items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillKnowledgeJobDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
executor: Any,
|
||||
*,
|
||||
interval_seconds: float = 2.0,
|
||||
concurrency: int = 3,
|
||||
lease_seconds: int = 900,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.executor = executor
|
||||
self.interval_seconds = interval_seconds
|
||||
self.lease_seconds = lease_seconds
|
||||
self.owner = f"{socket.gethostname()}:{uuid4()}"
|
||||
self.semaphore = asyncio.Semaphore(concurrency)
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running: set[str] = set()
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _run(self, item: dict) -> None:
|
||||
item_id = str(item["id"])
|
||||
async with self.semaphore:
|
||||
heartbeat = asyncio.create_task(self._heartbeat(item_id))
|
||||
try:
|
||||
await self.executor.run_item(item)
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
try:
|
||||
await heartbeat
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._running.discard(item_id)
|
||||
self._wake.set()
|
||||
|
||||
async def _heartbeat(self, item_id: str) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(min(30, max(5, self.lease_seconds // 3)))
|
||||
if not await self.store.renew_lease(
|
||||
item_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.lease_seconds,
|
||||
):
|
||||
return
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
for item in await self.store.list_queued_items(limit=20):
|
||||
item_id = str(item["id"])
|
||||
if item_id in self._running:
|
||||
continue
|
||||
claimed = await self.store.claim_item(
|
||||
item_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.lease_seconds,
|
||||
)
|
||||
if claimed is None:
|
||||
continue
|
||||
self._running.add(item_id)
|
||||
asyncio.create_task(self._run(claimed))
|
||||
except Exception:
|
||||
logger.exception("Skill knowledge dispatcher tick failed")
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Background executor for one skill x target matrix item."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.assistant_knowledge import export_package_from_skill_ir
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
|
||||
from deerflow.skill_knowledge.graph_writer import parameterized_statements, plan_from_ir
|
||||
from deerflow.skills.parser import parse_skill_file
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _skill_directory(app_config: Any, skill_name: str) -> Path:
|
||||
storage = get_or_new_skill_storage(app_config=app_config)
|
||||
root = storage.get_skills_root_path().resolve()
|
||||
for category in SkillCategory:
|
||||
direct = root / category.value / skill_name
|
||||
if (direct / "SKILL.md").is_file():
|
||||
return direct
|
||||
for category in SkillCategory:
|
||||
category_root = root / category.value
|
||||
if not category_root.is_dir():
|
||||
continue
|
||||
for skill_md in category_root.rglob("SKILL.md"):
|
||||
if skill_md.is_symlink():
|
||||
continue
|
||||
parsed = parse_skill_file(skill_md, category=category, relative_path=skill_md.parent.relative_to(category_root))
|
||||
if parsed and parsed.name == skill_name:
|
||||
return skill_md.parent
|
||||
raise FileNotFoundError(f"Skill '{skill_name}' not found under running skill roots")
|
||||
|
||||
|
||||
def _graph_projection(ir: dict[str, Any]) -> str:
|
||||
rows = []
|
||||
entities = {str(item.get("local_id")): item for item in ir.get("entities", [])}
|
||||
for relation in ir.get("relations", []):
|
||||
if float(relation.get("confidence") or 0) < 0.75 or not relation.get("evidence"):
|
||||
continue
|
||||
source = entities.get(str(relation.get("source_local_id")), {})
|
||||
target = entities.get(str(relation.get("target_local_id")), {})
|
||||
rows.append(
|
||||
json.dumps(
|
||||
{
|
||||
"source": source.get("name"),
|
||||
"source_type": source.get("type"),
|
||||
"predicate": relation.get("predicate"),
|
||||
"target": target.get("name"),
|
||||
"target_type": target.get("type"),
|
||||
"description": relation.get("description"),
|
||||
"confidence": relation.get("confidence"),
|
||||
"evidence": relation.get("evidence"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
class SkillKnowledgeJobExecutor:
|
||||
def __init__(self, app: Any) -> None:
|
||||
self.app = app
|
||||
self.store = app.state.skill_knowledge_store
|
||||
|
||||
async def _cancel_if_requested(self, item: dict[str, Any]) -> bool:
|
||||
job = await self.store.get_job(str(item["job_id"]))
|
||||
if not (job or {}).get("cancel_requested"):
|
||||
return False
|
||||
await self.store.update_item(
|
||||
str(item["id"]),
|
||||
status="canceled",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _assert_source_unchanged(self, item: dict[str, Any], expected_digest: str) -> None:
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
current = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
current = await enrich_converted_documents(current)
|
||||
if current.source_digest == expected_digest:
|
||||
return
|
||||
await self.store.mark_skill_stale(str(item["skill_name"]))
|
||||
raise RuntimeError("技能知识文件在归纳期间发生变化,旧版本保持激活,请重试")
|
||||
|
||||
async def _write_direct_graph(
|
||||
self,
|
||||
item: dict[str, Any],
|
||||
snapshot_id: str,
|
||||
ir: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
config = self.app.state.config.llmwiki.weknora.graph
|
||||
if not config.direct_write_enabled:
|
||||
return None
|
||||
if not config.neo4j_http_url or not config.username or not config.password:
|
||||
raise RuntimeError("Neo4j direct writer 已开启但后端连接配置不完整")
|
||||
plan = plan_from_ir(
|
||||
ir,
|
||||
target_type=str(item["target_type"]),
|
||||
target_id=str(item["target_id"]),
|
||||
binding_id=str(item["binding_id"]),
|
||||
snapshot_id=snapshot_id,
|
||||
job_id=str(item["job_id"]),
|
||||
)
|
||||
statements = parameterized_statements(plan)
|
||||
if not statements:
|
||||
return {"nodes": 0, "edges": 0, "statement_count": 0}
|
||||
endpoint = f"{config.neo4j_http_url.rstrip('/')}/db/{config.database}/tx/commit"
|
||||
async with httpx.AsyncClient(
|
||||
auth=(config.username, config.password),
|
||||
timeout=config.timeout_seconds,
|
||||
) as client:
|
||||
response = await client.post(endpoint, json={"statements": statements})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
errors = payload.get("errors") or []
|
||||
if errors:
|
||||
raise RuntimeError(f"Neo4j transaction failed: {str(errors[0].get('code') or 'unknown')}")
|
||||
return {
|
||||
"nodes": len(plan.nodes),
|
||||
"edges": len(plan.edges),
|
||||
"statement_count": len(statements),
|
||||
"scope": plan.scope,
|
||||
}
|
||||
|
||||
async def run_item(self, item: dict[str, Any]) -> None:
|
||||
item_id = str(item["id"])
|
||||
try:
|
||||
await self.store.update_item(item_id, status="running", phase="scanning", progress=5)
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
await self.store.update_item(item_id, phase="extracting", progress=30)
|
||||
skill_record = await self.app.state.skill_store.get_any(str(item["skill_name"]))
|
||||
ir = await asyncio.to_thread(
|
||||
build_skill_ir,
|
||||
scan,
|
||||
display_name=(skill_record or {}).get("name_zh"),
|
||||
description=str((skill_record or {}).get("description") or ""),
|
||||
)
|
||||
snapshot = await self.store.get_or_create_snapshot(
|
||||
skill_name=str(item["skill_name"]),
|
||||
source_digest=scan.source_digest,
|
||||
extractor_version=str(ir["extractor_version"]),
|
||||
prompt_hash=str(ir["prompt_hash"]),
|
||||
model_name=str(ir["model_name"]),
|
||||
manifest=scan.manifest,
|
||||
ir=ir,
|
||||
counts=scan.counts,
|
||||
)
|
||||
snapshot_id = str(snapshot["id"])
|
||||
await self.store.update_item(item_id, phase="awaiting_review", progress=45, snapshot_id=snapshot_id)
|
||||
job = await self.store.get_job(str(item["job_id"]))
|
||||
review_mode = str((job or {}).get("review_mode") or "auto_high_confidence")
|
||||
review_objects: list[tuple[str, dict]] = []
|
||||
if review_mode == "required":
|
||||
review_objects = [("entity", obj) for obj in ir.get("entities", [])] + [("relation", obj) for obj in ir.get("relations", [])]
|
||||
elif review_mode == "auto_high_confidence":
|
||||
review_objects = [(kind, obj) for kind, rows in (("entity", ir.get("entities", [])), ("relation", ir.get("relations", []))) for obj in rows if float(obj.get("confidence") or 0) < 0.9]
|
||||
if review_objects:
|
||||
existing = await self.store.list_reviews(item_id=item_id, status=None)
|
||||
if not existing:
|
||||
await self.store.create_reviews(
|
||||
job_id=str(item["job_id"]),
|
||||
item_id=item_id,
|
||||
binding_id=str(item["binding_id"]),
|
||||
snapshot_id=snapshot_id,
|
||||
objects=review_objects,
|
||||
)
|
||||
await self.store.update_item(item_id, status="awaiting_review", phase="awaiting_review", progress=50)
|
||||
return
|
||||
await self._project_and_activate(item, snapshot_id, ir, scan)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Skill knowledge item failed: item=%s", item_id)
|
||||
await self.store.update_item(
|
||||
item_id,
|
||||
status="failed",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
|
||||
async def resume_reviewed_item(self, item_id: str) -> None:
|
||||
if await self.store.pending_review_count(item_id):
|
||||
return
|
||||
jobs = await self.store.list_jobs(limit=100)
|
||||
item = None
|
||||
for job in jobs:
|
||||
detail = await self.store.get_job(str(job["id"]))
|
||||
item = next((row for row in (detail or {}).get("items", []) if row["id"] == item_id), None)
|
||||
if item:
|
||||
break
|
||||
if not item or not item.get("snapshot_id"):
|
||||
return
|
||||
snapshot = await self.store.get_snapshot(str(item["snapshot_id"]))
|
||||
ir = await self.store.effective_ir(str(item["snapshot_id"]), item_id)
|
||||
if not snapshot or not ir:
|
||||
return
|
||||
try:
|
||||
await self.store.update_item(item_id, status="running", phase="merging", progress=55)
|
||||
await self._project_and_activate(item, str(item["snapshot_id"]), ir, None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Reviewed skill knowledge item failed: item=%s", item_id)
|
||||
await self.store.update_item(
|
||||
item_id,
|
||||
status="failed",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
|
||||
async def _project_and_activate(self, item: dict[str, Any], snapshot_id: str, ir: dict[str, Any], scan: Any | None) -> None:
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
expected_digest = str(ir.get("skill", {}).get("source_digest") or "")
|
||||
if scan is None:
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
|
||||
target_type = str(item["target_type"])
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
binding_status = "synced"
|
||||
page = ir["wiki"]["skill_page"]
|
||||
if target_type == "assistant":
|
||||
package = export_package_from_skill_ir(ir)
|
||||
await self.app.state.assistant_knowledge_store.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key=str(item["skill_name"]),
|
||||
source_name=str(ir["skill"].get("display_name") or item["skill_name"]),
|
||||
trigger="skill_direct",
|
||||
created_by=(await self.store.get_job(str(item["job_id"]))).get("created_by"),
|
||||
skill_name=str(item["skill_name"]),
|
||||
)
|
||||
artifacts.append({"kind": "assistant_wiki_revision", "slug": page["slug"], "content": page["content"], "business_key": f"assistant:wiki:{page['slug']}"})
|
||||
elif str(item["target_mode"]) == "readonly_sidecar":
|
||||
artifacts.append({"kind": "sidecar_wiki", "slug": page["slug"], "content": page["content"], "business_key": f"sidecar:{item['target_id']}:{page['slug']}"})
|
||||
artifacts.append({"kind": "sidecar_graph", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "content": _graph_projection(ir)})
|
||||
binding_status = "pending_remote_write"
|
||||
else:
|
||||
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise ValueError("Target WeKnora mapping does not exist")
|
||||
runtime = get_resolved_llmwiki_runtime(self.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("普通知识库服务未配置")
|
||||
client = build_weknora_client(runtime)
|
||||
remote_kb_id = str(mapping["weknora_id"])
|
||||
if scan is not None:
|
||||
for file in scan.files:
|
||||
source_path = Path(scan.source_root) / file.path
|
||||
if file.text:
|
||||
remote = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/source/{file.path}", content=file.text)
|
||||
else:
|
||||
remote = await client.upload_document(
|
||||
remote_kb_id,
|
||||
filename=f"{item['skill_name']}-{source_path.name}",
|
||||
content=await asyncio.to_thread(source_path.read_bytes),
|
||||
)
|
||||
remote_id = str(remote.get("id") or remote.get("knowledge_id") or "") or None
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "raw_source",
|
||||
"slug": file.path,
|
||||
"remote_id": remote_id,
|
||||
"business_key": f"weknora:document:{remote_id}" if remote_id else None,
|
||||
"remote_refs": [{"provider": "weknora", "remote_kind": "document", "remote_id": remote_id}] if remote_id else [],
|
||||
}
|
||||
)
|
||||
target_mode = str(item["target_mode"])
|
||||
if target_mode == "wiki":
|
||||
remote_page: dict[str, Any]
|
||||
try:
|
||||
await client.get_wiki_page(remote_kb_id, page["slug"])
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
remote_page = await client.create_wiki_page(
|
||||
remote_kb_id,
|
||||
{
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"content": page["content"],
|
||||
"summary": str(ir["skill"].get("description") or ""),
|
||||
"page_type": "skill",
|
||||
},
|
||||
)
|
||||
else:
|
||||
remote_page = await client.update_wiki_page(
|
||||
remote_kb_id,
|
||||
page["slug"],
|
||||
{
|
||||
"title": page["title"],
|
||||
"content": page["content"],
|
||||
"summary": str(ir["skill"].get("description") or ""),
|
||||
},
|
||||
)
|
||||
remote_page_id = str(remote_page.get("id") or remote_page.get("page_id") or page["slug"])
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"slug": page["slug"],
|
||||
"content": page["content"],
|
||||
"remote_id": remote_page_id,
|
||||
"business_key": f"weknora:wiki_page:{remote_page_id}",
|
||||
"remote_refs": [
|
||||
{
|
||||
"provider": "weknora",
|
||||
"remote_kind": "wiki_page",
|
||||
"remote_id": remote_page_id,
|
||||
"slug": page["slug"],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
document_content = page["content"]
|
||||
artifact_kind = "summary_document"
|
||||
if target_mode == "faq":
|
||||
artifact_kind = "faq_projection"
|
||||
document_content = f"问题:{page['title']} 能解决什么问题,如何使用?\n\n回答:\n{page['content']}"
|
||||
elif target_mode == "conversation_archive":
|
||||
artifact_kind = "conversation_archive"
|
||||
document_content = f"# 技能归纳会话归档:{page['title']}\n\n以下内容由技能知识归纳任务生成。\n\n{page['content']}"
|
||||
remote_document = await client.create_manual_document(
|
||||
remote_kb_id,
|
||||
title=f"skills/{item['skill_name']}/{target_mode}.md",
|
||||
content=document_content,
|
||||
)
|
||||
remote_document_id = str(remote_document.get("id") or remote_document.get("knowledge_id") or "") or None
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": artifact_kind,
|
||||
"slug": f"skills/{item['skill_name']}/{target_mode}.md",
|
||||
"content": document_content,
|
||||
"remote_id": remote_document_id,
|
||||
"business_key": (f"weknora:document:{remote_document_id}" if remote_document_id else None),
|
||||
"remote_refs": (
|
||||
[
|
||||
{
|
||||
"provider": "weknora",
|
||||
"remote_kind": "document",
|
||||
"remote_id": remote_document_id,
|
||||
}
|
||||
]
|
||||
if remote_document_id
|
||||
else []
|
||||
),
|
||||
}
|
||||
)
|
||||
projection = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/graph-projection.jsonl", content=_graph_projection(ir))
|
||||
projection_id = str(projection.get("id") or projection.get("knowledge_id") or "") or None
|
||||
artifacts.append({"kind": "graph_projection", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "remote_id": projection_id})
|
||||
try:
|
||||
graph_audit = await self._write_direct_graph(item, snapshot_id, ir)
|
||||
if graph_audit is not None:
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "neo4j_direct_write_audit",
|
||||
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
|
||||
"content": json.dumps(graph_audit, ensure_ascii=False),
|
||||
"metadata": graph_audit,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Optional Neo4j direct graph write failed: item=%s", item["id"])
|
||||
binding_status = "partial_failed"
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "neo4j_direct_write_audit",
|
||||
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
|
||||
"content": json.dumps(
|
||||
{"status": "failed", "error": type(exc).__name__},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"metadata": {
|
||||
"status": "failed",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
}
|
||||
)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
|
||||
@@ -556,6 +556,14 @@ llmwiki:
|
||||
web_base_url: "http://119.254.86.251:8088"
|
||||
admin_email: "lqq@qq.com"
|
||||
admin_password: "1qaz@WSX"
|
||||
# 高风险能力,默认关闭。连接参数仅允许后端配置;生产环境请使用环境变量。
|
||||
graph:
|
||||
direct_write_enabled: false
|
||||
neo4j_http_url: ""
|
||||
username: ""
|
||||
password: ""
|
||||
database: "neo4j"
|
||||
timeout_seconds: 30
|
||||
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
|
||||
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
|
||||
# 且服务启动时只启动定时器,不会立刻全量向量化。
|
||||
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
"""Assistant knowledge package helpers."""
|
||||
|
||||
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
|
||||
|
||||
__all__ = ["export_package_from_skill_ir"]
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Normalize heterogeneous sources into the assistant import-package shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
|
||||
skill = ir.get("skill") or {}
|
||||
return {
|
||||
"source": {
|
||||
"type": "skill",
|
||||
"skill_name": skill.get("name"),
|
||||
"source_digest": skill.get("source_digest"),
|
||||
},
|
||||
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
|
||||
"documents": ir.get("files") or [],
|
||||
"chunks": [],
|
||||
"entities": ir.get("entities") or [],
|
||||
"relations": ir.get("relations") or [],
|
||||
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
|
||||
}
|
||||
@@ -44,6 +44,7 @@ class WeKnoraConnectionConfig(BaseModel):
|
||||
repr=False,
|
||||
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
||||
)
|
||||
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("api_base_url", "web_base_url")
|
||||
@@ -52,6 +53,23 @@ class WeKnoraConnectionConfig(BaseModel):
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class WeKnoraDirectGraphConfig(BaseModel):
|
||||
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
|
||||
|
||||
direct_write_enabled: bool = False
|
||||
neo4j_http_url: str = ""
|
||||
username: str = ""
|
||||
password: str = Field(default="", repr=False)
|
||||
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
|
||||
timeout_seconds: float = Field(default=30, gt=0, le=300)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("neo4j_http_url")
|
||||
@classmethod
|
||||
def _validate_url(cls, value: str) -> str:
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class LocalWikiEmbeddingConfig(BaseModel):
|
||||
enabled: bool = True
|
||||
provider: str = "openai"
|
||||
|
||||
@@ -156,6 +156,18 @@ class PromptPrefixSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SkillKnowledgeAutoSyncSettings(BaseModel):
|
||||
"""Watcher/scheduler policy for bindings that explicitly opt in."""
|
||||
|
||||
watch_enabled: bool = True
|
||||
schedule_enabled: bool = True
|
||||
schedule_cron: str = Field(default="0 */6 * * *", min_length=5, max_length=128)
|
||||
debounce_seconds: int = Field(default=120, ge=10, le=86_400)
|
||||
write_remote_enabled: bool = True
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class AppearanceRouteParam(BaseModel):
|
||||
"""Runtime query-param descriptor for top-bar shortcut buttons.
|
||||
|
||||
@@ -501,6 +513,11 @@ class SystemSettings(BaseModel):
|
||||
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
|
||||
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
|
||||
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
|
||||
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
|
||||
knowledge_base_product_version: Literal["v1", "v2"] = Field(
|
||||
default="v1",
|
||||
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
|
||||
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
|
||||
|
||||
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
"""ORM model for the single global DeerFlow assistant knowledge base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class AssistantKnowledgeBaseRow(Base):
|
||||
__tablename__ = "assistant_knowledge_base"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeSourceRow(Base):
|
||||
__tablename__ = "assistant_knowledge_sources"
|
||||
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
source_key: Mapped[str] = mapped_column(String(255))
|
||||
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
source_name: Mapped[str] = mapped_column(String(255), default="")
|
||||
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeImportJobRow(Base):
|
||||
__tablename__ = "assistant_knowledge_import_jobs"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
|
||||
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||
phase: Mapped[str] = mapped_column(String(32), default="exporting")
|
||||
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeWikiPageRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_pages"
|
||||
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
slug: Mapped[str] = mapped_column(String(512), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512), index=True)
|
||||
page_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
summary: Mapped[str] = mapped_column(String(2048), default="")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeWikiRevisionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_wiki_revisions"
|
||||
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
revision_no: Mapped[int] = mapped_column(Integer)
|
||||
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
|
||||
summary: Mapped[str] = mapped_column(String(2048), default="")
|
||||
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
change_type: Mapped[str] = mapped_column(String(32), default="import")
|
||||
status: Mapped[str] = mapped_column(String(24), default="published")
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
entity_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
normalized_name: Mapped[str] = mapped_column(String(512))
|
||||
display_name: Mapped[str] = mapped_column(String(512), index=True)
|
||||
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityContributionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entity_contributions"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
|
||||
|
||||
class AssistantKnowledgeRelationRow(Base):
|
||||
__tablename__ = "assistant_knowledge_relations"
|
||||
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
predicate: Mapped[str] = mapped_column(String(128), index=True)
|
||||
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
|
||||
|
||||
class AssistantKnowledgeRelationContributionRow(Base):
|
||||
__tablename__ = "assistant_knowledge_relation_contributions"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
relation_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
+601
@@ -0,0 +1,601 @@
|
||||
"""Repository for the single global assistant knowledge base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
AssistantKnowledgeImportJobRow,
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _dict(row: Any) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key, value in tuple(data.items()):
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
elif key.endswith("_json"):
|
||||
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
|
||||
return data
|
||||
|
||||
|
||||
class AssistantKnowledgeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get_base(self) -> dict[str, Any] | None:
|
||||
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
|
||||
existing = await self.get_base()
|
||||
if existing:
|
||||
return existing
|
||||
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def _upsert_source(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
base_id: str,
|
||||
source_type: str,
|
||||
source_key: str,
|
||||
source_name: str,
|
||||
metadata: dict,
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
) -> AssistantKnowledgeSourceRow:
|
||||
stmt = select(AssistantKnowledgeSourceRow).where(
|
||||
AssistantKnowledgeSourceRow.base_id == base_id,
|
||||
AssistantKnowledgeSourceRow.source_type == source_type,
|
||||
AssistantKnowledgeSourceRow.source_key == source_key,
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = AssistantKnowledgeSourceRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base_id,
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
skill_name=skill_name,
|
||||
knowledge_base_mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
status="active",
|
||||
metadata_json=metadata,
|
||||
created_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
else:
|
||||
row.source_name = source_name
|
||||
row.status = "active"
|
||||
row.metadata_json = metadata
|
||||
row.updated_at = _now()
|
||||
return row
|
||||
|
||||
async def _publish_page(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
base_id: str,
|
||||
slug: str,
|
||||
title: str,
|
||||
page_type: str,
|
||||
content: str,
|
||||
summary: str,
|
||||
search_terms: list[str],
|
||||
source_id: str,
|
||||
import_job_id: str,
|
||||
created_by: str | None,
|
||||
change_type: str,
|
||||
) -> AssistantKnowledgeWikiPageRow:
|
||||
now = _now()
|
||||
page = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base_id,
|
||||
AssistantKnowledgeWikiPageRow.slug == slug,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if page is None:
|
||||
page = AssistantKnowledgeWikiPageRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base_id,
|
||||
slug=slug,
|
||||
title=title,
|
||||
page_type=page_type,
|
||||
content_markdown="",
|
||||
summary="",
|
||||
search_text="",
|
||||
status="active",
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(page)
|
||||
await session.flush()
|
||||
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
|
||||
revision = AssistantKnowledgeWikiRevisionRow(
|
||||
id=str(uuid4()),
|
||||
page_id=page.id,
|
||||
revision_no=revision_no,
|
||||
content_markdown=content,
|
||||
summary=summary,
|
||||
source_id=source_id,
|
||||
import_job_id=import_job_id,
|
||||
change_type=change_type,
|
||||
status="published",
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
published_at=now,
|
||||
)
|
||||
session.add(revision)
|
||||
page.title = title
|
||||
page.page_type = page_type
|
||||
page.content_markdown = content
|
||||
page.summary = summary
|
||||
page.search_text = "\n".join([title, summary, content, *search_terms])
|
||||
page.current_revision_id = revision.id
|
||||
page.status = "active"
|
||||
page.updated_at = now
|
||||
return page
|
||||
|
||||
async def import_package(
|
||||
self,
|
||||
*,
|
||||
package: dict[str, Any],
|
||||
source_type: str,
|
||||
source_key: str,
|
||||
source_name: str,
|
||||
trigger: str,
|
||||
created_by: str | None,
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
base = await self.initialize(created_by=created_by)
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
source = await self._upsert_source(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
metadata=package.get("source") or {},
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
# A re-import replaces this source's prior contributions while retaining
|
||||
# canonical entities/relations that are still supported by other sources.
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
|
||||
job = AssistantKnowledgeImportJobRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
source_id=source.id,
|
||||
trigger=trigger,
|
||||
status="running",
|
||||
phase="aligning_entities",
|
||||
counts_json={},
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
source.last_import_job_id = job.id
|
||||
|
||||
entity_map: dict[str, str] = {}
|
||||
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
|
||||
raw_entities = list(package.get("entities") or [])
|
||||
for raw in raw_entities:
|
||||
display_name = str(raw.get("name") or raw.get("display_name") or "").strip()
|
||||
entity_type = str(raw.get("type") or raw.get("entity_type") or "concept")
|
||||
normalized = normalize_name(display_name)
|
||||
if not normalized:
|
||||
continue
|
||||
stmt = select(AssistantKnowledgeEntityRow).where(
|
||||
AssistantKnowledgeEntityRow.base_id == base["id"],
|
||||
AssistantKnowledgeEntityRow.entity_type == entity_type,
|
||||
AssistantKnowledgeEntityRow.normalized_name == normalized,
|
||||
)
|
||||
entity = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if entity is None:
|
||||
entity = AssistantKnowledgeEntityRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
entity_type=entity_type,
|
||||
normalized_name=normalized,
|
||||
display_name=display_name,
|
||||
aliases_json=raw.get("aliases") or [],
|
||||
attributes_json=raw.get("attributes") or {},
|
||||
status="active",
|
||||
)
|
||||
session.add(entity)
|
||||
await session.flush()
|
||||
raw_id = str(raw.get("id") or raw.get("local_id") or display_name)
|
||||
entity_map[raw_id] = entity.id
|
||||
entity_rows[entity.id] = (entity, raw)
|
||||
session.add(
|
||||
AssistantKnowledgeEntityContributionRow(
|
||||
id=str(uuid4()),
|
||||
entity_id=entity.id,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
source_wiki_slug=raw.get("source_wiki_slug"),
|
||||
source_chunk_id=raw.get("source_chunk_id"),
|
||||
source_skill_name=skill_name,
|
||||
evidence_json=raw.get("evidence") or [],
|
||||
confidence=float(raw.get("confidence") or 1.0),
|
||||
)
|
||||
)
|
||||
|
||||
raw_relations = list(package.get("relations") or [])
|
||||
relation_rows: dict[str, tuple[AssistantKnowledgeRelationRow, dict[str, Any]]] = {}
|
||||
for raw in raw_relations:
|
||||
source_entity_id = entity_map.get(str(raw.get("source_entity_id") or raw.get("source_local_id") or raw.get("source")))
|
||||
target_entity_id = entity_map.get(str(raw.get("target_entity_id") or raw.get("target_local_id") or raw.get("target")))
|
||||
confidence = float(raw.get("confidence") or 1.0)
|
||||
if not source_entity_id or not target_entity_id or confidence < 0.75:
|
||||
continue
|
||||
predicate = normalize_predicate(str(raw.get("predicate") or "related_to"))
|
||||
stmt = select(AssistantKnowledgeRelationRow).where(
|
||||
AssistantKnowledgeRelationRow.base_id == base["id"],
|
||||
AssistantKnowledgeRelationRow.source_entity_id == source_entity_id,
|
||||
AssistantKnowledgeRelationRow.predicate == predicate,
|
||||
AssistantKnowledgeRelationRow.target_entity_id == target_entity_id,
|
||||
)
|
||||
relation = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if relation is None:
|
||||
relation = AssistantKnowledgeRelationRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
source_entity_id=source_entity_id,
|
||||
predicate=predicate,
|
||||
target_entity_id=target_entity_id,
|
||||
description=str(raw.get("description") or ""),
|
||||
status="active",
|
||||
)
|
||||
session.add(relation)
|
||||
await session.flush()
|
||||
relation_rows[relation.id] = (relation, raw)
|
||||
session.add(
|
||||
AssistantKnowledgeRelationContributionRow(
|
||||
id=str(uuid4()),
|
||||
relation_id=relation.id,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
source_wiki_slug=raw.get("source_wiki_slug"),
|
||||
graph_edge_id=raw.get("graph_edge_id"),
|
||||
evidence_json=raw.get("evidence") or [],
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
|
||||
for entity, raw in entity_rows.values():
|
||||
aliases = list(dict.fromkeys([*list(entity.aliases_json or []), *list(raw.get("aliases") or [])]))
|
||||
entity.aliases_json = aliases
|
||||
content = "\n".join(
|
||||
[
|
||||
f"# {entity.display_name}",
|
||||
"",
|
||||
f"- 类型:{entity.entity_type}",
|
||||
f"- 别名:{'、'.join(aliases) if aliases else '无'}",
|
||||
f"- 本次来源:{source_name}",
|
||||
"",
|
||||
"## 属性",
|
||||
"",
|
||||
"```json",
|
||||
str(entity.attributes_json or {}),
|
||||
"```",
|
||||
]
|
||||
)
|
||||
entity_page = await self._publish_page(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
slug=f"entities/{entity.entity_type}/{entity.id}",
|
||||
title=entity.display_name,
|
||||
page_type="entity",
|
||||
content=content,
|
||||
summary=f"{entity.entity_type} 实体,来源:{source_name}",
|
||||
search_terms=aliases,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
created_by=created_by,
|
||||
change_type="entity_alignment",
|
||||
)
|
||||
entity.wiki_page_id = entity_page.id
|
||||
|
||||
for relation, raw in relation_rows.values():
|
||||
source_entity = await session.get(AssistantKnowledgeEntityRow, relation.source_entity_id)
|
||||
target_entity = await session.get(AssistantKnowledgeEntityRow, relation.target_entity_id)
|
||||
source_name_display = source_entity.display_name if source_entity else relation.source_entity_id
|
||||
target_name_display = target_entity.display_name if target_entity else relation.target_entity_id
|
||||
content = f"# {source_name_display} —{relation.predicate}→ {target_name_display}\n\n- 本次来源:{source_name}\n- 置信度:{float(raw.get('confidence') or 1.0):.2f}\n\n{relation.description}"
|
||||
relation_page = await self._publish_page(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
slug=f"relations/{relation.id}",
|
||||
title=f"{source_name_display} {relation.predicate} {target_name_display}",
|
||||
page_type="relation",
|
||||
content=content,
|
||||
summary=relation.description,
|
||||
search_terms=[source_name_display, target_name_display, relation.predicate],
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
created_by=created_by,
|
||||
change_type="relation_alignment",
|
||||
)
|
||||
relation.wiki_page_id = relation_page.id
|
||||
|
||||
pages = list(package.get("wiki_pages") or [])
|
||||
if not pages and package.get("wiki"):
|
||||
wiki = package["wiki"]
|
||||
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
|
||||
pages = [page] if page else []
|
||||
for raw in pages:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
slug = str(raw.get("slug") or f"assistant/sources/{source.id}").strip("/")
|
||||
title = str(raw.get("title") or source_name)
|
||||
content = str(raw.get("content") or raw.get("content_markdown") or "")
|
||||
summary = str(raw.get("summary") or "")
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.base_id == base["id"], AssistantKnowledgeWikiPageRow.slug == slug)
|
||||
page = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if page is None:
|
||||
page = AssistantKnowledgeWikiPageRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
slug=slug,
|
||||
title=title,
|
||||
page_type=str(raw.get("page_type") or "source"),
|
||||
content_markdown="",
|
||||
summary="",
|
||||
search_text="",
|
||||
status="active",
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(page)
|
||||
await session.flush()
|
||||
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one_or_none() or 0) + 1
|
||||
revision = AssistantKnowledgeWikiRevisionRow(
|
||||
id=str(uuid4()),
|
||||
page_id=page.id,
|
||||
revision_no=revision_no,
|
||||
content_markdown=content,
|
||||
summary=summary,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
change_type="skill_sync" if source_type == "skill" else "import",
|
||||
status="published",
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
published_at=now,
|
||||
)
|
||||
session.add(revision)
|
||||
page.title = title
|
||||
page.content_markdown = content
|
||||
page.summary = summary
|
||||
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
|
||||
page.current_revision_id = revision.id
|
||||
page.updated_at = now
|
||||
|
||||
counts = {
|
||||
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
|
||||
"documents": len(package.get("documents") or []),
|
||||
"chunks": len(package.get("chunks") or []),
|
||||
"entities": len(raw_entities),
|
||||
"relations": len(raw_relations),
|
||||
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
|
||||
"graph_edges": len((package.get("graph") or {}).get("edges") or []),
|
||||
}
|
||||
job.status = "completed"
|
||||
job.phase = "completed"
|
||||
job.counts_json = counts
|
||||
job.completed_at = now
|
||||
await self._retire_orphans(session, base["id"])
|
||||
await session.flush()
|
||||
result = _dict(job)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
async def _retire_orphans(self, session: AsyncSession, base_id: str) -> None:
|
||||
entities = (await session.execute(select(AssistantKnowledgeEntityRow).where(AssistantKnowledgeEntityRow.base_id == base_id))).scalars()
|
||||
for entity in entities:
|
||||
contribution = (await session.execute(select(AssistantKnowledgeEntityContributionRow.id).where(AssistantKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
|
||||
entity.status = "active" if contribution else "deleted"
|
||||
relations = (await session.execute(select(AssistantKnowledgeRelationRow).where(AssistantKnowledgeRelationRow.base_id == base_id))).scalars()
|
||||
for relation in relations:
|
||||
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
|
||||
relation.status = "active" if contribution else "deleted"
|
||||
|
||||
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
|
||||
if query:
|
||||
needle = f"%{query.strip()}%"
|
||||
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
|
||||
if page_type:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type == page_type)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_page(self, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
|
||||
if page is None:
|
||||
return []
|
||||
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_revision(self, revision_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def compare_revisions(self, left_id: str, right_id: str) -> dict[str, Any] | None:
|
||||
left, right = await self.get_revision(left_id), await self.get_revision(right_id)
|
||||
if not left or not right or left["page_id"] != right["page_id"]:
|
||||
return None
|
||||
diff = "\n".join(
|
||||
difflib.unified_diff(
|
||||
str(left["content_markdown"]).splitlines(),
|
||||
str(right["content_markdown"]).splitlines(),
|
||||
fromfile=f"revision-{left['revision_no']}",
|
||||
tofile=f"revision-{right['revision_no']}",
|
||||
lineterm="",
|
||||
)
|
||||
)
|
||||
return {"left": left, "right": right, "diff": diff}
|
||||
|
||||
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
|
||||
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
|
||||
if page is None or source is None or source.page_id != page.id:
|
||||
return None
|
||||
revision_no = int((await session.execute(select(func.max(AssistantKnowledgeWikiRevisionRow.revision_no)).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id))).scalar_one()) + 1
|
||||
row = AssistantKnowledgeWikiRevisionRow(
|
||||
id=str(uuid4()),
|
||||
page_id=page.id,
|
||||
revision_no=revision_no,
|
||||
content_markdown=source.content_markdown,
|
||||
summary=source.summary,
|
||||
source_id=source.source_id,
|
||||
import_job_id=source.import_job_id,
|
||||
change_type="rollback",
|
||||
status="published",
|
||||
created_by=actor,
|
||||
created_at=now,
|
||||
published_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
page.current_revision_id = row.id
|
||||
page.content_markdown = row.content_markdown
|
||||
page.summary = row.summary
|
||||
page.search_text = "\n".join([page.title, row.summary, row.content_markdown])
|
||||
page.updated_at = now
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def list_sources(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeSourceRow, source_id)
|
||||
return _dict(row) if row and row.status != "deleted" else None
|
||||
|
||||
async def get_entity(self, entity_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeEntityRow, entity_id)
|
||||
if row is None or row.status != "active":
|
||||
return None
|
||||
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.entity_id == entity_id))).scalars()
|
||||
result = _dict(row)
|
||||
result["contributions"] = [_dict(value) for value in contributions]
|
||||
return result
|
||||
|
||||
async def get_relation(self, relation_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeRelationRow, relation_id)
|
||||
if row is None or row.status != "active":
|
||||
return None
|
||||
contributions = (await session.execute(select(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.relation_id == relation_id))).scalars()
|
||||
result = _dict(row)
|
||||
result["contributions"] = [_dict(value) for value in contributions]
|
||||
return result
|
||||
|
||||
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AssistantKnowledgeImportJobRow, job_id)
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
source = await session.get(AssistantKnowledgeSourceRow, source_id)
|
||||
if source is None:
|
||||
return False
|
||||
source.status = "deleted"
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
|
||||
pages = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
|
||||
)
|
||||
).scalars()
|
||||
for page in pages:
|
||||
fallback = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiRevisionRow)
|
||||
.where(
|
||||
AssistantKnowledgeWikiRevisionRow.page_id == page.id,
|
||||
AssistantKnowledgeWikiRevisionRow.source_id != source_id,
|
||||
AssistantKnowledgeWikiRevisionRow.status == "published",
|
||||
)
|
||||
.order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if fallback is None:
|
||||
page.status = "deleted"
|
||||
page.current_revision_id = None
|
||||
else:
|
||||
page.current_revision_id = fallback.id
|
||||
page.content_markdown = fallback.content_markdown
|
||||
page.summary = fallback.summary
|
||||
page.search_text = "\n".join([page.title, fallback.summary, fallback.content_markdown])
|
||||
await session.execute(AssistantKnowledgeWikiRevisionRow.__table__.update().where(AssistantKnowledgeWikiRevisionRow.source_id == source_id).values(status="deleted"))
|
||||
await self._retire_orphans(session, source.base_id)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
def make_assistant_knowledge_store(session_factory):
|
||||
return AssistantKnowledgeRepository(session_factory) if session_factory is not None else None
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
"""Add skill distillation and assistant knowledge tables.
|
||||
|
||||
Revision ID: 20260901_01
|
||||
Revises: 20260825_01
|
||||
Create Date: 2026-09-01
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
AssistantKnowledgeImportJobRow,
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.persistence.skill_knowledge.model import (
|
||||
SkillKnowledgeArtifactRow,
|
||||
SkillKnowledgeBindingRow,
|
||||
SkillKnowledgeEntityContributionRow,
|
||||
SkillKnowledgeEntityRow,
|
||||
SkillKnowledgeRelationContributionRow,
|
||||
SkillKnowledgeRelationRow,
|
||||
SkillKnowledgeReviewRow,
|
||||
SkillKnowledgeSnapshotRow,
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
|
||||
revision = "20260901_01"
|
||||
down_revision = "20260825_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
TABLES = [
|
||||
SkillKnowledgeBindingRow,
|
||||
SkillKnowledgeSnapshotRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeArtifactRow,
|
||||
SkillKnowledgeReviewRow,
|
||||
SkillKnowledgeEntityRow,
|
||||
SkillKnowledgeEntityContributionRow,
|
||||
SkillKnowledgeRelationRow,
|
||||
SkillKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeImportJobRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
for model in TABLES:
|
||||
model.__table__.create(bind, checkfirst=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
for model in reversed(TABLES):
|
||||
model.__table__.drop(bind, checkfirst=True)
|
||||
+42
@@ -35,6 +35,17 @@ from deerflow.persistence.agents.model import (
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
AssistantKnowledgeImportJobRow,
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
|
||||
@@ -86,6 +97,18 @@ from deerflow.persistence.scheduled_tasks.model import (
|
||||
SchedulerThreadRow,
|
||||
)
|
||||
from deerflow.persistence.sensitive_words.model import SensitiveWordRow
|
||||
from deerflow.persistence.skill_knowledge.model import (
|
||||
SkillKnowledgeArtifactRow,
|
||||
SkillKnowledgeBindingRow,
|
||||
SkillKnowledgeEntityContributionRow,
|
||||
SkillKnowledgeEntityRow,
|
||||
SkillKnowledgeRelationContributionRow,
|
||||
SkillKnowledgeRelationRow,
|
||||
SkillKnowledgeReviewRow,
|
||||
SkillKnowledgeSnapshotRow,
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.task_buttons.model import TaskButtonRow
|
||||
@@ -107,6 +130,15 @@ __all__ = [
|
||||
"AgentSkillRow",
|
||||
"AdminLeaderboardDailyStatRow",
|
||||
"BusinessMappingRow",
|
||||
"AssistantKnowledgeBaseRow",
|
||||
"AssistantKnowledgeSourceRow",
|
||||
"AssistantKnowledgeImportJobRow",
|
||||
"AssistantKnowledgeWikiPageRow",
|
||||
"AssistantKnowledgeWikiRevisionRow",
|
||||
"AssistantKnowledgeEntityRow",
|
||||
"AssistantKnowledgeEntityContributionRow",
|
||||
"AssistantKnowledgeRelationRow",
|
||||
"AssistantKnowledgeRelationContributionRow",
|
||||
"CompactMenuOverrideRow",
|
||||
"ConcurrencySampleRow",
|
||||
"DashboardSessionRow",
|
||||
@@ -164,6 +196,16 @@ __all__ = [
|
||||
"SkillRow",
|
||||
"SkillDisplayAdapterRow",
|
||||
"SkillFavoriteRow",
|
||||
"SkillKnowledgeArtifactRow",
|
||||
"SkillKnowledgeBindingRow",
|
||||
"SkillKnowledgeEntityContributionRow",
|
||||
"SkillKnowledgeEntityRow",
|
||||
"SkillKnowledgeRelationContributionRow",
|
||||
"SkillKnowledgeRelationRow",
|
||||
"SkillKnowledgeReviewRow",
|
||||
"SkillKnowledgeSnapshotRow",
|
||||
"SkillKnowledgeSyncItemRow",
|
||||
"SkillKnowledgeSyncJobRow",
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"TaskButtonRow",
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
from deerflow.persistence.skill_knowledge.model import ( # noqa: F401
|
||||
SkillKnowledgeArtifactRow,
|
||||
SkillKnowledgeBindingRow,
|
||||
SkillKnowledgeEntityContributionRow,
|
||||
SkillKnowledgeEntityRow,
|
||||
SkillKnowledgeRelationContributionRow,
|
||||
SkillKnowledgeRelationRow,
|
||||
SkillKnowledgeReviewRow,
|
||||
SkillKnowledgeSnapshotRow,
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store # noqa: F401
|
||||
|
||||
__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
"""ORM rows for skill-to-knowledge distillation and review."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class SkillKnowledgeBindingRow(Base):
|
||||
__tablename__ = "skill_knowledge_bindings"
|
||||
__table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
skill_name: Mapped[str] = mapped_column(String(255), index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
|
||||
target_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
target_name: Mapped[str] = mapped_column(String(255), default="")
|
||||
remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
|
||||
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
|
||||
auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
|
||||
current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
|
||||
synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
|
||||
last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class SkillKnowledgeSnapshotRow(Base):
|
||||
__tablename__ = "skill_knowledge_snapshots"
|
||||
__table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
skill_name: Mapped[str] = mapped_column(String(255), index=True)
|
||||
source_digest: Mapped[str] = mapped_column(String(80), index=True)
|
||||
extractor_version: Mapped[str] = mapped_column(String(64))
|
||||
prompt_hash: Mapped[str] = mapped_column(String(80))
|
||||
model_name: Mapped[str] = mapped_column(String(128))
|
||||
manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="ready")
|
||||
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
|
||||
|
||||
class SkillKnowledgeSyncJobRow(Base):
|
||||
__tablename__ = "skill_knowledge_sync_jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
trigger: Mapped[str] = mapped_column(String(32), default="manual")
|
||||
target_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||
total_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
success_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
failed_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
|
||||
review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class SkillKnowledgeSyncItemRow(Base):
|
||||
__tablename__ = "skill_knowledge_sync_items"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32))
|
||||
target_mode: Mapped[str] = mapped_column(String(32))
|
||||
target_id: Mapped[str] = mapped_column(String(128))
|
||||
skill_name: Mapped[str] = mapped_column(String(255), index=True)
|
||||
snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
|
||||
phase: Mapped[str] = mapped_column(String(32), default="scanning")
|
||||
progress: Mapped[int] = mapped_column(Integer, default=0)
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0)
|
||||
active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class SkillKnowledgeArtifactRow(Base):
|
||||
__tablename__ = "skill_knowledge_artifacts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(48), index=True)
|
||||
slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
|
||||
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class SkillKnowledgeReviewRow(Base):
|
||||
__tablename__ = "skill_knowledge_reviews"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
item_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
object_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
object_key: Mapped[str] = mapped_column(String(128))
|
||||
original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
|
||||
reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
|
||||
|
||||
class SkillKnowledgeEntityRow(Base):
|
||||
__tablename__ = "skill_knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
entity_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
normalized_name: Mapped[str] = mapped_column(String(512))
|
||||
display_name: Mapped[str] = mapped_column(String(512))
|
||||
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class SkillKnowledgeEntityContributionRow(Base):
|
||||
__tablename__ = "skill_knowledge_entity_contributions"
|
||||
__table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
local_entity_id: Mapped[str] = mapped_column(String(128))
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
|
||||
|
||||
class SkillKnowledgeRelationRow(Base):
|
||||
__tablename__ = "skill_knowledge_relations"
|
||||
__table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
target_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
predicate: Mapped[str] = mapped_column(String(128), index=True)
|
||||
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class SkillKnowledgeRelationContributionRow(Base):
|
||||
__tablename__ = "skill_knowledge_relation_contributions"
|
||||
__table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
relation_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
local_relation_id: Mapped[str] = mapped_column(String(128))
|
||||
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
+758
@@ -0,0 +1,758 @@
|
||||
"""SQL repository for skill knowledge jobs, snapshots and contributions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.skill_knowledge.model import (
|
||||
SkillKnowledgeArtifactRow,
|
||||
SkillKnowledgeBindingRow,
|
||||
SkillKnowledgeEntityContributionRow,
|
||||
SkillKnowledgeEntityRow,
|
||||
SkillKnowledgeRelationContributionRow,
|
||||
SkillKnowledgeRelationRow,
|
||||
SkillKnowledgeReviewRow,
|
||||
SkillKnowledgeSnapshotRow,
|
||||
SkillKnowledgeSyncItemRow,
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _dict(row: Any) -> dict[str, Any]:
|
||||
list_json_fields = {
|
||||
"aliases_json",
|
||||
"evidence_json",
|
||||
"manifest_json",
|
||||
"refs_json",
|
||||
"targets_json",
|
||||
}
|
||||
data = row.to_dict()
|
||||
for key, value in tuple(data.items()):
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
elif key.endswith("_json"):
|
||||
data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
|
||||
return data
|
||||
|
||||
|
||||
class SkillKnowledgeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
|
||||
entities = (
|
||||
await session.execute(
|
||||
select(SkillKnowledgeEntityRow).where(
|
||||
SkillKnowledgeEntityRow.target_type == target_type,
|
||||
SkillKnowledgeEntityRow.target_id == target_id,
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
for entity in entities:
|
||||
contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
|
||||
entity.status = "active" if contribution else "deleted"
|
||||
relations = (
|
||||
await session.execute(
|
||||
select(SkillKnowledgeRelationRow).where(
|
||||
SkillKnowledgeRelationRow.target_type == target_type,
|
||||
SkillKnowledgeRelationRow.target_id == target_id,
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
for relation in relations:
|
||||
contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
|
||||
relation.status = "active" if contribution else "deleted"
|
||||
|
||||
async def create_job(
|
||||
self,
|
||||
*,
|
||||
skill_names: list[str],
|
||||
targets: list[dict[str, Any]],
|
||||
created_by: str | None,
|
||||
force: bool = False,
|
||||
trigger: str = "manual",
|
||||
review_mode: str = "auto_high_confidence",
|
||||
) -> dict[str, Any]:
|
||||
names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
|
||||
normalized_targets: list[dict[str, Any]] = []
|
||||
seen_targets: set[tuple[str, str]] = set()
|
||||
for target in targets:
|
||||
target_type = str(target.get("target_type") or "").strip()
|
||||
target_id = str(target.get("target_id") or "").strip()
|
||||
if not target_type or not target_id or (target_type, target_id) in seen_targets:
|
||||
continue
|
||||
seen_targets.add((target_type, target_id))
|
||||
normalized_targets.append(
|
||||
{
|
||||
"target_type": target_type,
|
||||
"target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
|
||||
"target_id": target_id,
|
||||
"target_name": str(target.get("target_name") or target_id),
|
||||
"remote_business_key": target.get("remote_business_key"),
|
||||
"readonly": bool(target.get("readonly", False)),
|
||||
}
|
||||
)
|
||||
job_id = str(uuid4())
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
job = SkillKnowledgeSyncJobRow(
|
||||
id=job_id,
|
||||
trigger=trigger,
|
||||
target_count=len(normalized_targets),
|
||||
targets_json=normalized_targets,
|
||||
status="queued",
|
||||
total_count=0,
|
||||
review_mode=review_mode,
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
)
|
||||
session.add(job)
|
||||
for skill_name in names:
|
||||
for target in normalized_targets:
|
||||
stmt = select(SkillKnowledgeBindingRow).where(
|
||||
SkillKnowledgeBindingRow.skill_name == skill_name,
|
||||
SkillKnowledgeBindingRow.target_type == target["target_type"],
|
||||
SkillKnowledgeBindingRow.target_id == target["target_id"],
|
||||
)
|
||||
binding = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if binding is None:
|
||||
binding = SkillKnowledgeBindingRow(
|
||||
id=str(uuid4()),
|
||||
skill_name=skill_name,
|
||||
target_type=target["target_type"],
|
||||
target_mode=target["target_mode"],
|
||||
target_id=target["target_id"],
|
||||
target_name=target["target_name"],
|
||||
remote_business_key=target.get("remote_business_key"),
|
||||
status="queued",
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(binding)
|
||||
else:
|
||||
active_item = (
|
||||
await session.execute(
|
||||
select(SkillKnowledgeSyncItemRow.id)
|
||||
.where(
|
||||
SkillKnowledgeSyncItemRow.binding_id == binding.id,
|
||||
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if active_item is not None:
|
||||
continue
|
||||
binding.target_mode = target["target_mode"]
|
||||
binding.target_name = target["target_name"]
|
||||
binding.remote_business_key = target.get("remote_business_key")
|
||||
binding.status = "queued"
|
||||
binding.active = True
|
||||
binding.updated_at = now
|
||||
item = SkillKnowledgeSyncItemRow(
|
||||
id=str(uuid4()),
|
||||
job_id=job_id,
|
||||
binding_id=binding.id,
|
||||
target_type=target["target_type"],
|
||||
target_mode=target["target_mode"],
|
||||
target_id=target["target_id"],
|
||||
skill_name=skill_name,
|
||||
status="queued",
|
||||
phase="scanning",
|
||||
progress=0,
|
||||
active_dedupe_key=binding.id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(item)
|
||||
binding.last_job_id = job_id
|
||||
binding.last_item_id = item.id
|
||||
job.total_count += 1
|
||||
if job.total_count == 0:
|
||||
job.status = "completed"
|
||||
job.completed_at = now
|
||||
await session.flush()
|
||||
result = _dict(job)
|
||||
result["force"] = force
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
|
||||
stmt = select(SkillKnowledgeBindingRow)
|
||||
if skill_names:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
|
||||
if target_id:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
|
||||
if not include_detached:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
|
||||
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(SkillKnowledgeBindingRow, binding_id)
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
|
||||
stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
|
||||
if active_only:
|
||||
stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def has_active_item(self, binding_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
value = (
|
||||
await session.execute(
|
||||
select(SkillKnowledgeSyncItemRow.id)
|
||||
.where(
|
||||
SkillKnowledgeSyncItemRow.binding_id == binding_id,
|
||||
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return value is not None
|
||||
|
||||
async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(SkillKnowledgeBindingRow, binding_id)
|
||||
if row is None or not row.active:
|
||||
return None
|
||||
row.auto_sync_enabled = enabled
|
||||
row.updated_at = _now()
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
|
||||
if job is None:
|
||||
return None
|
||||
items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
|
||||
result = _dict(job)
|
||||
result["items"] = [_dict(item) for item in items]
|
||||
return result
|
||||
|
||||
async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
now = _now()
|
||||
stmt = (
|
||||
select(SkillKnowledgeSyncItemRow)
|
||||
.where(
|
||||
or_(
|
||||
SkillKnowledgeSyncItemRow.status == "queued",
|
||||
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
|
||||
)
|
||||
)
|
||||
.order_by(SkillKnowledgeSyncItemRow.created_at)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def claim_item(
|
||||
self,
|
||||
item_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
lease_seconds: int = 900,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomically claim a queued item, or reclaim one whose worker lease expired."""
|
||||
now = _now()
|
||||
stmt = (
|
||||
update(SkillKnowledgeSyncItemRow)
|
||||
.where(
|
||||
SkillKnowledgeSyncItemRow.id == item_id,
|
||||
or_(
|
||||
SkillKnowledgeSyncItemRow.status == "queued",
|
||||
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="running",
|
||||
started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
|
||||
attempts=SkillKnowledgeSyncItemRow.attempts + 1,
|
||||
lease_owner=owner,
|
||||
lease_until=now + timedelta(seconds=lease_seconds),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
if result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return None
|
||||
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
await self.refresh_job(data["job_id"])
|
||||
return data
|
||||
|
||||
async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(SkillKnowledgeSyncItemRow)
|
||||
.where(
|
||||
SkillKnowledgeSyncItemRow.id == item_id,
|
||||
SkillKnowledgeSyncItemRow.status == "running",
|
||||
SkillKnowledgeSyncItemRow.lease_owner == owner,
|
||||
)
|
||||
.values(
|
||||
lease_until=now + timedelta(seconds=lease_seconds),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
async def update_item(
|
||||
self,
|
||||
item_id: str,
|
||||
*,
|
||||
status: str | None = None,
|
||||
phase: str | None = None,
|
||||
progress: int | None = None,
|
||||
snapshot_id: str | None = None,
|
||||
error_code: str | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
|
||||
if row is None:
|
||||
return None
|
||||
if status is not None:
|
||||
row.status = status
|
||||
if status == "running" and row.started_at is None:
|
||||
row.started_at = _now()
|
||||
row.attempts += 1
|
||||
if status in {"completed", "partial_failed", "failed", "canceled"}:
|
||||
row.completed_at = _now()
|
||||
row.active_dedupe_key = None
|
||||
elif status == "queued":
|
||||
row.completed_at = None
|
||||
if status != "running":
|
||||
row.lease_owner = None
|
||||
row.lease_until = None
|
||||
if phase is not None:
|
||||
row.phase = phase
|
||||
if progress is not None:
|
||||
row.progress = max(0, min(100, progress))
|
||||
if snapshot_id is not None:
|
||||
row.snapshot_id = snapshot_id
|
||||
row.last_error_code = error_code
|
||||
row.last_error_message = error_message
|
||||
row.updated_at = _now()
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
await self.refresh_job(data["job_id"])
|
||||
return data
|
||||
|
||||
async def refresh_job(self, job_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
|
||||
if job is None:
|
||||
return
|
||||
rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
|
||||
counts = {str(status): int(count) for status, count in rows}
|
||||
job.success_count = counts.get("completed", 0)
|
||||
job.partial_failed_count = counts.get("partial_failed", 0)
|
||||
job.failed_count = counts.get("failed", 0)
|
||||
active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
|
||||
if counts.get("running", 0) or counts.get("awaiting_review", 0):
|
||||
job.status = "running"
|
||||
job.started_at = job.started_at or _now()
|
||||
elif active == 0 and sum(counts.values()) >= job.total_count:
|
||||
if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
|
||||
job.status = "failed"
|
||||
elif job.failed_count or job.partial_failed_count:
|
||||
job.status = "partial_failed"
|
||||
else:
|
||||
job.status = "completed"
|
||||
job.completed_at = _now()
|
||||
await session.commit()
|
||||
|
||||
async def cancel_job(self, job_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
|
||||
if job is None:
|
||||
return False
|
||||
job.cancel_requested = True
|
||||
await session.execute(
|
||||
SkillKnowledgeSyncItemRow.__table__.update()
|
||||
.where(
|
||||
SkillKnowledgeSyncItemRow.job_id == job_id,
|
||||
SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
|
||||
)
|
||||
.values(
|
||||
status="canceled",
|
||||
phase="completed",
|
||||
completed_at=_now(),
|
||||
progress=100,
|
||||
active_dedupe_key=None,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
)
|
||||
)
|
||||
job.status = "canceled"
|
||||
job.completed_at = _now()
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
|
||||
if row is None:
|
||||
return None
|
||||
active_item = (
|
||||
await session.execute(
|
||||
select(SkillKnowledgeSyncItemRow.id)
|
||||
.where(
|
||||
SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
|
||||
SkillKnowledgeSyncItemRow.id != row.id,
|
||||
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if active_item is not None:
|
||||
return None
|
||||
row.status = "queued"
|
||||
row.phase = "scanning"
|
||||
row.progress = 0
|
||||
row.completed_at = None
|
||||
row.active_dedupe_key = row.binding_id
|
||||
row.updated_at = _now()
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
await self.refresh_job(data["job_id"])
|
||||
return data
|
||||
|
||||
async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
|
||||
stmt = select(SkillKnowledgeSnapshotRow).where(
|
||||
SkillKnowledgeSnapshotRow.skill_name == skill_name,
|
||||
SkillKnowledgeSnapshotRow.source_digest == source_digest,
|
||||
SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
|
||||
SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
|
||||
SkillKnowledgeSnapshotRow.model_name == model_name,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = SkillKnowledgeSnapshotRow(
|
||||
id=str(uuid4()),
|
||||
skill_name=skill_name,
|
||||
source_digest=source_digest,
|
||||
extractor_version=extractor_version,
|
||||
prompt_hash=prompt_hash,
|
||||
model_name=model_name,
|
||||
manifest_json=manifest,
|
||||
ir_json=ir,
|
||||
counts_json=counts,
|
||||
status="ready",
|
||||
created_at=_now(),
|
||||
)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
|
||||
rows = [
|
||||
SkillKnowledgeReviewRow(
|
||||
id=str(uuid4()),
|
||||
job_id=job_id,
|
||||
item_id=item_id,
|
||||
binding_id=binding_id,
|
||||
snapshot_id=snapshot_id,
|
||||
object_type=kind,
|
||||
object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
|
||||
original_json=obj,
|
||||
status="pending",
|
||||
created_at=_now(),
|
||||
)
|
||||
for kind, obj in objects
|
||||
]
|
||||
async with self._sf() as session:
|
||||
session.add_all(rows)
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
|
||||
stmt = select(SkillKnowledgeReviewRow)
|
||||
if job_id:
|
||||
stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
|
||||
if item_id:
|
||||
stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
|
||||
if status:
|
||||
stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
|
||||
stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
|
||||
final_status = "approved" if action == "approve" else "rejected"
|
||||
affected_items: set[str] = set()
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
|
||||
for row in rows:
|
||||
row.status = final_status
|
||||
row.edited_json = (edits or {}).get(row.id)
|
||||
row.reason = reason
|
||||
row.reviewed_by = actor
|
||||
row.reviewed_at = _now()
|
||||
affected_items.add(row.item_id)
|
||||
await session.commit()
|
||||
return sorted(affected_items)
|
||||
|
||||
async def pending_review_count(self, item_id: str) -> int:
|
||||
stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
|
||||
async with self._sf() as session:
|
||||
return int((await session.execute(stmt)).scalar_one())
|
||||
|
||||
async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
|
||||
snapshot = await self.get_snapshot(snapshot_id)
|
||||
if snapshot is None:
|
||||
return None
|
||||
ir = dict(snapshot.get("ir") or {})
|
||||
reviews = await self.list_reviews(item_id=item_id, status=None)
|
||||
if not reviews:
|
||||
return ir
|
||||
by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
|
||||
for key in ("entities", "relations"):
|
||||
kind = key[:-1]
|
||||
accepted = []
|
||||
for obj in ir.get(key, []):
|
||||
review = by_key.get((kind, str(obj.get("local_id"))))
|
||||
if review and review.get("status") == "rejected":
|
||||
continue
|
||||
accepted.append((review or {}).get("edited") or obj)
|
||||
ir[key] = accepted
|
||||
return ir
|
||||
|
||||
async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
item = await session.get(SkillKnowledgeSyncItemRow, item_id)
|
||||
if item is None:
|
||||
raise KeyError(item_id)
|
||||
binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
|
||||
if binding is None:
|
||||
raise KeyError(item.binding_id)
|
||||
if not binding.active:
|
||||
raise RuntimeError("Binding was detached before staged activation")
|
||||
await session.execute(
|
||||
SkillKnowledgeArtifactRow.__table__.update()
|
||||
.where(
|
||||
SkillKnowledgeArtifactRow.binding_id == binding.id,
|
||||
SkillKnowledgeArtifactRow.status == "active",
|
||||
)
|
||||
.values(status="retired", updated_at=now)
|
||||
)
|
||||
for artifact in artifacts:
|
||||
session.add(
|
||||
SkillKnowledgeArtifactRow(
|
||||
id=str(uuid4()),
|
||||
binding_id=binding.id,
|
||||
snapshot_id=snapshot_id,
|
||||
kind=str(artifact.get("kind") or "wiki_page"),
|
||||
slug=artifact.get("slug"),
|
||||
content_text=artifact.get("content"),
|
||||
remote_id=artifact.get("remote_id"),
|
||||
business_key=artifact.get("business_key"),
|
||||
remote_refs_json=artifact.get("remote_refs") or [],
|
||||
status="active",
|
||||
metadata_json=artifact.get("metadata") or {},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
# Retire contributions from the old active version only after all staged artifacts are ready.
|
||||
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
|
||||
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
|
||||
entity_ids: dict[str, str] = {}
|
||||
for entity in ir.get("entities", []):
|
||||
normalized = normalize_name(str(entity.get("name") or ""))
|
||||
if not normalized:
|
||||
continue
|
||||
entity_type = str(entity.get("type") or "concept")
|
||||
stmt = select(SkillKnowledgeEntityRow).where(
|
||||
SkillKnowledgeEntityRow.target_type == binding.target_type,
|
||||
SkillKnowledgeEntityRow.target_id == binding.target_id,
|
||||
SkillKnowledgeEntityRow.entity_type == entity_type,
|
||||
SkillKnowledgeEntityRow.normalized_name == normalized,
|
||||
)
|
||||
canonical = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if canonical is None:
|
||||
canonical = SkillKnowledgeEntityRow(
|
||||
id=str(uuid4()),
|
||||
target_type=binding.target_type,
|
||||
target_id=binding.target_id,
|
||||
entity_type=entity_type,
|
||||
normalized_name=normalized,
|
||||
display_name=str(entity.get("name") or normalized),
|
||||
aliases_json=entity.get("aliases") or [],
|
||||
attributes_json=entity.get("attributes") or {},
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(canonical)
|
||||
await session.flush()
|
||||
else:
|
||||
canonical.status = "active"
|
||||
entity_ids[str(entity.get("local_id"))] = canonical.id
|
||||
session.add(
|
||||
SkillKnowledgeEntityContributionRow(
|
||||
id=str(uuid4()),
|
||||
entity_id=canonical.id,
|
||||
binding_id=binding.id,
|
||||
snapshot_id=snapshot_id,
|
||||
local_entity_id=str(entity.get("local_id")),
|
||||
evidence_json=entity.get("evidence") or [],
|
||||
attributes_json=entity.get("attributes") or {},
|
||||
confidence=float(entity.get("confidence") or 0),
|
||||
)
|
||||
)
|
||||
for relation in ir.get("relations", []):
|
||||
source_id = entity_ids.get(str(relation.get("source_local_id")))
|
||||
target_id = entity_ids.get(str(relation.get("target_local_id")))
|
||||
confidence = float(relation.get("confidence") or 0)
|
||||
if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
|
||||
continue
|
||||
predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
|
||||
stmt = select(SkillKnowledgeRelationRow).where(
|
||||
SkillKnowledgeRelationRow.target_type == binding.target_type,
|
||||
SkillKnowledgeRelationRow.target_id == binding.target_id,
|
||||
SkillKnowledgeRelationRow.source_entity_id == source_id,
|
||||
SkillKnowledgeRelationRow.predicate == predicate,
|
||||
SkillKnowledgeRelationRow.target_entity_id == target_id,
|
||||
)
|
||||
canonical = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if canonical is None:
|
||||
canonical = SkillKnowledgeRelationRow(
|
||||
id=str(uuid4()),
|
||||
target_type=binding.target_type,
|
||||
target_id=binding.target_id,
|
||||
source_entity_id=source_id,
|
||||
predicate=predicate,
|
||||
target_entity_id=target_id,
|
||||
description=str(relation.get("description") or ""),
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(canonical)
|
||||
await session.flush()
|
||||
else:
|
||||
canonical.status = "active"
|
||||
session.add(
|
||||
SkillKnowledgeRelationContributionRow(
|
||||
id=str(uuid4()),
|
||||
relation_id=canonical.id,
|
||||
binding_id=binding.id,
|
||||
snapshot_id=snapshot_id,
|
||||
local_relation_id=str(relation.get("local_id")),
|
||||
evidence_json=relation.get("evidence") or [],
|
||||
confidence=confidence,
|
||||
)
|
||||
)
|
||||
await self._retire_target_orphans(
|
||||
session,
|
||||
target_type=binding.target_type,
|
||||
target_id=binding.target_id,
|
||||
)
|
||||
binding.current_snapshot_id = snapshot_id
|
||||
binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
|
||||
binding.synced_digest = binding.source_digest
|
||||
binding.status = status
|
||||
binding.last_synced_at = now
|
||||
binding.last_error_code = None
|
||||
binding.last_error_message = None
|
||||
binding.updated_at = now
|
||||
item.snapshot_id = snapshot_id
|
||||
item.status = "completed" if status in {"synced", "pending_remote_write"} else status
|
||||
item.phase = "completed"
|
||||
item.progress = 100
|
||||
item.completed_at = now
|
||||
item.active_dedupe_key = None
|
||||
item.lease_owner = None
|
||||
item.lease_until = None
|
||||
item.updated_at = now
|
||||
await session.flush()
|
||||
result = _dict(binding)
|
||||
await session.commit()
|
||||
await self.refresh_job(item.job_id)
|
||||
return result
|
||||
|
||||
async def detach_binding(
|
||||
self,
|
||||
binding_id: str,
|
||||
*,
|
||||
delete_contributions: bool,
|
||||
cleanup_failed: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
binding = await session.get(SkillKnowledgeBindingRow, binding_id)
|
||||
if binding is None:
|
||||
return None
|
||||
binding.active = False
|
||||
binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
|
||||
binding.updated_at = _now()
|
||||
if delete_contributions:
|
||||
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
|
||||
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
|
||||
await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
|
||||
await self._retire_target_orphans(
|
||||
session,
|
||||
target_type=binding.target_type,
|
||||
target_id=binding.target_id,
|
||||
)
|
||||
await session.flush()
|
||||
data = _dict(binding)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def mark_skill_stale(self, skill_name: str) -> int:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
SkillKnowledgeBindingRow.__table__.update()
|
||||
.where(
|
||||
SkillKnowledgeBindingRow.skill_name == skill_name,
|
||||
SkillKnowledgeBindingRow.active.is_(True),
|
||||
SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
|
||||
)
|
||||
.values(status="stale", updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
def make_skill_knowledge_store(session_factory):
|
||||
return SkillKnowledgeRepository(session_factory) if session_factory is not None else None
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
"""Skill knowledge distillation primitives.
|
||||
|
||||
The package is deliberately independent from ``app.*``. Gateway composition
|
||||
injects persistence and remote adapters at runtime.
|
||||
"""
|
||||
|
||||
from deerflow.skill_knowledge.extractor import build_skill_ir
|
||||
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
|
||||
|
||||
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"""IR construction and validation.
|
||||
|
||||
V1 intentionally starts with deterministic, evidence-backed extraction. The
|
||||
schema is compatible with a future model-backed extractor, while never letting
|
||||
unvalidated model output flow directly into persistence or graph writers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.skill_knowledge.scanner import ScanResult
|
||||
|
||||
EXTRACTOR_VERSION = "skill-knowledge-v1"
|
||||
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
|
||||
PREDICATE_PATTERNS = {
|
||||
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
|
||||
}
|
||||
|
||||
|
||||
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
|
||||
line_start = text.count("\n", 0, start) + 1
|
||||
line_end = text.count("\n", 0, end) + 1
|
||||
quote = " ".join(text[start:end].split())[:240]
|
||||
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
|
||||
|
||||
|
||||
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
|
||||
entities: list[dict[str, Any]] = [
|
||||
{
|
||||
"local_id": "ent_skill",
|
||||
"type": "skill",
|
||||
"name": display_name or scan.skill_name,
|
||||
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
|
||||
"description": description,
|
||||
"attributes": {},
|
||||
"evidence": [],
|
||||
"confidence": 1.0,
|
||||
}
|
||||
]
|
||||
relations: list[dict[str, Any]] = []
|
||||
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
|
||||
|
||||
for file in scan.files:
|
||||
if not file.text:
|
||||
continue
|
||||
for predicate, pattern in PREDICATE_PATTERNS.items():
|
||||
for match in pattern.finditer(file.text):
|
||||
name = match.group(1).strip(" `\"'“”()[]")
|
||||
if not name or len(name) > 80:
|
||||
continue
|
||||
key = ("concept", name.casefold())
|
||||
entity_id = by_key.get(key)
|
||||
if entity_id is None:
|
||||
entity_id = f"ent_{len(entities):04d}"
|
||||
by_key[key] = entity_id
|
||||
entities.append(
|
||||
{
|
||||
"local_id": entity_id,
|
||||
"type": "concept",
|
||||
"name": name,
|
||||
"aliases": [],
|
||||
"description": "从技能业务说明中抽取的知识对象",
|
||||
"attributes": {},
|
||||
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
|
||||
"confidence": 0.82,
|
||||
}
|
||||
)
|
||||
relations.append(
|
||||
{
|
||||
"local_id": f"rel_{len(relations):04d}",
|
||||
"source_local_id": "ent_skill",
|
||||
"predicate": predicate,
|
||||
"target_local_id": entity_id,
|
||||
"description": " ".join(match.group(0).split())[:240],
|
||||
"relation_source": "explicit_text",
|
||||
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
|
||||
"confidence": 0.82,
|
||||
}
|
||||
)
|
||||
for ref in file.references:
|
||||
name = ref["target_path"]
|
||||
key = ("file", name.casefold())
|
||||
entity_id = by_key.get(key)
|
||||
if entity_id is None:
|
||||
entity_id = f"ent_{len(entities):04d}"
|
||||
by_key[key] = entity_id
|
||||
entities.append(
|
||||
{
|
||||
"local_id": entity_id,
|
||||
"type": "file",
|
||||
"name": name,
|
||||
"aliases": [],
|
||||
"description": "技能文档引用的资料文件",
|
||||
"attributes": {},
|
||||
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
|
||||
"confidence": 1.0,
|
||||
}
|
||||
)
|
||||
relations.append(
|
||||
{
|
||||
"local_id": f"rel_{len(relations):04d}",
|
||||
"source_local_id": "ent_skill",
|
||||
"predicate": "references",
|
||||
"target_local_id": entity_id,
|
||||
"description": f"{file.path} 引用 {name}",
|
||||
"relation_source": "deterministic_structure",
|
||||
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
|
||||
"confidence": 1.0,
|
||||
}
|
||||
)
|
||||
|
||||
title = display_name or scan.skill_name
|
||||
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
|
||||
content = "\n".join(
|
||||
[
|
||||
f"# {title}",
|
||||
"",
|
||||
description or "由技能知识文件安全归纳生成。",
|
||||
"",
|
||||
"## 知识资料",
|
||||
"",
|
||||
*(summaries or ["- 暂无可归纳知识文件"]),
|
||||
"",
|
||||
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
|
||||
]
|
||||
)
|
||||
return {
|
||||
"extractor_version": EXTRACTOR_VERSION,
|
||||
"prompt_hash": PROMPT_HASH,
|
||||
"model_name": "deterministic-v1",
|
||||
"skill": {
|
||||
"name": scan.skill_name,
|
||||
"display_name": title,
|
||||
"description": description,
|
||||
"source_root": scan.source_root,
|
||||
"source_digest": scan.source_digest,
|
||||
},
|
||||
"files": scan.manifest,
|
||||
"entities": entities,
|
||||
"relations": relations,
|
||||
"wiki": {
|
||||
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
|
||||
"pages": [],
|
||||
},
|
||||
"safety": {"blocked_files": scan.blocked_files},
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
"""Guarded graph-write plans.
|
||||
|
||||
No caller can submit Cypher. The writer accepts only normalized node/edge
|
||||
records and executes fixed parameterized templates when explicitly enabled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
class GraphTransaction(Protocol):
|
||||
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GraphWritePlan:
|
||||
scope: str
|
||||
nodes: tuple[dict[str, Any], ...]
|
||||
edges: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
NODE_QUERY = """
|
||||
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
|
||||
SET n.display_name = $display_name, n.entity_type = $entity_type,
|
||||
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
|
||||
n.confidence = $confidence, n.updated_at = datetime()
|
||||
""".strip()
|
||||
EDGE_QUERY = """
|
||||
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
|
||||
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
|
||||
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
|
||||
SET r.predicate = $predicate, r.binding_id = $binding_id,
|
||||
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
|
||||
r.evidence = $evidence, r.updated_at = datetime()
|
||||
""".strip()
|
||||
|
||||
|
||||
def _key(*parts: str) -> str:
|
||||
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
|
||||
|
||||
|
||||
def plan_from_ir(
|
||||
ir: dict[str, Any],
|
||||
*,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
binding_id: str,
|
||||
snapshot_id: str,
|
||||
job_id: str,
|
||||
) -> GraphWritePlan:
|
||||
"""Build a stable write plan; callers cannot supply executable Cypher."""
|
||||
scope = f"{target_type}:{target_id}"
|
||||
local_keys: dict[str, str] = {}
|
||||
nodes: list[dict[str, Any]] = []
|
||||
for entity in ir.get("entities") or []:
|
||||
entity_type = str(entity.get("type") or "concept")
|
||||
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
|
||||
if not normalized_name:
|
||||
continue
|
||||
entity_key = _key(entity_type, normalized_name)
|
||||
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
|
||||
nodes.append(
|
||||
{
|
||||
"entity_key": entity_key,
|
||||
"display_name": str(entity.get("name") or normalized_name),
|
||||
"entity_type": entity_type,
|
||||
"binding_id": binding_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"job_id": job_id,
|
||||
"confidence": float(entity.get("confidence") or 1.0),
|
||||
}
|
||||
)
|
||||
edges: list[dict[str, Any]] = []
|
||||
for relation in ir.get("relations") or []:
|
||||
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
|
||||
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
|
||||
evidence = list(relation.get("evidence") or [])
|
||||
confidence = float(relation.get("confidence") or 0)
|
||||
predicate = str(relation.get("predicate") or "related_to")
|
||||
if not source_key or not target_key or confidence < 0.75 or not evidence:
|
||||
continue
|
||||
edges.append(
|
||||
{
|
||||
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
|
||||
"source_key": source_key,
|
||||
"target_key": target_key,
|
||||
"predicate": predicate,
|
||||
"binding_id": binding_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"job_id": job_id,
|
||||
"confidence": confidence,
|
||||
"evidence": evidence,
|
||||
}
|
||||
)
|
||||
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
|
||||
|
||||
|
||||
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
|
||||
"""Return Neo4j transactional HTTP statements using only fixed templates."""
|
||||
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
|
||||
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
|
||||
return statements
|
||||
|
||||
|
||||
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
|
||||
if not enabled:
|
||||
return {"nodes": 0, "edges": 0}
|
||||
for node in plan.nodes:
|
||||
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
|
||||
for edge in plan.edges:
|
||||
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
|
||||
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Portable entity/relation normalization used by both knowledge products."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
def normalize_name(value: str) -> str:
|
||||
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
|
||||
text = text.strip("`'\"“”‘’《》〈〉[](){}")
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_predicate(value: str) -> str:
|
||||
normalized = normalize_name(value).replace(" ", "_")
|
||||
aliases = {
|
||||
"依赖": "depends_on",
|
||||
"需要": "depends_on",
|
||||
"requires": "depends_on",
|
||||
"调用": "uses",
|
||||
"使用": "uses",
|
||||
"uses": "uses",
|
||||
"生成": "produces",
|
||||
"输出": "produces",
|
||||
"读取": "consumes",
|
||||
"输入": "consumes",
|
||||
"引用": "references",
|
||||
}
|
||||
return aliases.get(normalized, normalized or "related_to")
|
||||
+351
@@ -0,0 +1,351 @@
|
||||
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
CODE_SUFFIXES = {
|
||||
".py",
|
||||
".pyc",
|
||||
".pyo",
|
||||
".ts",
|
||||
".tsx",
|
||||
".js",
|
||||
".jsx",
|
||||
".mjs",
|
||||
".cjs",
|
||||
".sh",
|
||||
".bash",
|
||||
".zsh",
|
||||
".fish",
|
||||
".bat",
|
||||
".cmd",
|
||||
".ps1",
|
||||
".psm1",
|
||||
".java",
|
||||
".kt",
|
||||
".kts",
|
||||
".go",
|
||||
".rs",
|
||||
".c",
|
||||
".cc",
|
||||
".cpp",
|
||||
".h",
|
||||
".hpp",
|
||||
".cs",
|
||||
".php",
|
||||
".rb",
|
||||
".pl",
|
||||
".lua",
|
||||
".swift",
|
||||
".scala",
|
||||
".sql",
|
||||
".vue",
|
||||
".svelte",
|
||||
}
|
||||
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
|
||||
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
|
||||
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
|
||||
EXCLUDED_FILENAMES = {
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"bun.lockb",
|
||||
"poetry.lock",
|
||||
"uv.lock",
|
||||
"pipfile.lock",
|
||||
"cargo.lock",
|
||||
"composer.lock",
|
||||
}
|
||||
SENSITIVE_NAME_RE = re.compile(
|
||||
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
|
||||
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
|
||||
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
|
||||
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
|
||||
MAX_TEXT_BYTES = 2 * 1024 * 1024
|
||||
MAX_ARCHIVE_FILES = 1_000
|
||||
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
|
||||
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScannedFile:
|
||||
path: str
|
||||
kind: str
|
||||
sha256: str
|
||||
size: int
|
||||
text: str = ""
|
||||
title: str = ""
|
||||
references: list[dict[str, Any]] = field(default_factory=list)
|
||||
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
|
||||
|
||||
def manifest(self) -> dict[str, Any]:
|
||||
return {
|
||||
"path": self.path,
|
||||
"kind": self.kind,
|
||||
"sha256": self.sha256,
|
||||
"size": self.size,
|
||||
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
|
||||
"title": self.title,
|
||||
"references": self.references,
|
||||
"safety": self.safety,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScanResult:
|
||||
skill_name: str
|
||||
source_root: str
|
||||
source_digest: str
|
||||
files: list[ScannedFile]
|
||||
blocked_files: list[dict[str, str]]
|
||||
skipped_code_count: int
|
||||
skipped_other_count: int
|
||||
|
||||
@property
|
||||
def manifest(self) -> list[dict[str, Any]]:
|
||||
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
|
||||
|
||||
@property
|
||||
def counts(self) -> dict[str, int]:
|
||||
return {
|
||||
"knowledge_files": len(self.files),
|
||||
"blocked_files": len(self.blocked_files),
|
||||
"skipped_code_files": self.skipped_code_count,
|
||||
"skipped_other_files": self.skipped_other_count,
|
||||
}
|
||||
|
||||
|
||||
def _safe_relative(path: Path, root: Path) -> str:
|
||||
resolved = path.resolve()
|
||||
resolved_root = root.resolve()
|
||||
try:
|
||||
relative = resolved.relative_to(resolved_root)
|
||||
except ValueError as exc:
|
||||
raise ValueError("Skill file escapes its source root") from exc
|
||||
return relative.as_posix()
|
||||
|
||||
|
||||
def _strip_markdown_code(text: str) -> str:
|
||||
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
|
||||
|
||||
|
||||
def _read_text(path: Path, suffix: str) -> str:
|
||||
raw = path.read_bytes()
|
||||
if len(raw) > MAX_TEXT_BYTES:
|
||||
raise ValueError("text file exceeds 2 MiB parsing limit")
|
||||
text = raw.decode("utf-8-sig", errors="replace")
|
||||
if suffix in {".md", ".mdx"}:
|
||||
return _strip_markdown_code(text)
|
||||
if suffix in {".csv", ".tsv"}:
|
||||
delimiter = "\t" if suffix == ".tsv" else ","
|
||||
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
|
||||
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
|
||||
if suffix == ".json":
|
||||
try:
|
||||
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
|
||||
except json.JSONDecodeError:
|
||||
return text
|
||||
return text
|
||||
|
||||
|
||||
def _title(text: str, fallback: str) -> str:
|
||||
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
|
||||
return match.group(1).strip() if match else fallback
|
||||
|
||||
|
||||
def _kind(suffix: str) -> str:
|
||||
return {
|
||||
".md": "markdown",
|
||||
".mdx": "markdown",
|
||||
".txt": "text",
|
||||
".csv": "table",
|
||||
".tsv": "table",
|
||||
".xlsx": "spreadsheet",
|
||||
".xls": "spreadsheet",
|
||||
".pdf": "pdf",
|
||||
".docx": "office",
|
||||
".pptx": "office",
|
||||
".png": "image",
|
||||
".jpg": "image",
|
||||
".jpeg": "image",
|
||||
".webp": "image",
|
||||
".gif": "image",
|
||||
".bmp": "image",
|
||||
".zip": "archive",
|
||||
}.get(suffix, "structured_data")
|
||||
|
||||
|
||||
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
|
||||
files: list[ScannedFile] = []
|
||||
blocked: list[dict[str, str]] = []
|
||||
skipped_code = 0
|
||||
skipped_other = 0
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
members = archive.infolist()
|
||||
if len(members) > MAX_ARCHIVE_FILES:
|
||||
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
|
||||
total_size = sum(member.file_size for member in members)
|
||||
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
|
||||
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
|
||||
for member in members:
|
||||
member_path = Path(member.filename.replace("\\", "/"))
|
||||
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
|
||||
unix_mode = member.external_attr >> 16
|
||||
if member.is_dir():
|
||||
continue
|
||||
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
|
||||
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
|
||||
continue
|
||||
suffix = member_path.suffix.lower()
|
||||
lower_name = member_path.name.lower()
|
||||
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
|
||||
skipped_code += 1
|
||||
continue
|
||||
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
|
||||
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
|
||||
continue
|
||||
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
|
||||
skipped_other += 1
|
||||
continue
|
||||
raw = archive.read(member)
|
||||
text = raw.decode("utf-8-sig", errors="replace")
|
||||
if suffix in {".md", ".mdx"}:
|
||||
text = _strip_markdown_code(text)
|
||||
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
|
||||
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
|
||||
continue
|
||||
files.append(
|
||||
ScannedFile(
|
||||
path=virtual_path,
|
||||
kind=_kind(suffix),
|
||||
sha256=hashlib.sha256(raw).hexdigest(),
|
||||
size=len(raw),
|
||||
text=text,
|
||||
title=_title(text, member_path.stem),
|
||||
)
|
||||
)
|
||||
return files, blocked, skipped_code, skipped_other
|
||||
|
||||
|
||||
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
|
||||
"""Scan one already-resolved skill directory without following symlinks."""
|
||||
|
||||
root = skill_root.resolve()
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
|
||||
files: list[ScannedFile] = []
|
||||
blocked: list[dict[str, str]] = []
|
||||
skipped_code = 0
|
||||
skipped_other = 0
|
||||
|
||||
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
continue
|
||||
relative = _safe_relative(path, root)
|
||||
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
|
||||
continue
|
||||
lower_name = path.name.lower()
|
||||
suffix = path.suffix.lower()
|
||||
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
|
||||
skipped_code += 1
|
||||
continue
|
||||
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
|
||||
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
|
||||
continue
|
||||
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
|
||||
skipped_other += 1
|
||||
continue
|
||||
|
||||
# Never upload an archive wholesale: it may contain excluded code or a
|
||||
# blocked secret. Only independently validated knowledge members enter
|
||||
# the manifest and digest.
|
||||
if suffix == ".zip":
|
||||
try:
|
||||
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
|
||||
files.extend(nested)
|
||||
blocked.extend(nested_blocked)
|
||||
skipped_code += nested_code
|
||||
skipped_other += nested_other
|
||||
except (OSError, zipfile.BadZipFile):
|
||||
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
|
||||
continue
|
||||
|
||||
raw = path.read_bytes()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
text = ""
|
||||
warnings: list[str] = []
|
||||
if suffix in TEXT_SUFFIXES:
|
||||
try:
|
||||
text = _read_text(path, suffix)
|
||||
except ValueError as exc:
|
||||
warnings.append(str(exc))
|
||||
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
|
||||
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
|
||||
continue
|
||||
references = []
|
||||
if suffix in {".md", ".mdx"}:
|
||||
for target in MARKDOWN_LINK_RE.findall(text):
|
||||
normalized = target.replace("\\", "/").lstrip("./")
|
||||
if normalized:
|
||||
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
|
||||
files.append(
|
||||
ScannedFile(
|
||||
path=relative,
|
||||
kind=_kind(suffix),
|
||||
sha256=digest,
|
||||
size=len(raw),
|
||||
text=text,
|
||||
title=_title(text, path.stem),
|
||||
references=references,
|
||||
safety={"status": "allowed", "warnings": warnings},
|
||||
)
|
||||
)
|
||||
|
||||
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
|
||||
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
|
||||
return ScanResult(
|
||||
skill_name=skill_name,
|
||||
source_root=str(root),
|
||||
source_digest=source_digest,
|
||||
files=files,
|
||||
blocked_files=blocked,
|
||||
skipped_code_count=skipped_code,
|
||||
skipped_other_count=skipped_other,
|
||||
)
|
||||
|
||||
|
||||
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
|
||||
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
|
||||
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
|
||||
|
||||
allowed: list[ScannedFile] = []
|
||||
root = Path(scan.source_root)
|
||||
for item in scan.files:
|
||||
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
|
||||
text = await extract_file_to_markdown(root / item.path)
|
||||
if text:
|
||||
text = _strip_markdown_code(text)
|
||||
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
|
||||
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
|
||||
continue
|
||||
item.text = text
|
||||
item.title = _title(text, item.title)
|
||||
else:
|
||||
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
|
||||
allowed.append(item)
|
||||
scan.files = allowed
|
||||
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
|
||||
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
|
||||
return scan
|
||||
@@ -167,6 +167,24 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
async def extract_file_to_markdown(file_path: Path) -> str | None:
|
||||
"""Extract a supported document to Markdown without creating a sidecar file.
|
||||
|
||||
Knowledge-ingest scanners use this variant because running a read-only
|
||||
distillation task must never add generated ``.md`` files to a live skill.
|
||||
Conversion always runs off the event loop; Office/PDF parsers can be CPU
|
||||
intensive even for deceptively small inputs.
|
||||
"""
|
||||
|
||||
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
|
||||
return None
|
||||
try:
|
||||
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
|
||||
except Exception as exc:
|
||||
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
|
||||
return None
|
||||
|
||||
|
||||
# Regex for bold-only lines that look like section headings.
|
||||
# Targets SEC filing structural headings that pymupdf4llm renders as **bold**
|
||||
# rather than # Markdown headings (because they use same font size as body text,
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
)
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repository():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
yield AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_base_entity_alignment_search_and_revision(repository: AssistantKnowledgeRepository) -> None:
|
||||
first = await repository.initialize(created_by="admin")
|
||||
second = await repository.initialize(created_by="someone-else")
|
||||
assert first["id"] == second["id"]
|
||||
|
||||
package = {
|
||||
"source": {"type": "skill"},
|
||||
"wiki_pages": [{"slug": "skills/demo/overview", "title": "示例知识", "content": "张三负责测试。"}],
|
||||
"entities": [{"local_id": "e1", "type": "person", "name": "张三", "aliases": ["Zhang San"]}],
|
||||
"relations": [],
|
||||
"documents": [],
|
||||
"chunks": [],
|
||||
"graph": {"nodes": [], "edges": []},
|
||||
}
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key="demo",
|
||||
source_name="示例技能",
|
||||
trigger="skill_direct",
|
||||
created_by="admin",
|
||||
skill_name="demo",
|
||||
)
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key="kb-1",
|
||||
source_name="普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
results = await repository.list_pages(query="张三")
|
||||
alias_results = await repository.list_pages(query="Zhang San")
|
||||
revisions = await repository.list_revisions("skills/demo/overview")
|
||||
assert {row["page_type"] for row in results} == {"source", "entity"}
|
||||
assert [row["page_type"] for row in alias_results] == ["entity"]
|
||||
assert len(revisions) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
package = {
|
||||
"source": {},
|
||||
"wiki_pages": [],
|
||||
"entities": [{"local_id": "e1", "type": "concept", "name": "共享实体"}],
|
||||
"relations": [],
|
||||
}
|
||||
for source_key in ("one", "two"):
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key=source_key,
|
||||
source_name=source_key,
|
||||
trigger="skill_direct",
|
||||
created_by="admin",
|
||||
skill_name=source_key,
|
||||
)
|
||||
|
||||
sources = await repository.list_sources()
|
||||
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "one"))
|
||||
async with repository._sf() as session:
|
||||
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
|
||||
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow))).scalars().all()
|
||||
assert entity.status == "active"
|
||||
assert len(contributions) == 1
|
||||
|
||||
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "two"))
|
||||
async with repository._sf() as session:
|
||||
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
|
||||
assert entity.status == "deleted"
|
||||
@@ -0,0 +1,62 @@
|
||||
import pytest
|
||||
|
||||
from deerflow.skill_knowledge.graph_writer import (
|
||||
EDGE_QUERY,
|
||||
NODE_QUERY,
|
||||
execute_plan,
|
||||
parameterized_statements,
|
||||
plan_from_ir,
|
||||
)
|
||||
|
||||
|
||||
class RecordingTransaction:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
async def run(self, query: str, parameters: dict) -> None:
|
||||
self.calls.append((query, parameters))
|
||||
|
||||
|
||||
def _plan():
|
||||
return plan_from_ir(
|
||||
{
|
||||
"entities": [
|
||||
{"local_id": "a", "type": "person", "name": "张三", "confidence": 1},
|
||||
{"local_id": "b", "type": "system", "name": "知识库", "confidence": 1},
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"source_local_id": "a",
|
||||
"target_local_id": "b",
|
||||
"predicate": "uses",
|
||||
"confidence": 0.9,
|
||||
"evidence": [{"file": "SKILL.md", "line": 2}],
|
||||
}
|
||||
],
|
||||
},
|
||||
target_type="weknora",
|
||||
target_id="kb-1",
|
||||
binding_id="binding-1",
|
||||
snapshot_id="snapshot-1",
|
||||
job_id="job-1",
|
||||
)
|
||||
|
||||
|
||||
def test_direct_graph_plan_uses_only_fixed_parameterized_templates() -> None:
|
||||
statements = parameterized_statements(_plan())
|
||||
assert len(statements) == 3
|
||||
assert {row["statement"] for row in statements} == {NODE_QUERY, EDGE_QUERY}
|
||||
assert all("parameters" in row for row in statements)
|
||||
assert all("张三" not in row["statement"] for row in statements)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_graph_writer_obeys_explicit_enable_switch() -> None:
|
||||
transaction = RecordingTransaction()
|
||||
disabled = await execute_plan(transaction, _plan(), enabled=False)
|
||||
assert disabled == {"nodes": 0, "edges": 0}
|
||||
assert transaction.calls == []
|
||||
|
||||
enabled = await execute_plan(transaction, _plan(), enabled=True)
|
||||
assert enabled == {"nodes": 2, "edges": 1}
|
||||
assert len(transaction.calls) == 3
|
||||
@@ -0,0 +1,95 @@
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.skill_knowledge import SkillKnowledgeRepository
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repository():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
yield SkillKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_matrix_job_creates_one_item_and_binding_per_pair(repository: SkillKnowledgeRepository) -> None:
|
||||
job = await repository.create_job(
|
||||
skill_names=["a", "b", "c"],
|
||||
targets=[
|
||||
{"target_type": "weknora", "target_mode": "wiki", "target_id": "kb-1", "target_name": "一"},
|
||||
{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"},
|
||||
],
|
||||
created_by="admin",
|
||||
)
|
||||
detail = await repository.get_job(job["id"])
|
||||
bindings = await repository.list_bindings()
|
||||
|
||||
assert detail is not None
|
||||
assert detail["total_count"] == 6
|
||||
assert len(detail["items"]) == 6
|
||||
assert len(bindings) == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detach_delete_only_removes_binding_contributions(repository: SkillKnowledgeRepository) -> None:
|
||||
job = await repository.create_job(
|
||||
skill_names=["a", "b"],
|
||||
targets=[{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"}],
|
||||
created_by="admin",
|
||||
review_mode="off",
|
||||
)
|
||||
detail = await repository.get_job(job["id"])
|
||||
assert detail is not None
|
||||
binding = (await repository.list_bindings(skill_names=["a"]))[0]
|
||||
detached = await repository.detach_binding(binding["id"], delete_contributions=True)
|
||||
assert detached and detached["status"] == "detached_deleted"
|
||||
assert len(await repository.list_bindings()) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_item_claim_is_atomic_and_records_worker_lease(repository: SkillKnowledgeRepository) -> None:
|
||||
job = await repository.create_job(
|
||||
skill_names=["a"],
|
||||
targets=[
|
||||
{
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": "base",
|
||||
"target_name": "知识梳理",
|
||||
}
|
||||
],
|
||||
created_by="admin",
|
||||
)
|
||||
detail = await repository.get_job(job["id"])
|
||||
item_id = detail["items"][0]["id"]
|
||||
|
||||
first = await repository.claim_item(item_id, owner="worker-a")
|
||||
second = await repository.claim_item(item_id, owner="worker-b")
|
||||
|
||||
assert first and first["lease_owner"] == "worker-a"
|
||||
assert first["attempts"] == 1
|
||||
assert second is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_binding_allows_only_one_active_item(repository: SkillKnowledgeRepository) -> None:
|
||||
target = {
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": "base",
|
||||
"target_name": "知识梳理",
|
||||
}
|
||||
first = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
|
||||
duplicate = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
|
||||
assert first["total_count"] == 1
|
||||
assert duplicate["total_count"] == 0
|
||||
assert duplicate["status"] == "completed"
|
||||
|
||||
detail = await repository.get_job(first["id"])
|
||||
await repository.update_item(detail["items"][0]["id"], status="failed")
|
||||
retry = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
|
||||
assert retry["total_count"] == 1
|
||||
@@ -0,0 +1,52 @@
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
from deerflow.skill_knowledge import build_skill_ir, scan_skill
|
||||
|
||||
|
||||
def test_scanner_excludes_code_blocks_source_and_sensitive_files(tmp_path: Path) -> None:
|
||||
(tmp_path / "SKILL.md").write_text(
|
||||
"# 示例\n本技能使用 客户画像数据。\n```python\nPASSWORD = 'must-not-enter-ir'\n```\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "worker.py").write_text("# uses hidden system\nSECRET='never-read'", encoding="utf-8")
|
||||
(tmp_path / ".env").write_text("TOKEN=super-secret-value", encoding="utf-8")
|
||||
|
||||
scan = scan_skill("example", tmp_path)
|
||||
ir = build_skill_ir(scan)
|
||||
|
||||
assert scan.skipped_code_count == 1
|
||||
assert scan.blocked_files == [{"path": ".env", "reason": "疑似密钥或敏感配置文件"}]
|
||||
assert "must-not-enter-ir" not in str(ir)
|
||||
assert "never-read" not in str(ir)
|
||||
assert any(relation["predicate"] == "uses" for relation in ir["relations"])
|
||||
|
||||
|
||||
def test_code_only_change_does_not_change_digest(tmp_path: Path) -> None:
|
||||
(tmp_path / "SKILL.md").write_text("# Stable knowledge", encoding="utf-8")
|
||||
code = tmp_path / "worker.py"
|
||||
code.write_text("print(1)", encoding="utf-8")
|
||||
before = scan_skill("example", tmp_path).source_digest
|
||||
code.write_text("print(2)", encoding="utf-8")
|
||||
after = scan_skill("example", tmp_path).source_digest
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_zip_is_inspected_and_only_safe_knowledge_members_are_read(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "materials.zip"
|
||||
with ZipFile(archive, "w") as output:
|
||||
output.writestr("guide.md", "# 操作手册\n仅提炼这段知识。")
|
||||
output.writestr("worker.py", "SECRET = 'code-must-not-be-read'")
|
||||
output.writestr("../escape.md", "path traversal")
|
||||
|
||||
scan = scan_skill("example", tmp_path)
|
||||
|
||||
assert [file.path for file in scan.files] == ["materials.zip!/guide.md"]
|
||||
assert scan.skipped_code_count == 1
|
||||
assert scan.blocked_files == [
|
||||
{
|
||||
"path": "materials.zip!/../escape.md",
|
||||
"reason": "压缩包包含路径穿越或符号链接",
|
||||
}
|
||||
]
|
||||
assert "code-must-not-be-read" not in str(scan.manifest)
|
||||
Reference in new issue
Block a user