From dea4697ea091b0c7b9fb51461be2b237de7042a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Fri, 4 Sep 2026 18:41:07 +0800 Subject: [PATCH] feat: improve knowledge sync and entity alignment --- docs/WIKI_ROUNDTRIP_ACCEPTANCE_20260903.md | 10 +- frontend-web/src/components/page-sidebar.tsx | 14 +- .../workspace/settings/settings-dialog.tsx | 31 +- .../settings/skill-settings-page.tsx | 13 +- .../workspace/skills/SkillKnowledgePanel.tsx | 77 +- .../workspace/workspace-nav-menu.tsx | 21 +- frontend-web/src/pages/AdminSkillsPage.tsx | 14 +- .../src/pages/AssistantKnowledgePage.tsx | 265 ++++- frontend-web/src/pages/SkillsPage.tsx | 52 +- frontend-web/src/pages/WorkspaceRoutes.tsx | 291 +++--- .../api/assistant-knowledge.ts | 45 + .../src/strategy-components/api/llmwiki.ts | 294 +++++- .../api/skill-knowledge.ts | 19 +- .../llmwiki/WeKnoraKnowledgeDetailPage.tsx | 384 +++---- .../llmwiki/WeKnoraKnowledgeWorkspace.tsx | 131 ++- .../llmwiki/WeKnoraPackageTransfersPage.tsx | 221 ++++ .../llmwiki/WikiLibraryReader.tsx | 26 +- offline-backend-20260512/backend/CLAUDE.md | 49 +- offline-backend-20260512/backend/README.md | 53 +- .../backend/app/gateway/deps.py | 11 + .../app/gateway/llmwiki_index_scheduler.py | 9 +- .../backend/app/gateway/llmwiki_rag.py | 12 +- .../gateway/routers/assistant_knowledge.py | 83 +- .../backend/app/gateway/routers/llmwiki.py | 23 +- .../app/gateway/routers/llmwiki_index.py | 154 ++- .../app/gateway/routers/skill_knowledge.py | 116 ++- offline-backend-20260512/backend/config.yaml | 12 +- .../harness/deerflow/config/llmwiki_config.py | 5 +- .../deerflow/config/system_settings.py | 12 + .../weknora/local_index/embedding.py | 305 +++++- .../weknora/local_index/embedding_worker.py | 98 ++ .../integrations/weknora/local_index/sync.py | 5 +- .../persistence/assistant_knowledge/store.py | 969 +++++++++++++++++- .../persistence/skill_knowledge/store.py | 43 +- .../backend/pyproject.toml | 2 +- .../test_assistant_knowledge_repository.py | 262 +++++ .../backend/tests/test_llmwiki_wiki_sync.py | 13 +- .../tests/test_wiki_package_roundtrip.py | 232 ++++- 38 files changed, 3742 insertions(+), 634 deletions(-) create mode 100644 frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraPackageTransfersPage.tsx create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding_worker.py diff --git a/docs/WIKI_ROUNDTRIP_ACCEPTANCE_20260903.md b/docs/WIKI_ROUNDTRIP_ACCEPTANCE_20260903.md index 69d8a887..21d1cde2 100644 --- a/docs/WIKI_ROUNDTRIP_ACCEPTANCE_20260903.md +++ b/docs/WIKI_ROUNDTRIP_ACCEPTANCE_20260903.md @@ -14,7 +14,7 @@ WeKnora;没有把业务代码发布到服务器,没有提交 Git,也没有 - 实现:FastEmbed / ONNX Runtime CPU,官方 `BAAI/bge-m3` 权重。 - 配置名:`bge-embedding-m3`,1024 维,每批 2 条,2 个 CPU 推理线程。 -- 运行依赖:后端 `local-embedding` extra;当前本地虚拟环境已安装。 +- 运行依赖:FastEmbed 随标准后端依赖安装;模型必须提前放在离线目录中,仍按需加载,不影响主服务启动。 - 权重:约 2.3 GB,持久缓存 `C:\Users\ji\.cache\deerflow\embeddings`,不进入业务仓库。 - 配置:`llmwiki.local_wiki_index.embedding.provider: local`;可用 `local_model_path` 指定离线权重目录,详见后端 README。 @@ -101,15 +101,15 @@ WeKnora;没有把业务代码发布到服务器,没有提交 Git,也没有 多来源事实冲突合并验收,不能视为这一历史需求已全面完成。 4. 重复上传仍会保留独立来源记录。向量导出已消除旧版本重复,但上传身份幂等、部分远端 写入失败后的无损重试、跨实例图片资源打包仍需独立验证。 -5. 自动沉淀挂在本系统成功向量化完成后;未新增文件监听或定时远程写入。直接在外部 - WeKnora 修改的内容仍需通过本系统同步/向量化入口接入。 +5. 当前版本已补充自动增量链路:服务启动立即补扫,之后默认每 30 秒扫描全部普通 + 知识库;Wiki 抽取完成后无需手工点击向量化,并自动以来源快照覆盖同步到知识梳理总库。 6. 回流当前只允许新建空 Wiki 普通库,避免覆盖已有页面。普通库现有页面列表的超大规模 分页与助手问答的最终生成回答不属于这轮通过结论。 ## 自动化检查 -- 后端相关测试:46 passed(assistant repository、Wiki sync、package roundtrip、技能扫描、 - 技能关系仓储、知识库管理)。 +- 后端相关测试:118 passed(assistant repository、Wiki sync、package roundtrip、 + WeKnora 接口、向量检索与知识库管理)。 - 修改的 Python 功能文件 Ruff 检查通过;app.py 既有启动顺序导致的 E402 不在此次改动中处理。 - `git diff --check` 通过(只有仓库换行提示)。 - 前端完整 `pnpm typecheck` 未通过:57 个既有/本轮范围外错误,涉及 nextra 缺失、Markdown diff --git a/frontend-web/src/components/page-sidebar.tsx b/frontend-web/src/components/page-sidebar.tsx index dd85aeeb..f57adc65 100644 --- a/frontend-web/src/components/page-sidebar.tsx +++ b/frontend-web/src/components/page-sidebar.tsx @@ -56,7 +56,7 @@ type PageSidebarProps = { headerHidden?: boolean; }; -type SettingsSection = "account" | "appearance" | "memory" | "tools" | "about"; +type SettingsSection = "account" | "appearance" | "memory" | "tools" | "encoder" | "about"; const hasActiveChild = (item: MenuItem, pathname: string): boolean => menuItemContainsPath(item, pathname); @@ -270,6 +270,12 @@ function SecondaryNavSidebar({ headerHidden }: { headerHidden: boolean }) { 模型配置 )} + {isAdmin && ( + openSettings("encoder")}> + + 本地编码器 + + )} {isAdmin && ( navigate("/page/strategy/settings/model-concurrency")} @@ -585,6 +591,12 @@ function LegacyAccordionSidebar({ 模型配置 )} + {isAdmin && ( + openSettings("encoder")}> + + 本地编码器 + + )} {isAdmin && ( navigate("/page/strategy/admin/login-whitelist")}> diff --git a/frontend-web/src/components/workspace/settings/settings-dialog.tsx b/frontend-web/src/components/workspace/settings/settings-dialog.tsx index 00f587dd..e104cc64 100644 --- a/frontend-web/src/components/workspace/settings/settings-dialog.tsx +++ b/frontend-web/src/components/workspace/settings/settings-dialog.tsx @@ -1,6 +1,6 @@ "use client"; -import { BellIcon, PaletteIcon, SparklesIcon, UserIcon, WrenchIcon } from "lucide-react"; +import { BellIcon, CpuIcon, PaletteIcon, SparklesIcon, UserIcon, WrenchIcon } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { @@ -14,16 +14,19 @@ import { AboutSettingsPage } from "@/components/workspace/settings/about-setting import { AccountSettingsPage } from "@/components/workspace/settings/account-settings-page"; import { AppearanceSettingsPage } from "@/components/workspace/settings/appearance-settings-page"; import { MemorySettingsPage } from "@/components/workspace/settings/memory-settings-page"; -import { ToolSettingsPage } from "@/components/workspace/settings/tool-settings-page"; -import { useI18n } from "@/core/i18n/hooks"; -import { cn } from "@/lib/utils"; +import { ToolSettingsPage } from "@/components/workspace/settings/tool-settings-page"; +import { getStoredAuth } from "@/core/auth"; +import { useI18n } from "@/core/i18n/hooks"; +import { cn } from "@/lib/utils"; +import { LocalEncoderSettingsCard } from "@/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeWorkspace"; type SettingsSection = | "account" | "appearance" - | "memory" - | "tools" - | "about"; + | "memory" + | "tools" + | "encoder" + | "about"; type SettingsDialogProps = React.ComponentProps & { defaultSection?: SettingsSection; @@ -31,7 +34,8 @@ type SettingsDialogProps = React.ComponentProps & { export function SettingsDialog(props: SettingsDialogProps) { const { defaultSection = "appearance", ...dialogProps } = props; - const { t } = useI18n(); + const { t } = useI18n(); + const isAdmin = getStoredAuth()?.system_role === "admin"; const [activeSection, setActiveSection] = useState(defaultSection); @@ -63,7 +67,10 @@ export function SettingsDialog(props: SettingsDialogProps) { // label: t.settings.sections.memory, // icon: BrainIcon, // }, - { id: "tools", label: t.settings.sections.tools, icon: WrenchIcon }, + { id: "tools", label: t.settings.sections.tools, icon: WrenchIcon }, + ...(isAdmin + ? [{ id: "encoder", label: "本地编码器", icon: CpuIcon }] + : []), { id: "account", label: "个人信息", @@ -75,7 +82,8 @@ export function SettingsDialog(props: SettingsDialogProps) { t.settings.sections.appearance, t.settings.sections.memory, t.settings.sections.tools, - t.settings.sections.about, + t.settings.sections.about, + isAdmin, ], ); return ( @@ -125,7 +133,8 @@ export function SettingsDialog(props: SettingsDialogProps) { props.onOpenChange?.(false)} /> )} {activeSection === "memory" && } - {activeSection === "tools" && } + {activeSection === "tools" && } + {activeSection === "encoder" && isAdmin && } {activeSection === "about" && } diff --git a/frontend-web/src/components/workspace/settings/skill-settings-page.tsx b/frontend-web/src/components/workspace/settings/skill-settings-page.tsx index 9e5685a6..89f53cd0 100644 --- a/frontend-web/src/components/workspace/settings/skill-settings-page.tsx +++ b/frontend-web/src/components/workspace/settings/skill-settings-page.tsx @@ -102,9 +102,16 @@ import { cn } from "@/lib/utils"; import { SettingsSection } from "./settings-section"; import { SkillStateSheet } from "./skill-state-sheet"; -export function SkillSettingsPage({ onClose }: { onClose?: () => void } = {}) { - const { t } = useI18n(); - return ( +export function SkillSettingsPage({ + onClose, + embedded = false, +}: { + onClose?: () => void; + embedded?: boolean; +} = {}) { + const { t } = useI18n(); + if (embedded) return ; + return ( = { completed: "完成", }; -export function SkillKnowledgePanel() { - const { skills, isLoading } = useAllSkillsAdmin(true); +const STATUS_LABELS: Record = { + queued: "等待处理", + running: "处理中", + awaiting_review: "等待管理员审核", + completed: "已完成", + partial_failed: "部分失败", + failed: "失败", + canceled: "已取消", +}; + +export function SkillKnowledgePanel({ adminMode = false }: { adminMode?: boolean }) { + const visibleSkills = useSkills(); + const allSkills = useAllSkillsAdmin(adminMode); + const skills = adminMode ? allSkills.skills : visibleSkills.skills; + const isLoading = adminMode ? allSkills.isLoading : visibleSkills.isLoading; const liveSkills = useMemo(() => skills.filter((skill) => (skill.state ?? "active") === "active"), [skills]); const [selectedSkills, setSelectedSkills] = useState([]); const [skillQuery, setSkillQuery] = useState(""); const [selectedTargetKey, setSelectedTargetKey] = useState(null); - const [reviewMode, setReviewMode] = useState<"auto_high_confidence" | "off">("off"); const runtime = useLlmWikiRuntime(); const knowledgeBases = useLlmWikiKnowledgeBases("all", runtime.data?.provider === "weknora"); - const bindings = useSkillKnowledgeBindings(); - const jobs = useSkillKnowledgeJobs(); + const bindings = useSkillKnowledgeBindings(undefined, !adminMode); + const jobs = useSkillKnowledgeJobs(!adminMode); const actions = useSkillKnowledgeActions(); const targets = useMemo(() => { @@ -122,7 +133,7 @@ export function SkillKnowledgePanel() { const job = await actions.createJob.mutateAsync({ skill_names: selectedSkills, targets: selectedTargets, - review_mode: reviewMode, + review_mode: "off", }); toast.success(`已创建 ${job.total_count} 个归纳项`); setSelectedSkills([]); @@ -132,17 +143,17 @@ export function SkillKnowledgePanel() { }; return ( -
+
-

技能到知识库管理

-

一次选择多个技能归纳到一个 WeKnora Wiki 知识库;代码与脚本不会被读取。

+

归纳技能知识

+

选择技能和一个目标知识库后,系统会在后台提取 Markdown 等知识资料,忽略 Python 等代码文件,自动生成 Wiki、关系和向量索引。

- 1. 选择技能({selectedSkills.length}) + 第一步:选择技能(已选 {selectedSkills.length}) @@ -162,6 +173,13 @@ export function SkillKnowledgePanel() { toggle(selectedSkills, skill.name, setSelectedSkills)} /> {skill.name_zh || skill.name} {skill.name} + {(bindings.data ?? []).some((binding) => binding.skill_name === skill.name) && ( + + {(bindings.data ?? []).some((binding) => binding.skill_name === skill.name && `${binding.target_type}:${binding.target_id}` === selectedTargetKey) + ? "已在目标库" + : `已归纳 ${(bindings.data ?? []).filter((binding) => binding.skill_name === skill.name).length} 库`} + + )} ))} {!isLoading && !filteredSkills.length && ( @@ -174,7 +192,7 @@ export function SkillKnowledgePanel() {
- 2. 选择目标知识库({selectedTargetKey ? 1 : 0}) + 第二步:选择一个目标知识库
{targets.map((target) => { @@ -207,19 +225,12 @@ export function SkillKnowledgePanel() { )}
-
- 审核策略 - +
+ 第三步:提交后可离开本页。归纳、Wiki 生成、向量化及同步到“知识梳理”总库均在后台完成。
-
@@ -228,22 +239,26 @@ export function SkillKnowledgePanel() {
-

归纳任务

后台执行、失败可单项重试

+

我的归纳任务

任务会自动刷新,失败项可直接重试

{(jobs.data ?? []).map((job) => { - const done = job.success_count + job.partial_failed_count + job.failed_count; + const countedDone = job.success_count + job.partial_failed_count + job.failed_count; + const itemDone = (job.items ?? []).filter((item) => + ["completed", "partial_failed", "failed", "canceled"].includes(item.status), + ).length; + const done = Math.max(countedDone, itemDone); const progress = job.total_count ? Math.round((done / job.total_count) * 100) : 0; return (
-
{job.status}{done}/{job.total_count} 项{new Date(job.created_at).toLocaleString()}
+
{STATUS_LABELS[job.status] || job.status}{done}/{job.total_count} 项{new Date(job.created_at).toLocaleString()}
{job.items?.map((item) => (
{item.skill_name} - {PHASE_LABELS[item.phase] || item.phase} + {PHASE_LABELS[item.phase] || STATUS_LABELS[item.status] || item.phase} {item.progress}% {item.status === "failed" && }
@@ -256,8 +271,8 @@ export function SkillKnowledgePanel() {
-
当前有效绑定:{bindings.data?.length ?? 0};同名同类型实体按来源贡献合并。
-
{bindings.data?.map((binding) =>
{binding.skill_name}→ {binding.target_name}{binding.status}{binding.remote_business_key}
)}
+
已归纳关联:{bindings.data?.length ?? 0}
+
{bindings.data?.map((binding) =>
{binding.skill_name}已归纳到 {binding.target_name}{STATUS_LABELS[binding.status] || binding.status}{adminMode && }
)}
diff --git a/frontend-web/src/components/workspace/workspace-nav-menu.tsx b/frontend-web/src/components/workspace/workspace-nav-menu.tsx index d3f48f8e..57c71201 100644 --- a/frontend-web/src/components/workspace/workspace-nav-menu.tsx +++ b/frontend-web/src/components/workspace/workspace-nav-menu.tsx @@ -83,7 +83,7 @@ export function WorkspaceNavMenu() { const isAdmin = getStoredAuth()?.system_role === "admin"; const [settingsOpen, setSettingsOpen] = useState(false); const [settingsDefaultSection, setSettingsDefaultSection] = useState< - "appearance" | "memory" | "tools" | "about" + "appearance" | "memory" | "tools" | "encoder" | "about" >("appearance"); const [notificationOpen, setNotificationOpen] = useState(false); const [mounted, setMounted] = useState(false); @@ -189,6 +189,17 @@ export function WorkspaceNavMenu() { {t.common.settings} + {isAdmin && ( + { + setSettingsDefaultSection("encoder"); + setSettingsOpen(true); + }} + > + + 本地编码器 + + )} {isAdmin && ( @@ -254,14 +265,6 @@ export function WorkspaceNavMenu() { 技能管理 )} - {isAdmin && ( - navigate("/page/workspace/admin/skills?tab=knowledge")} - > - - 技能到知识库管理 - - )} {isAdmin && ( navigate("/page/workspace/admin/models")} diff --git a/frontend-web/src/pages/AdminSkillsPage.tsx b/frontend-web/src/pages/AdminSkillsPage.tsx index 6ac3ffc4..e69fad27 100644 --- a/frontend-web/src/pages/AdminSkillsPage.tsx +++ b/frontend-web/src/pages/AdminSkillsPage.tsx @@ -3,7 +3,6 @@ import { ChevronLeftIcon, ChevronRightIcon, - DatabaseZap, Replace, SearchIcon, SparklesIcon, @@ -20,7 +19,6 @@ import { WorkspaceHeader, } from "@/components/workspace/workspace-container"; import { SkillRow } from "@/components/workspace/settings/skill-settings-page"; -import { SkillKnowledgePanel } from "@/components/workspace/skills/SkillKnowledgePanel"; import { getStoredAuth } from "@/core/auth"; import { useAllSkillsAdmin } from "@/core/skills/hooks"; import { desensitize } from "@/lib/desensitize"; @@ -37,7 +35,7 @@ export default function AdminSkillsPage() { const knowledgeBindings = useSkillKnowledgeBindings(); const activeTab = useMemo(() => { const tab = searchParams.get("tab"); - return tab === "addresses" || tab === "knowledge" ? tab : "skills"; + return tab === "addresses" ? tab : "skills"; }, [searchParams]); const changeTab = (value: string) => { @@ -89,10 +87,6 @@ export default function AdminSkillsPage() { 技能地址管理 - - - 技能到知识库管理 -
@@ -189,12 +183,6 @@ export default function AdminSkillsPage() { > - - - diff --git a/frontend-web/src/pages/AssistantKnowledgePage.tsx b/frontend-web/src/pages/AssistantKnowledgePage.tsx index 55a6d2ea..c60de7ca 100644 --- a/frontend-web/src/pages/AssistantKnowledgePage.tsx +++ b/frontend-web/src/pages/AssistantKnowledgePage.tsx @@ -2,10 +2,12 @@ import { ArrowLeft, BookOpenText, Database, + Edit3, Loader2, Plus, RefreshCw, Search, + Trash2, Upload, } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; @@ -17,12 +19,14 @@ import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, + DialogDescription, + DialogFooter, DialogHeader, DialogTitle, - DialogFooter, } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Textarea } from "@/components/ui/textarea"; import { KnowledgeProductNav } from "@/components/workspace/knowledge-product-nav"; import { getStoredAuth } from "@/core/auth"; import { buildAppHashUrl } from "@/core/llmwiki/source-links"; @@ -33,6 +37,7 @@ import { useAssistantKnowledgeBases, useAssistantKnowledgeSources, useAssistantWikiPages, + type AssistantKnowledgeBase, } from "@/strategy-components/api/assistant-knowledge"; import { WikiLibraryReader, @@ -53,6 +58,8 @@ export default function AssistantKnowledgePage() { const client = useQueryClient(); const [tab, setTab] = useState("wiki"); const [createOpen, setCreateOpen] = useState(false); + const [editingBase, setEditingBase] = + useState(null); const [name, setName] = useState(""); const [query, setQuery] = useState(""); const [searching, setSearching] = useState(false); @@ -124,6 +131,25 @@ export default function AssistantKnowledgePage() { }; const refresh = () => void client.invalidateQueries({ queryKey: ["assistant-knowledge"] }); + const removeBase = async (target: AssistantKnowledgeBase) => { + if (target.is_global) { + toast.error("知识梳理总库承担全库沉淀,不能删除"); + return; + } + if ( + !window.confirm( + `确定删除助手知识库“${target.name}”吗?库内 Wiki、向量、实体、关系和导入记录都会删除。`, + ) + ) + return; + try { + await actions.deleteBase.mutateAsync(target.id); + if (baseId === target.id) setParams({}); + toast.success("助手知识库已删除"); + } catch (error) { + toast.error((error as Error).message); + } + }; return (
@@ -150,34 +176,69 @@ export default function AssistantKnowledgePage() { )}
{bases.data?.map((b) => ( -
-

- {b.name} -

-

- {b.is_global - ? "汇聚普通知识库的分析成果,保留分类、标签和知识关联。" - : b.description || - "独立的 Wiki 知识空间,支持导入、导出与向量检索。"} -

-
- {b.page_count || 0} Wiki - {b.source_count || 0} 来源 - {b.relation_count || 0} 关系 -
- +

+ {b.name} +

+

+ {b.description || + (b.is_global + ? "汇聚普通知识库的分析成果,保留分类、标签和知识关联。" + : "独立的 Wiki 知识空间,支持导入、导出与向量检索。")} +

+
+ {b.page_count || 0} Wiki + {b.source_count || 0} 来源 + {b.relation_count || 0} 关系 +
+ + {isAdmin && ( +
+ + {!b.is_global && ( + + )} + {b.is_global && ( + + 系统大库不可删除 + + )} +
+ )} +
))}
{!bases.isLoading && !bases.data?.length && isAdmin && ( @@ -222,6 +283,29 @@ export default function AssistantKnowledgePage() { 刷新 + {isAdmin && ( + + )} + {isAdmin && base && !base.is_global && ( + + )} {isAdmin && ( ))} @@ -425,6 +521,111 @@ export default function AssistantKnowledgePage() { + {editingBase && ( + setEditingBase(null)} + onSave={async (input) => { + try { + await actions.updateBase.mutateAsync({ + id: editingBase.id, + ...input, + }); + setEditingBase(null); + toast.success("助手知识库信息已保存"); + } catch (error) { + toast.error((error as Error).message); + } + }} + /> + )} ); } + +function EditAssistantBaseDialog({ + base, + pending, + onClose, + onSave, +}: { + base: AssistantKnowledgeBase; + pending: boolean; + onClose: () => void; + onSave: (input: { name: string; description: string }) => Promise; +}) { + const [name, setName] = useState(base.name); + const [description, setDescription] = useState(base.description || ""); + + return ( + { + if (!open && !pending) onClose(); + }} + > + +
{ + event.preventDefault(); + if (!pending && name.trim()) { + void onSave({ + name: name.trim(), + description: description.trim(), + }); + } + }} + > + + 编辑助手知识库 + + 修改知识库名称和描述。知识内容、向量和关联关系不会受到影响。 + + +
+
+ + setName(event.target.value)} + maxLength={255} + disabled={pending} + required + /> +
+
+ +