From 378534535349fc6444c504126df67fe1c2975b54 Mon Sep 17 00:00:00 2001 From: Administrator Date: Mon, 24 Aug 2026 00:49:31 +0800 Subject: [PATCH] feat: add local LLMWiki wiki vector index --- .gitignore | 6 + .../workspace/skill-display-sources.tsx | 7 +- frontend-web/src/core/llmwiki/source-links.ts | 3 +- .../src/strategy-components/api/llmwiki.ts | 110 ++++++- .../llmwiki/WeKnoraSourceDrawer.tsx | 15 +- .../llmwiki/WeKnoraWikiManagementPage.tsx | 51 ++- offline-backend-20260512/backend/CLAUDE.md | 49 +++ offline-backend-20260512/backend/README.md | 35 +++ .../backend/app/gateway/app.py | 41 ++- .../backend/app/gateway/auth_middleware.py | 32 +- .../backend/app/gateway/deps.py | 67 +++- .../app/gateway/llmwiki_index_scheduler.py | 53 ++++ .../backend/app/gateway/llmwiki_rag.py | 62 ++-- .../app/gateway/routers/external_llmwiki.py | 133 ++++++++ .../backend/app/gateway/routers/llmwiki.py | 242 ++++++++++++-- .../app/gateway/routers/llmwiki_index.py | 290 +++++++++++++++++ offline-backend-20260512/backend/config.yaml | 39 +++ .../LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md | 3 +- .../harness/deerflow/config/llmwiki_config.py | 109 ++++++- .../weknora/local_index/__init__.py | 7 + .../weknora/local_index/chunking.py | 108 +++++++ .../weknora/local_index/embedding.py | 107 +++++++ .../weknora/local_index/normalize.py | 86 +++++ .../weknora/local_index/retrieval.py | 248 +++++++++++++++ .../weknora/local_index/runtime.py | 48 +++ .../weknora/local_index/search.py | 176 +++++++++++ .../integrations/weknora/local_index/sync.py | 173 ++++++++++ .../weknora/local_index/vector_cache.py | 86 +++++ .../weknora/local_index/vector_codec.py | 33 ++ .../deerflow/persistence/llmwiki/base.py | 10 +- .../deerflow/persistence/llmwiki/memory.py | 24 +- .../deerflow/persistence/llmwiki/model.py | 5 +- .../deerflow/persistence/llmwiki/sql.py | 17 +- .../persistence/llmwiki_index/__init__.py | 21 ++ .../persistence/llmwiki_index/base.py | 54 ++++ .../persistence/llmwiki_index/memory.py | 160 ++++++++++ .../persistence/llmwiki_index/model.py | 94 ++++++ .../deerflow/persistence/llmwiki_index/sql.py | 295 ++++++++++++++++++ .../20260823_01_llmwiki_local_wiki_index.py | 143 +++++++++ .../deerflow/persistence/models/__init__.py | 4 + .../tools/builtins/llmwiki_search_tool.py | 74 ++--- .../backend/packages/harness/pyproject.toml | 3 +- .../backend/pyproject.toml | 1 + .../tests/test_external_llmwiki_search.py | 51 +++ .../test_llmwiki_local_index_repository.py | 68 ++++ .../tests/test_llmwiki_vector_search.py | 89 ++++++ .../tests/test_llmwiki_wiki_retrieval.py | 154 +++++++++ .../backend/tests/test_llmwiki_wiki_sync.py | 116 +++++++ 48 files changed, 3650 insertions(+), 152 deletions(-) create mode 100644 offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py create mode 100644 offline-backend-20260512/backend/app/gateway/routers/external_llmwiki.py create mode 100644 offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/__init__.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/chunking.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/normalize.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/retrieval.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/runtime.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/sync.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_cache.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_codec.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/base.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/memory.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/model.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260823_01_llmwiki_local_wiki_index.py create mode 100644 offline-backend-20260512/backend/tests/test_external_llmwiki_search.py create mode 100644 offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py create mode 100644 offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py create mode 100644 offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py create mode 100644 offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py diff --git a/.gitignore b/.gitignore index 32e39254..09774bbe 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,12 @@ offline-backend-20260512/backend/tests/* !offline-backend-20260512/backend/tests/test_model_management.py # Keep fixed-question exact-match and streaming regression coverage versioned. !offline-backend-20260512/backend/tests/test_fixed_questions.py +# Keep the local LLMWiki vector-index and external API regression coverage versioned. +!offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py +!offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py +!offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py +!offline-backend-20260512/backend/tests/test_external_llmwiki_search.py +!offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py deploy .history diff --git a/frontend-web/src/components/workspace/skill-display-sources.tsx b/frontend-web/src/components/workspace/skill-display-sources.tsx index b3db288a..6828ee22 100644 --- a/frontend-web/src/components/workspace/skill-display-sources.tsx +++ b/frontend-web/src/components/workspace/skill-display-sources.tsx @@ -73,6 +73,9 @@ export function SourceCard({ : null; const meta = [ source.source, + previewTarget?.wikiSlug && source.raw?.matched_sections + ? `Wiki 章节 ${String((source.raw.matched_sections as Array<{ heading?: string }>)[0]?.heading || "")}` + : null, source.author ? `作者 ${source.author}` : null, source.type && source.type !== "result" ? source.type : null, source.time, @@ -132,7 +135,7 @@ export function SourceCard({ previewTarget ? ( - 知识库文档 + {previewTarget.wikiSlug ? "Wiki 页面" : "知识库文档"} ) : ( @@ -151,7 +154,7 @@ export function SourceCard({ }} > - 查看详情 + {previewTarget.wikiSlug ? "查看 Wiki" : "查看详情"} )} diff --git a/frontend-web/src/core/llmwiki/source-links.ts b/frontend-web/src/core/llmwiki/source-links.ts index d91c0845..43d776d1 100644 --- a/frontend-web/src/core/llmwiki/source-links.ts +++ b/frontend-web/src/core/llmwiki/source-links.ts @@ -44,7 +44,7 @@ export function parseWeKnoraPreviewUrl(url?: string | null): WeKnoraSourceTarget params.get("wikiSlug")?.trim() || params.get("slug")?.trim() || ""; - return knowledgeBaseId && chunkId + return knowledgeBaseId && (wikiSlug || chunkId || documentId) ? { knowledgeBaseId, chunkId, @@ -75,6 +75,7 @@ export function resolveWeKnoraSourceTarget(source: KnowledgeSourceLike): WeKnora fromPreview?.documentId || ""; const chunkId = source.chunk_id || source.chunkId || fromPreview?.chunkId || ""; + if (!wikiSlug && !chunkId && !documentId) return null; return { knowledgeBaseId, chunkId: chunkId || undefined, diff --git a/frontend-web/src/strategy-components/api/llmwiki.ts b/frontend-web/src/strategy-components/api/llmwiki.ts index 9f899052..389e3055 100644 --- a/frontend-web/src/strategy-components/api/llmwiki.ts +++ b/frontend-web/src/strategy-components/api/llmwiki.ts @@ -143,6 +143,22 @@ export interface LlmWikiWikiPageList { total_pages: number; } +export interface LlmWikiWikiIndexStatus { + knowledge_base_id: string; + state: "not_synced" | "idle" | "running" | "partial" | "failed" | "rebuilding" | "rebuild_required"; + local_index_enabled: boolean; + wiki_index_enabled: boolean; + fully_vectorized: boolean; + retrieval_mode: "vector" | "wiki_api"; + local_page_count?: number; + ready_page_count?: number; + failed_page_count?: number; + vector_count?: number; + last_error?: string | null; + last_started_at?: string | null; + last_completed_at?: string | null; +} + export interface LlmWikiSourceContext { knowledge_base_name?: string; document_title?: string; @@ -206,9 +222,13 @@ export interface LlmWikiWikiPageUpdate { async function readJson(response: Response, context: string): Promise { if (!response.ok) { const payload = await response.json().catch(() => null as unknown); - const detail = - payload && typeof payload === "object" && "detail" in payload - ? String((payload as { detail: unknown }).detail) + const rawDetail = payload && typeof payload === "object" && "detail" in payload + ? (payload as { detail: unknown }).detail + : null; + const detail = rawDetail && typeof rawDetail === "object" && "message" in rawDetail + ? String((rawDetail as { message: unknown }).message) + : rawDetail + ? String(rawDetail) : `${context}失败（${response.status}）`; throw new Error(detail); } @@ -412,6 +432,66 @@ export async function getLlmWikiWikiPage( ); } +export async function getLlmWikiWikiIndexStatus( + knowledgeBaseId: string, +): Promise { + return readJson( + await apiFetch( + `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki-index-status`, + ), + "读取 Wiki 向量化状态", + ); +} + +export async function vectorizeLlmWikiKnowledgeBase( + knowledgeBaseId: string, +): Promise<{ job_id: string; status: string; knowledge_base_id: string }> { + return readJson( + await apiFetch( + `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/vectorize`, + { method: "POST" }, + ), + "触发 Wiki 向量化", + ); +} + +export async function exportLlmWikiWikiExcel( + knowledgeBaseId: string, + knowledgeBaseName = "LLMWiki", +): Promise { + const response = await apiFetch( + `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki/export.xlsx`, + ); + if (!response.ok) { + await readJson(response, "导出 Wiki Excel"); + return; + } + const blob = await response.blob(); + const objectUrl = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = objectUrl; + link.download = `${knowledgeBaseName.replace(/[\\/:*?"<>|]/g, "_") || "LLMWiki"}-Wiki.xlsx`; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(objectUrl); +} + +/** Read the DeerFlow-local Wiki mirror used by new vector citations. */ +export async function getLocalLlmWikiWikiPage( + knowledgeBaseId: string, + slug: string, +): Promise { + const safeSlug = slug.split("/").map(encodeURIComponent).join("/"); + const page = await readJson( + await apiFetch( + `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/local-wiki/pages/${safeSlug}`, + ), + "读取本地 Wiki 页面详情", + ); + return { ...page, version: page.version ?? page.page_version ?? 0 }; +} + export async function createLlmWikiWikiPage( knowledgeBaseId: string, input: LlmWikiWikiPageCreate, @@ -755,6 +835,22 @@ export function useLlmWikiWikiPage( }); } +export function useLlmWikiWikiIndexStatus( + knowledgeBaseId: string | null, + enabled = true, +) { + const { userId } = useLiveAccount(); + return useQuery({ + queryKey: ["llmwiki", "wiki-index-status", knowledgeBaseId, userId ?? "anonymous"], + queryFn: () => getLlmWikiWikiIndexStatus(knowledgeBaseId!), + enabled: enabled && !!knowledgeBaseId, + refetchInterval: (query) => { + const state = query.state.data?.state; + return state === "running" || state === "rebuilding" ? 2_000 : 10_000; + }, + }); +} + export function useLlmWikiSettings(enabled = true) { const { userId } = useLiveAccount(); return useQuery({ @@ -822,6 +918,14 @@ export function useLlmWikiActions() { deleteLlmWikiWikiPage(id, slug), onSuccess: invalidateKnowledge, }), + vectorizeWiki: useMutation({ + mutationFn: ({ id }: { id: string }) => vectorizeLlmWikiKnowledgeBase(id), + onSuccess: invalidateKnowledge, + }), + exportWikiExcel: useMutation({ + mutationFn: ({ id, name }: { id: string; name?: string }) => + exportLlmWikiWikiExcel(id, name), + }), publication: useMutation({ mutationFn: ({ id, publish }: { id: string; publish: boolean }) => setLlmWikiPublication(id, publish), diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx index b88ee2d6..937935af 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx @@ -20,7 +20,7 @@ import { reasoningPlugins } from "@/core/streamdown"; import { getLlmWikiDocument, getLlmWikiDocumentPreview, - getLlmWikiWikiPage, + getLocalLlmWikiWikiPage, listLlmWikiDocumentChunks, type LlmWikiChunk, type LlmWikiDocument, @@ -528,12 +528,17 @@ export function WeKnoraSourceDrawer({ if (detailTarget.wikiSlug) { tasks.push( - getLlmWikiWikiPage(detailTarget.knowledgeBaseId, detailTarget.wikiSlug) + getLocalLlmWikiWikiPage(detailTarget.knowledgeBaseId, detailTarget.wikiSlug) .then((page) => { loadedWikiPage = page; if (!cancelled) setWikiPage(page); }) - .catch(() => undefined), + .catch((error) => { + // New Wiki citations have no legacy document/chunk fallback, so + // surface mirror/index failures instead of rendering an empty drawer. + if (!detailTarget.documentId) throw error; + return undefined; + }), ); } @@ -649,7 +654,7 @@ export function WeKnoraSourceDrawer({ )} > {resolvedTitle} - 知识库文档详情 + 知识库 Wiki 页面详情
@@ -690,7 +695,7 @@ export function WeKnoraSourceDrawer({ {loading && (
- 正在加载完整文档预览 + 正在加载 Wiki 页面
)} diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx index 743d6b2a..3eddeb72 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx @@ -1,4 +1,4 @@ -import { ArrowLeft, ChevronDown, ChevronRight, FilePlus2, FileText, Folder, Loader2, RefreshCw, Save, Search, Trash2 } from "lucide-react"; +import { ArrowLeft, ChevronDown, ChevronRight, Cpu, Download, FilePlus2, FileText, Folder, Loader2, RefreshCw, Save, Search, Trash2 } from "lucide-react"; import { type Dispatch, type SetStateAction, useEffect, useMemo, useState } from "react"; import { Navigate, useNavigate, useParams } from "react-router-dom"; import { toast } from "sonner"; @@ -13,6 +13,7 @@ import { useLlmWikiActions, useLlmWikiKnowledgeBase, useLlmWikiRuntime, + useLlmWikiWikiIndexStatus, useLlmWikiWikiPage, useLlmWikiWikiPages, } from "@/strategy-components/api/llmwiki"; @@ -298,6 +299,7 @@ export function WeKnoraWikiManagementPage() { const canWrite = !!details.data?.can_write && !details.data?.is_conversation_deposit; const pages = useLlmWikiWikiPages(noteId || null, 1, 500, "", !!noteId); + const indexStatus = useLlmWikiWikiIndexStatus(noteId || null, !!noteId); const detail = useLlmWikiWikiPage( noteId || null, selectedSlug || null, @@ -561,6 +563,34 @@ export function WeKnoraWikiManagementPage() { } }; + const vectorize = async () => { + if (!canWrite) return; + try { + await actions.vectorizeWiki.mutateAsync({ id: noteId }); + toast.success("已提交整库 Wiki 向量化任务"); + await indexStatus.refetch(); + } catch (error) { + toast.error(error instanceof Error ? error.message : "触发向量化失败"); + } + }; + + const exportExcel = async () => { + try { + await actions.exportWikiExcel.mutateAsync({ id: noteId, name: title }); + toast.success("Wiki 已全部导出为 Excel"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "导出 Wiki Excel 失败"); + } + }; + + const vectorStatusText = !indexStatus.data?.local_index_enabled + ? "本地向量未启用，检索使用 Wiki 接口" + : indexStatus.data.fully_vectorized + ? `整库已向量化（${indexStatus.data.vector_count ?? 0} 个向量章节）` + : indexStatus.data.state === "running" || indexStatus.data.state === "rebuilding" + ? "整库向量化进行中，完成前检索使用 Wiki 接口" + : "整库尚未完成向量化，检索使用 Wiki 接口"; + return (
@@ -571,9 +601,26 @@ export function WeKnoraWikiManagementPage() {

{title}

-

Wiki 文档新增、编辑、删除

+

Wiki 文档新增、编辑、删除 · {vectorStatusText}

+ +