feat: add local LLMWiki wiki vector index

.gitignore

@@ -95,6 +95,12 @@ offline-backend-20260512/backend/tests/*
 !offline-backend-20260512/backend/tests/test_model_management.py
 # Keep fixed-question exact-match and streaming regression coverage versioned.
 !offline-backend-20260512/backend/tests/test_fixed_questions.py
+# Keep the local LLMWiki vector-index and external API regression coverage versioned.
+!offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py
+!offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py
+!offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py
+!offline-backend-20260512/backend/tests/test_external_llmwiki_search.py
+!offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py
 deploy
 .history
 

frontend-web/src/components/workspace/skill-display-sources.tsx

@@ -73,6 +73,9 @@ export function SourceCard({
       : null;
   const meta = [
     source.source,
+    previewTarget?.wikiSlug && source.raw?.matched_sections
+      ? `Wiki 章节 ${String((source.raw.matched_sections as Array<{ heading?: string }>)[0]?.heading || "")}`
+      : null,
     source.author ? `作者 ${source.author}` : null,
     source.type && source.type !== "result" ? source.type : null,
     source.time,
@@ -132,7 +135,7 @@ export function SourceCard({
             previewTarget ? (
               <span className="text-muted-foreground bg-muted mt-0.5 inline-flex shrink-0 items-center gap-1 rounded-md px-1.5 py-1 text-[11px] font-medium">
                 <FileTextIcon className="size-3.5" />
-                知识库文档
+                {previewTarget.wikiSlug ? "Wiki 页面" : "知识库文档"}
               </span>
             ) : (
               <ExternalLinkIcon className="text-muted-foreground/60 group-hover:text-primary mt-1 size-3.5 shrink-0 transition-colors" />
@@ -151,7 +154,7 @@ export function SourceCard({
               }}
             >
               <ExternalLinkIcon className="mr-1.5 size-3.5" />
-              查看详情
+              {previewTarget.wikiSlug ? "查看 Wiki" : "查看详情"}
             </Button>
           </div>
         )}

frontend-web/src/core/llmwiki/source-links.ts

@@ -44,7 +44,7 @@ export function parseWeKnoraPreviewUrl(url?: string | null): WeKnoraSourceTarget
     params.get("wikiSlug")?.trim() ||
     params.get("slug")?.trim() ||
     "";
-  return knowledgeBaseId && chunkId
+  return knowledgeBaseId && (wikiSlug || chunkId || documentId)
     ? {
         knowledgeBaseId,
         chunkId,
@@ -75,6 +75,7 @@ export function resolveWeKnoraSourceTarget(source: KnowledgeSourceLike): WeKnora
     fromPreview?.documentId ||
     "";
   const chunkId = source.chunk_id || source.chunkId || fromPreview?.chunkId || "";
+  if (!wikiSlug && !chunkId && !documentId) return null;
   return {
     knowledgeBaseId,
     chunkId: chunkId || undefined,

frontend-web/src/strategy-components/api/llmwiki.ts

@@ -143,6 +143,22 @@ export interface LlmWikiWikiPageList {
   total_pages: number;
 }
 
+export interface LlmWikiWikiIndexStatus {
+  knowledge_base_id: string;
+  state: "not_synced" | "idle" | "running" | "partial" | "failed" | "rebuilding" | "rebuild_required";
+  local_index_enabled: boolean;
+  wiki_index_enabled: boolean;
+  fully_vectorized: boolean;
+  retrieval_mode: "vector" | "wiki_api";
+  local_page_count?: number;
+  ready_page_count?: number;
+  failed_page_count?: number;
+  vector_count?: number;
+  last_error?: string | null;
+  last_started_at?: string | null;
+  last_completed_at?: string | null;
+}
+
 export interface LlmWikiSourceContext {
   knowledge_base_name?: string;
   document_title?: string;
@@ -206,9 +222,13 @@ export interface LlmWikiWikiPageUpdate {
 async function readJson<T>(response: Response, context: string): Promise<T> {
   if (!response.ok) {
     const payload = await response.json().catch(() => null as unknown);
-    const detail =
-      payload && typeof payload === "object" && "detail" in payload
-        ? String((payload as { detail: unknown }).detail)
+    const rawDetail = payload && typeof payload === "object" && "detail" in payload
+      ? (payload as { detail: unknown }).detail
+      : null;
+    const detail = rawDetail && typeof rawDetail === "object" && "message" in rawDetail
+      ? String((rawDetail as { message: unknown }).message)
+      : rawDetail
+        ? String(rawDetail)
         : `${context}失败（${response.status}）`;
     throw new Error(detail);
   }
@@ -412,6 +432,66 @@ export async function getLlmWikiWikiPage(
   );
 }
 
+export async function getLlmWikiWikiIndexStatus(
+  knowledgeBaseId: string,
+): Promise<LlmWikiWikiIndexStatus> {
+  return readJson(
+    await apiFetch(
+      `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki-index-status`,
+    ),
+    "读取 Wiki 向量化状态",
+  );
+}
+
+export async function vectorizeLlmWikiKnowledgeBase(
+  knowledgeBaseId: string,
+): Promise<{ job_id: string; status: string; knowledge_base_id: string }> {
+  return readJson(
+    await apiFetch(
+      `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/vectorize`,
+      { method: "POST" },
+    ),
+    "触发 Wiki 向量化",
+  );
+}
+
+export async function exportLlmWikiWikiExcel(
+  knowledgeBaseId: string,
+  knowledgeBaseName = "LLMWiki",
+): Promise<void> {
+  const response = await apiFetch(
+    `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki/export.xlsx`,
+  );
+  if (!response.ok) {
+    await readJson<void>(response, "导出 Wiki Excel");
+    return;
+  }
+  const blob = await response.blob();
+  const objectUrl = URL.createObjectURL(blob);
+  const link = document.createElement("a");
+  link.href = objectUrl;
+  link.download = `${knowledgeBaseName.replace(/[\\/:*?"<>|]/g, "_") || "LLMWiki"}-Wiki.xlsx`;
+  document.body.appendChild(link);
+  link.click();
+  link.remove();
+  URL.revokeObjectURL(objectUrl);
+}
+
+/** Read the DeerFlow-local Wiki mirror used by new vector citations. */
+export async function getLocalLlmWikiWikiPage(
+  knowledgeBaseId: string,
+  slug: string,
+): Promise<LlmWikiWikiPage> {
+  const safeSlug = slug.split("/").map(encodeURIComponent).join("/");
+  const page = await readJson<LlmWikiWikiPage & { page_version?: number }>(
+    await apiFetch(
+      `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/local-wiki/pages/${safeSlug}`,
+    ),
+    "读取本地 Wiki 页面详情",
+  );
+  return { ...page, version: page.version ?? page.page_version ?? 0 };
+}
+
 export async function createLlmWikiWikiPage(
   knowledgeBaseId: string,
   input: LlmWikiWikiPageCreate,
@@ -755,6 +835,22 @@ export function useLlmWikiWikiPage(
   });
 }
 
+export function useLlmWikiWikiIndexStatus(
+  knowledgeBaseId: string | null,
+  enabled = true,
+) {
+  const { userId } = useLiveAccount();
+  return useQuery({
+    queryKey: ["llmwiki", "wiki-index-status", knowledgeBaseId, userId ?? "anonymous"],
+    queryFn: () => getLlmWikiWikiIndexStatus(knowledgeBaseId!),
+    enabled: enabled && !!knowledgeBaseId,
+    refetchInterval: (query) => {
+      const state = query.state.data?.state;
+      return state === "running" || state === "rebuilding" ? 2_000 : 10_000;
+    },
+  });
+}
+
 export function useLlmWikiSettings(enabled = true) {
   const { userId } = useLiveAccount();
   return useQuery({
@@ -822,6 +918,14 @@ export function useLlmWikiActions() {
         deleteLlmWikiWikiPage(id, slug),
       onSuccess: invalidateKnowledge,
     }),
+    vectorizeWiki: useMutation({
+      mutationFn: ({ id }: { id: string }) => vectorizeLlmWikiKnowledgeBase(id),
+      onSuccess: invalidateKnowledge,
+    }),
+    exportWikiExcel: useMutation({
+      mutationFn: ({ id, name }: { id: string; name?: string }) =>
+        exportLlmWikiWikiExcel(id, name),
+    }),
     publication: useMutation({
       mutationFn: ({ id, publish }: { id: string; publish: boolean }) =>
         setLlmWikiPublication(id, publish),

frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx

@@ -20,7 +20,7 @@ import { reasoningPlugins } from "@/core/streamdown";
 import {
   getLlmWikiDocument,
   getLlmWikiDocumentPreview,
-  getLlmWikiWikiPage,
+  getLocalLlmWikiWikiPage,
   listLlmWikiDocumentChunks,
   type LlmWikiChunk,
   type LlmWikiDocument,
@@ -528,12 +528,17 @@ export function WeKnoraSourceDrawer({
 
       if (detailTarget.wikiSlug) {
         tasks.push(
-          getLlmWikiWikiPage(detailTarget.knowledgeBaseId, detailTarget.wikiSlug)
+          getLocalLlmWikiWikiPage(detailTarget.knowledgeBaseId, detailTarget.wikiSlug)
             .then((page) => {
               loadedWikiPage = page;
               if (!cancelled) setWikiPage(page);
             })
-            .catch(() => undefined),
+            .catch((error) => {
+              // New Wiki citations have no legacy document/chunk fallback, so
+              // surface mirror/index failures instead of rendering an empty drawer.
+              if (!detailTarget.documentId) throw error;
+              return undefined;
+            }),
         );
       }
 
@@ -649,7 +654,7 @@ export function WeKnoraSourceDrawer({
         )}
       >
         <SheetTitle className="sr-only">{resolvedTitle}</SheetTitle>
-        <SheetDescription className="sr-only">知识库文档详情</SheetDescription>
+        <SheetDescription className="sr-only">知识库 Wiki 页面详情</SheetDescription>
         <div className="relative flex h-full min-h-0 flex-col bg-background">
           <div className="shrink-0 border-b px-5 py-4 pr-64">
             <div className="flex min-w-0 items-center gap-3">
@@ -690,7 +695,7 @@ export function WeKnoraSourceDrawer({
           {loading && (
             <div className="flex min-h-0 flex-1 items-center justify-center text-sm text-muted-foreground">
               <Loader2 className="mr-2 size-5 animate-spin" />
-              正在加载完整文档预览
+              正在加载 Wiki 页面
             </div>
           )}
 

frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx

@@ -1,4 +1,4 @@
-import { ArrowLeft, ChevronDown, ChevronRight, FilePlus2, FileText, Folder, Loader2, RefreshCw, Save, Search, Trash2 } from "lucide-react";
+import { ArrowLeft, ChevronDown, ChevronRight, Cpu, Download, FilePlus2, FileText, Folder, Loader2, RefreshCw, Save, Search, Trash2 } from "lucide-react";
 import { type Dispatch, type SetStateAction, useEffect, useMemo, useState } from "react";
 import { Navigate, useNavigate, useParams } from "react-router-dom";
 import { toast } from "sonner";
@@ -13,6 +13,7 @@ import {
   useLlmWikiActions,
   useLlmWikiKnowledgeBase,
   useLlmWikiRuntime,
+  useLlmWikiWikiIndexStatus,
   useLlmWikiWikiPage,
   useLlmWikiWikiPages,
 } from "@/strategy-components/api/llmwiki";
@@ -298,6 +299,7 @@ export function WeKnoraWikiManagementPage() {
 
   const canWrite = !!details.data?.can_write && !details.data?.is_conversation_deposit;
   const pages = useLlmWikiWikiPages(noteId || null, 1, 500, "", !!noteId);
+  const indexStatus = useLlmWikiWikiIndexStatus(noteId || null, !!noteId);
   const detail = useLlmWikiWikiPage(
     noteId || null,
     selectedSlug || null,
@@ -561,6 +563,34 @@ export function WeKnoraWikiManagementPage() {
     }
   };
 
+  const vectorize = async () => {
+    if (!canWrite) return;
+    try {
+      await actions.vectorizeWiki.mutateAsync({ id: noteId });
+      toast.success("已提交整库 Wiki 向量化任务");
+      await indexStatus.refetch();
+    } catch (error) {
+      toast.error(error instanceof Error ? error.message : "触发向量化失败");
+    }
+  };
+
+  const exportExcel = async () => {
+    try {
+      await actions.exportWikiExcel.mutateAsync({ id: noteId, name: title });
+      toast.success("Wiki 已全部导出为 Excel");
+    } catch (error) {
+      toast.error(error instanceof Error ? error.message : "导出 Wiki Excel 失败");
+    }
+  };
+
+  const vectorStatusText = !indexStatus.data?.local_index_enabled
+    ? "本地向量未启用，检索使用 Wiki 接口"
+    : indexStatus.data.fully_vectorized
+      ? `整库已向量化（${indexStatus.data.vector_count ?? 0} 个向量章节）`
+      : indexStatus.data.state === "running" || indexStatus.data.state === "rebuilding"
+        ? "整库向量化进行中，完成前检索使用 Wiki 接口"
+        : "整库尚未完成向量化，检索使用 Wiki 接口";
+
   return (
     <div className="flex h-full min-h-0 flex-col bg-background">
       <header className="flex min-h-14 shrink-0 flex-wrap items-center justify-between gap-3 border-b px-4 py-2">
@@ -571,9 +601,26 @@ export function WeKnoraWikiManagementPage() {
             </Button>
             <h1 className="truncate text-base font-semibold text-foreground">{title}</h1>
           </div>
-          <p className="ml-9 mt-0.5 text-xs text-muted-foreground">Wiki 文档新增、编辑、删除</p>
+          <p className="ml-9 mt-0.5 text-xs text-muted-foreground">Wiki 文档新增、编辑、删除 · {vectorStatusText}</p>
         </div>
         <div className="flex items-center gap-2">
+          <Button
+            variant="outline"
+            size="sm"
+            onClick={() => void vectorize()}
+            disabled={!canWrite || actions.vectorizeWiki.isPending || indexStatus.data?.state === "running" || indexStatus.data?.state === "rebuilding"}
+          >
+            {actions.vectorizeWiki.isPending || indexStatus.data?.state === "running" || indexStatus.data?.state === "rebuilding" ? (
+              <Loader2 className="mr-2 size-4 animate-spin" />
+            ) : (
+              <Cpu className="mr-2 size-4" />
+            )}
+            手动向量化
+          </Button>
+          <Button variant="outline" size="sm" onClick={() => void exportExcel()} disabled={actions.exportWikiExcel.isPending}>
+            {actions.exportWikiExcel.isPending ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Download className="mr-2 size-4" />}
+            导出 Excel
+          </Button>
           <Button variant="outline" size="sm" onClick={() => void pages.refetch()}>
             <RefreshCw className="mr-2 size-4" />
             刷新

offline-backend-20260512/backend/CLAUDE.md

@@ -1329,3 +1329,52 @@ candidates fail with 409 rather than overwrite a later edit. Ambiguous or
 ordinary evidence questions must remain non-mutating, and cancellation or a
 stream error must never save a partial report. The rewrite prompt receives only
 the selected range plus the selected evidence set, never a new web search.
+
+### DeerFlow-local LLMWiki Wiki index
+
+`llmwiki.local_wiki_index` is an opt-in local-vector subsystem. WeKnora is the
+Wiki generation/sync source and the Wiki-only fallback while an entire mapping
+has not completed vectorization. The readiness gate requires `state=idle`, a
+successful full scan, matching embedding fingerprint, no failed pages,
+`ready_page_count == local_page_count`, and vectors for a non-empty library.
+Ready mappings read DeerFlow's `llmwiki_wiki_pages` /
+`llmwiki_wiki_vectors` / `llmwiki_wiki_sync_states` tables; incomplete mappings
+call only WeKnora `/knowledgebase/{id}/wiki/pages?query=...` plus Wiki-page
+detail. Never call `/knowledge-search` from internal search, RAG, or the
+`llmwiki_search` tool, and never return raw chunks as the fallback. Core code lives under
+`deerflow.integrations.weknora.local_index` and
+`deerflow.persistence.llmwiki_index`; preserve the harness → app import
+firewall. Gateway composition injects the store/search/sync services through
+`local_index.runtime`, and the lease-protected scheduler starts only when both
+`enabled` and `sync_enabled` are true.
+
+Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
+its vectors are replaced in one transaction only after strict batched
+embedding succeeds. Never delete the old generation before a remote embedding
+call and never fall back to WeKnora raw chunks while the local feature flag is
+enabled. NumPy matrices are per-mapping LRU snapshots invalidated by
+`index_revision`; large decode/matrix work runs through `asyncio.to_thread`.
+During a fingerprint rebuild the mapping uses the Wiki-page API until every
+page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests
+per mapping, so ready libraries stay local while incomplete libraries use that
+Wiki-only fallback.
+
+Owners manually force a full sync/vectorization through `POST
+/api/llmwiki/knowledge-bases/{id}/vectorize`; status is available through `GET
+/api/llmwiki/knowledge-bases/{id}/wiki-index-status`. The Wiki management page
+also exports all remote processed Wiki pages via `GET
+/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx`. Directory export must keep
+the full path, current directory, parent path, and dynamic per-level columns;
+long Markdown is split into Excel-safe continuation columns instead of being
+truncated. Workbook creation runs in `asyncio.to_thread`.
+
+Internal search authorization always starts from DeerFlow mapping ids and is
+rechecked by the Gateway. External endpoints use the isolated
+`/api/external/llmwiki/wiki` namespace and `X-API-Key`; require published +
+external-enabled + index-enabled on every request and response, and hard-deny
+the `system` / `对话沉淀` mapping. Never return remote WeKnora ids, provenance
+refs, credentials, or vector bytes externally. Regression tests:
+`tests/test_llmwiki_local_index_repository.py`,
+`tests/test_llmwiki_wiki_sync.py`, `tests/test_llmwiki_vector_search.py`, and
+`tests/test_external_llmwiki_search.py`, and
+`tests/test_llmwiki_wiki_retrieval.py`.

offline-backend-20260512/backend/README.md

@@ -746,3 +746,38 @@ without deleting the audit row. DeerFlow stores only the artifact reference
 (`threadId` + sandbox `path`, plus the optional session/node/name/MIME fields),
 never the Markdown body and never a task-system delivery. Continue to preview
 the file through the existing thread-artifact endpoint.
+
+## DeerFlow-local LLMWiki Wiki vector index
+
+LLMWiki can mirror processed WeKnora Wiki pages into DeerFlow and search only
+processed Wiki content. A knowledge base uses exact NumPy cosine similarity
+only after its whole Wiki has completed one coherent vectorization pass; until
+then internal search calls WeKnora's Wiki-page search/list API and reads the
+matched Wiki pages. It never falls back to raw documents or chunks. The local
+vector feature is disabled by default. Configure
+`llmwiki.local_wiki_index.embedding` first, then enable `enabled`,
+`sync_enabled`, and `internal_search_enabled` in `config.yaml`.
+
+The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and
+`llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are
+stored in DeerFlow's configured SQL database. Multi-worker sync is protected by
+database leases; searches use revision-checked per-knowledge-base LRU snapshots
+after the full-library readiness gate succeeds. Knowledge-base owners can click
+**手动向量化** in Wiki management (`POST
+/api/llmwiki/knowledge-bases/{id}/vectorize`) and follow its status at
+`GET /api/llmwiki/knowledge-bases/{id}/wiki-index-status`. Existing chunk
+citations remain readable while all new retrieval citations open Wiki pages.
+
+Wiki management also provides **导出 Excel**. `GET
+/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki
+page, including its full directory path, current directory, parent-directory
+path, one column per directory level, metadata, and complete Markdown content
+(split across continuation columns when Excel's per-cell limit is reached).
+
+Authenticated operations are available below `/api/llmwiki/wiki-index` and
+`POST /api/llmwiki/wiki-vector-search`. The service-to-service endpoints below
+`/api/external/llmwiki/wiki` require `X-API-Key`, are rate-limited, and only
+expose mappings that are both internally published and explicitly marked
+`external_search_enabled`; the system `对话沉淀` mapping is always denied.
+See `docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md` for configuration,
+security boundaries, rollout, and API contracts.

offline-backend-20260512/backend/app/gateway/app.py

@@ -49,6 +49,8 @@ from app.gateway.routers import (
     light_apps,
     llm_metrics,
     llmwiki,
+    llmwiki_index,
+    external_llmwiki,
     mcp,
     memory,
     menu_overrides,
@@ -989,6 +991,22 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
         # Run Alembic migrations before anything else touches the schema.
         await _run_db_migrations()
 
+        # Local Wiki mirror/index scans are opt-in and run outside request
+        # handling. Each knowledge base still uses a database lease, so this is
+        # safe when multiple Gateway workers start the same scheduler.
+        app.state.llmwiki_index_scheduler_task = None
+        local_wiki_index = app.state.config.llmwiki.local_wiki_index
+        if (
+            local_wiki_index.enabled
+            and local_wiki_index.sync_enabled
+            and getattr(app.state, "llmwiki_sync_service", None) is not None
+        ):
+            from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
+
+            app.state.llmwiki_index_scheduler_task = asyncio.create_task(
+                local_wiki_index_scheduler_loop(app)
+            )
+
         # ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
         try:
             from deerflow.config.memory_config import load_recall_budget_cache
@@ -1171,6 +1189,21 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
                 logger.exception("Scheduled task service failed to start")
 
         yield
+
+        llmwiki_index_scheduler_task = getattr(app.state, "llmwiki_index_scheduler_task", None)
+        if llmwiki_index_scheduler_task is not None:
+            llmwiki_index_scheduler_task.cancel()
+            try:
+                await llmwiki_index_scheduler_task
+            except asyncio.CancelledError:
+                pass
+            except Exception:
+                logger.exception("Local Wiki index scheduler shutdown error")
+        llmwiki_index_jobs = list(getattr(app.state, "llmwiki_index_jobs", set()))
+        for job in llmwiki_index_jobs:
+            job.cancel()
+        if llmwiki_index_jobs:
+            await asyncio.gather(*llmwiki_index_jobs, return_exceptions=True)
 
         # ?? curator ???
         curator_task = getattr(app.state, "curator_scheduler_task", None)
@@ -1457,6 +1490,8 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
 
     # DeerFlow-native LLMWiki management/search BFF (conditionally backed by WeKnora).
     app.include_router(llmwiki.router)
+    app.include_router(llmwiki_index.router)
+    app.include_router(external_llmwiki.router)
 
     # Memory API is mounted at /api/memory
     app.include_router(memory.router)
@@ -1654,9 +1689,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
     # --- ?????????? /page/workspace/roundtable/planning?---
     # ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
     # ??? frontend-web/docs/multi-agent-backend-dev.md
-    app.include_router(multi_agent.router)   # Step 2?init(SSE) + run/stream(leader|special)
-    app.include_router(intent.router)        # Step 1?/api/intent/init + /stream
-    app.include_router(recommend.router)     # Step 1?2?/api/recommend/stream?? init?
+    app.include_router(multi_agent.router)  # Step 2?init(SSE) + run/stream(leader|special)
+    app.include_router(intent.router)  # Step 1?/api/intent/init + /stream
+    app.include_router(recommend.router)  # Step 1?2?/api/recommend/stream?? init?
     # Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
 
     # Health probes are always unauthenticated (see

offline-backend-20260512/backend/app/gateway/auth_middleware.py

@@ -9,6 +9,8 @@ owner filtering works automatically via the sentinel pattern.
 Fine-grained permission checks remain in authz.py decorators.
 """
 
+import hashlib
+import hmac
 import re
 from collections.abc import Callable
 
@@ -81,6 +83,30 @@ def _is_public(path: str) -> bool:
     return any(path.startswith(prefix) for prefix in _PUBLIC_PATH_PREFIXES)
 
 
+def _is_external_llmwiki(path: str) -> bool:
+    return path.startswith("/api/external/llmwiki/")
+
+
+def _authorize_external_llmwiki(request: Request) -> JSONResponse | None:
+    """Authenticate the isolated service-to-service namespace before routing."""
+
+    from deerflow.config import get_app_config
+
+    config = get_app_config().llmwiki.local_wiki_index.external_api
+    supplied = request.headers.get("X-API-Key", "")
+    valid = bool(config.enabled and supplied) and any(
+        hmac.compare_digest(supplied, candidate)
+        for candidate in (config.api_key, config.previous_api_key)
+        if candidate
+    )
+    if not valid:
+        return JSONResponse(status_code=401, content={"detail": "Invalid API key"})
+    request.state.external_api_key_fingerprint = hashlib.sha256(
+        supplied.encode("utf-8")
+    ).hexdigest()[:8]
+    return None
+
+
 def _is_weknora_embed_proxy_request(request: Request) -> bool:
     """Let the WeKnora embed proxy verify its own signed iframe cookie.
 
@@ -137,7 +163,11 @@ class AuthMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next: Callable) -> Response:
         # Match the application-relative path so a reverse-proxy prefix
         # (e.g. /xxx/api/public/...) does not defeat the public whitelist.
-        if _is_public(app_relative_path(request)):
+        relative_path = app_relative_path(request)
+        if _is_external_llmwiki(relative_path):
+            rejected = _authorize_external_llmwiki(request)
+            return rejected if rejected is not None else await call_next(request)
+        if _is_public(relative_path):
             return await call_next(request)
         if _is_weknora_embed_proxy_request(request):
             return await call_next(request)

offline-backend-20260512/backend/app/gateway/deps.py

@@ -308,8 +308,61 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
         from deerflow.integrations.weknora.runtime import register_llmwiki_store
         from deerflow.persistence.llmwiki import make_llmwiki_store
 
-        app.state.llmwiki_store = make_llmwiki_store(sf)
-        register_llmwiki_store(app.state.llmwiki_store)
+        app.state.llmwiki_store = make_llmwiki_store(sf)
+        register_llmwiki_store(app.state.llmwiki_store)
+
+        # DeerFlow-local Wiki mirror/vector index. The store is always wired so
+        # migrations and status inspection work while feature flags are off;
+        # network/search services only exist when explicitly enabled.
+        from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
+        from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
+
+        app.state.llmwiki_index_store = make_llmwiki_index_store(sf)
+        app.state.llmwiki_embedding = None
+        app.state.llmwiki_vector_cache = None
+        app.state.llmwiki_vector_search = None
+        app.state.llmwiki_sync_service = None
+        app.state.llmwiki_retrieval_service = None
+        local_index_config = config.llmwiki.local_wiki_index
+        from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
+
+        resolved_llmwiki = get_resolved_llmwiki_runtime(config)
+        weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
+        if local_index_config.enabled:
+            from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
+            from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
+            from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+            from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+
+            app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
+            app.state.llmwiki_vector_cache = WikiVectorCache(
+                app.state.llmwiki_index_store,
+                max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
+                revision_check_seconds=local_index_config.cache_revision_check_seconds,
+            )
+            app.state.llmwiki_vector_search = WikiVectorSearchService(local_index_config, app.state.llmwiki_embedding, app.state.llmwiki_vector_cache)
+            if weknora_client is not None:
+                app.state.llmwiki_sync_service = WikiSyncService(
+                    local_index_config,
+                    app.state.llmwiki_index_store,
+                    app.state.llmwiki_embedding,
+                    app.state.llmwiki_vector_cache,
+                    weknora_client,
+                )
+        if weknora_client is not None or app.state.llmwiki_vector_search is not None:
+            from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
+
+            app.state.llmwiki_retrieval_service = WikiRetrievalService(
+                app.state.llmwiki_index_store,
+                vector_search=(app.state.llmwiki_vector_search if local_index_config.internal_search_enabled else None),
+                client=weknora_client,
+            )
+        register_local_wiki_index_runtime(
+            store=app.state.llmwiki_index_store,
+            search=app.state.llmwiki_vector_search,
+            sync=app.state.llmwiki_sync_service,
+            retrieval=app.state.llmwiki_retrieval_service,
+        )
 
         from deerflow.persistence.taskcop_tasks import make_taskcop_task_store
 
@@ -499,8 +552,14 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
 
         try:
             yield
-        finally:
-            from deerflow.persistence.notifications import set_notification_store
+        finally:
+            from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
+
+            register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
+            llmwiki_embedding = getattr(app.state, "llmwiki_embedding", None)
+            if llmwiki_embedding is not None:
+                await llmwiki_embedding.close()
+            from deerflow.persistence.notifications import set_notification_store
 
             set_notification_store(None)
             dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)

offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py

@@ -0,0 +1,53 @@
+"""Best-effort periodic full scans for the DeerFlow-local Wiki index."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Coroutine
+from typing import Any
+
+from fastapi import FastAPI
+
+logger = logging.getLogger(__name__)
+
+
+def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
+    """Keep fire-and-forget index jobs strongly referenced until completion."""
+
+    jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None)
+    if jobs is None:
+        jobs = set()
+        app.state.llmwiki_index_jobs = jobs
+    task = asyncio.create_task(coroutine)
+    jobs.add(task)
+    task.add_done_callback(jobs.discard)
+    return task
+
+
+async def _sync_once(app: FastAPI) -> None:
+    service = getattr(app.state, "llmwiki_sync_service", None)
+    mapping_store = getattr(app.state, "llmwiki_store", None)
+    if service is None or mapping_store is None:
+        return
+    config = app.state.config.llmwiki.local_wiki_index
+    mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
+    semaphore = asyncio.Semaphore(config.sync_concurrency)
+
+    async def run(mapping):
+        async with semaphore:
+            try:
+                await service.sync_mapping(mapping)
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True)
+
+    await asyncio.gather(*(run(mapping) for mapping in mappings))
+
+
+async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
+    interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
+    while True:
+        await _sync_once(app)
+        await asyncio.sleep(interval)

offline-backend-20260512/backend/app/gateway/llmwiki_rag.py

@@ -26,8 +26,8 @@ from langchain_core.messages import HumanMessage
 from app.gateway.deps import get_optional_user_from_request
 from app.gateway.utils import sanitize_log_param
 from deerflow.config.agents_config import load_agent_config, validate_agent_id
-from deerflow.integrations.weknora.client import WeKnoraError
-from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+from deerflow.integrations.weknora.runtime import get_resolved_llmwiki_runtime
 from deerflow.persistence.llmwiki import LlmWikiStore
 from deerflow.runtime.user_context import get_effective_user_id
 
@@ -146,6 +146,24 @@ def _truncate(text: str, max_chars: int) -> str:
 
 
 def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index: int) -> dict[str, Any]:
+    if result.get("kind") == "wiki_page":
+        sections = list(result.get("matched_sections") or [])
+        content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
+        raw = {**result, "source": mapping.get("name") or "LLMWiki"}
+        return {
+            "index": index,
+            "id": result.get("wiki_page_id") or result.get("id"),
+            "title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
+            "type": "llmwiki_wiki",
+            "snippet": _truncate(content or str(result.get("summary") or ""), 360),
+            "content": content,
+            "source": mapping.get("name") or "LLMWiki",
+            "score": float(result.get("score") or 0),
+            "url": result.get("url") or "",
+            "skillName": "llmwiki_search",
+            "mode": "citation",
+            "raw": raw,
+        }
     chunk_id = str(result.get("chunk_id") or "")
     title = str(result.get("title") or result.get("filename") or mapping.get("name") or f"LLMWiki 片段 {index}")
     content = str(result.get("content") or "")
@@ -175,10 +193,7 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
 
 def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
     if not sources:
-        return (
-            "当前用户选择了 LLMWiki 知识库，但 DeerFlow 未检索到可用片段。"
-            "请直接说明没有检索到可用背景资料，不要编造知识库内容或引用编号。"
-        )
+        return "当前用户选择了 LLMWiki 知识库，但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料，不要编造知识库内容或引用编号。"
 
     lines = [
         "当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
@@ -187,7 +202,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
         "",
         f"用户问题：{query}",
         "",
-        f"<llmwiki_reference_batch source_count=\"{len(sources)}\">",
+        f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
     ]
     for source in sources:
         meta = " / ".join(
@@ -203,7 +218,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
             lines.append(f"来源：{meta}")
         lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
         lines.append("")
-    lines.append("</llmwiki_reference_batch>")
+    lines.append("</llmwiki_wiki_reference_batch>")
     return "\n".join(lines).strip()
 
 
@@ -218,7 +233,8 @@ async def build_llmwiki_rag_context(
         return LlmWikiRagContext([], 0, "", [], "")
 
     runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
-    if not runtime.weknora_enabled:
+    local_config = request.app.state.config.llmwiki.local_wiki_index
+    if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
         return LlmWikiRagContext(selected_ids, 0, "", [], "")
     store = _store(request)
     if store is None:
@@ -235,29 +251,25 @@ async def build_llmwiki_rag_context(
         if row is None:
             raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
         rows.append(row)
-    remote_to_local = {str(row["weknora_id"]): row for row in rows}
-
-    try:
-        results = await build_weknora_client(runtime).search(query, [str(row["weknora_id"]) for row in rows])
-    except WeKnoraError as exc:
-        logger.warning(
-            "LLMWiki pre-retrieval failed for user=%s kb_count=%s: %s",
-            sanitize_log_param(user_id),
-            len(rows),
-            exc,
-        )
+    search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
+    if search_service is None:
         return LlmWikiRagContext(
             selected_ids,
             0,
-            f"当前用户选择了 LLMWiki 知识库，但自动检索失败：{exc}。请说明暂时无法检索知识库背景资料，不要编造引用。",
+            "Wiki 检索暂不可用。请明确说明系统检索不可用，不要表述为资料不存在。",
             [],
             query,
         )
-
+    try:
+        payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
+    except WikiIndexError as exc:
+        logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
+        return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用（{exc.code}）。请明确说明系统检索状态，不要声称知识库没有资料。", [], query)
+    mapping_by_id = {str(row["id"]): row for row in rows}
     sources: list[dict[str, Any]] = []
-    for result in results:
-        mapping = remote_to_local.get(str(result.get("knowledge_base_id") or ""))
-        if mapping is None or not str(result.get("content") or "").strip():
+    for result in payload.get("results") or []:
+        mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
+        if mapping is None:
             continue
         sources.append(_source_from_result(result, mapping, len(sources) + 1))
         if len(sources) >= _MAX_SOURCES:

offline-backend-20260512/backend/app/gateway/routers/external_llmwiki.py

@@ -0,0 +1,133 @@
+"""API-key protected public Wiki vector search backed only by local data."""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict, deque
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+
+router = APIRouter(prefix="/api/external/llmwiki/wiki", tags=["external-llmwiki"])
+_requests: dict[str, deque[float]] = defaultdict(deque)
+
+
+class ExternalWikiSearchRequest(BaseModel):
+    query: str = Field(..., min_length=1, max_length=10000)
+    knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+    limit: int | None = Field(default=None, ge=1, le=100)
+    include_content: bool = False
+
+
+def _config(request: Request):
+    config = request.app.state.config.llmwiki.local_wiki_index
+    if not config.enabled or not config.external_api.enabled:
+        raise HTTPException(status_code=404, detail="Not found")
+    return config, config.external_api
+
+
+def _rate_limit(request: Request, rpm: int) -> None:
+    ip = request.client.host if request.client else "unknown"
+    key = f"{getattr(request.state, 'external_api_key_fingerprint', 'unknown')}:{ip}"
+    now = time.monotonic()
+    queue = _requests[key]
+    while queue and queue[0] <= now - 60:
+        queue.popleft()
+    if len(queue) >= rpm:
+        raise HTTPException(status_code=429, detail="Rate limit exceeded")
+    queue.append(now)
+
+
+def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
+    return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
+
+
+async def _external_mappings(request: Request, requested: list[str]) -> list[dict[str, Any]]:
+    mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+    allowed = {row["id"]: row for row in mappings if row.get("publication_status") == "published" and row.get("external_search_enabled", False) and row.get("wiki_index_enabled", True) and not _is_conversation_deposit(row)}
+    if requested:
+        ids = list(dict.fromkeys(requested))
+        if any(mapping_id not in allowed for mapping_id in ids):
+            raise HTTPException(status_code=404, detail="Knowledge base not found")
+        return [allowed[mapping_id] for mapping_id in ids]
+    return list(allowed.values())
+
+
+@router.post("/vector-search")
+async def external_vector_search(request: Request, body: ExternalWikiSearchRequest) -> dict[str, Any]:
+    _, external = _config(request)
+    _rate_limit(request, external.requests_per_minute)
+    if len(body.query) > external.max_query_chars:
+        raise HTTPException(status_code=413, detail="Query is too large")
+    if len(body.knowledge_base_ids) > external.max_knowledge_bases:
+        raise HTTPException(status_code=413, detail="Too many knowledge bases")
+    limit = body.limit or external.default_results
+    if limit > external.max_results:
+        raise HTTPException(status_code=400, detail="Result limit is too large")
+    mappings = await _external_mappings(request, body.knowledge_base_ids)
+    search = getattr(request.app.state, "llmwiki_vector_search", None)
+    if search is None:
+        raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
+    try:
+        result = await search.search(body.query, mappings, top_k_pages=limit, external=True)
+    except WikiIndexError as exc:
+        raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
+    # Re-read publication/external flags after vector computation so a switch
+    # turned off during an in-flight request cannot leak a cached result.
+    still_allowed = {row["id"] for row in await _external_mappings(request, [])}
+    result["results"] = [item for item in result["results"] if item.get("knowledge_base_id") in still_allowed]
+    remaining = external.include_content_max_chars if body.include_content else 0
+    items = []
+    for result_item in result["results"]:
+        section = (result_item.get("matched_sections") or [{}])[0]
+        snippet = str(section.get("content") or "")[:1000]
+        content = ""
+        if remaining > 0:
+            content = "\n\n".join(str(item.get("content") or "") for item in result_item.get("matched_sections") or [])[:remaining]
+            remaining -= len(content)
+        item = {
+            "knowledge_base_id": result_item["knowledge_base_id"],
+            "knowledge_base_name": result_item.get("knowledge_base_name") or "",
+            "wiki_slug": result_item["wiki_slug"],
+            "title": result_item.get("title") or "",
+            "summary": result_item.get("summary") or "",
+            "matched_heading": section.get("heading"),
+            "snippet": snippet,
+            "page_type": result_item.get("page_type") or "article",
+            "page_version": result_item.get("page_version") or 0,
+            "updated_at": result_item.get("updated_at"),
+            "score": result_item.get("score") or 0,
+        }
+        if body.include_content:
+            item["content"] = content
+        items.append(item)
+    return {"query": body.query, "mode": "vector", "items": items, "count": len(items), "partial": result.get("partial", False)}
+
+
+@router.get("/pages/{mapping_id}/{slug:path}")
+async def external_wiki_page(request: Request, mapping_id: str, slug: str) -> dict[str, Any]:
+    _, external = _config(request)
+    _rate_limit(request, external.requests_per_minute)
+    mappings = await _external_mappings(request, [mapping_id])
+    if not mappings:
+        raise HTTPException(status_code=404, detail="Wiki page not found")
+    page = await request.app.state.llmwiki_index_store.get_page(mapping_id, slug.strip("/"))
+    if page is None or page.get("status") != "published" or page.get("index_status") != "ready" or page.get("is_remote_deleted"):
+        raise HTTPException(status_code=404, detail="Wiki page not found")
+    # Recheck after reading the page so publication/external revocation takes
+    # effect even when it races this detail request.
+    await _external_mappings(request, [mapping_id])
+    return {
+        "knowledge_base_id": mapping_id,
+        "knowledge_base_name": mappings[0].get("name") or "",
+        "wiki_slug": page["slug"],
+        "title": page.get("title") or "",
+        "summary": page.get("summary") or "",
+        "content": str(page.get("content_md") or "")[: external.include_content_max_chars],
+        "page_type": page.get("page_type") or "article",
+        "page_version": page.get("page_version") or 0,
+        "updated_at": page.get("remote_updated_at") or page.get("updated_at"),
+    }

offline-backend-20260512/backend/app/gateway/routers/llmwiki.py

@@ -2,18 +2,21 @@
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
 import json
 import logging
 import re
 import time
 from dataclasses import replace
+from datetime import UTC, datetime
+from io import BytesIO
 from typing import Any, Literal
 from urllib.parse import quote, urlencode, urlsplit
 
 import httpx
 from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
-from fastapi.responses import JSONResponse, RedirectResponse, Response
+from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
 from pydantic import BaseModel, Field
 
 from app.gateway.csrf_middleware import is_secure_request
@@ -27,17 +30,28 @@ from app.gateway.llmwiki_deposit import (
     ensure_conversation_deposit_mapping,
 )
 from app.gateway.weknora_embed import (
-    WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
     WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
+)
+from app.gateway.weknora_embed import (
+    WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
+)
+from app.gateway.weknora_embed import (
     WEKNORA_EMBED_TTL_SECONDS as _WEKNORA_EMBED_TTL_SECONDS,
+)
+from app.gateway.weknora_embed import (
     WeKnoraEmbedContext,
+)
+from app.gateway.weknora_embed import (
     sign_weknora_embed_payload as _sign_embed_payload,
+)
+from app.gateway.weknora_embed import (
     verify_weknora_embed_cookie as _verify_embed_cookie,
 )
 from deerflow.config.agents_config import load_agent_config
 from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
 from deerflow.config.system_settings import load_system_settings, save_system_settings
 from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
 from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
 from deerflow.persistence.llmwiki import LlmWikiStore
 
@@ -168,6 +182,8 @@ class SearchRequest(BaseModel):
     query: str = Field(..., min_length=1, max_length=4000)
     scope: Literal["personal", "public", "all"] = "all"
     knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+    top_k: int = Field(default=8, ge=1, le=50)
+    include_drafts: bool = False
 
 
 class KnowledgeUrlImport(BaseModel):
@@ -223,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
     return value
 
 
+def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
+    service = getattr(request.app.state, "llmwiki_sync_service", None)
+    config = request.app.state.config.llmwiki.local_wiki_index
+    if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
+        from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
+
+        spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
+
+
 async def _actor(request: Request) -> tuple[str, bool]:
     user = await get_optional_user_from_request(request)
     if user is None:
@@ -575,12 +600,12 @@ async def _authorize_weknora_api_proxy(
     # page to render upload controls and current-tenant state. They do not
     # expose other knowledge-base contents.
     if read_method and tail and tail[0] in {
-        "models",
-        "system",
-        "storage-backends",
-        "vector-stores",
-        "web-search-providers",
-        "mcp-services",
+            "models",
+            "system",
+            "storage-backends",
+            "vector-stores",
+            "web-search-providers",
+            "mcp-services",
     }:
         return
 
@@ -654,7 +679,7 @@ def _with_public_prefix(prefix: str, path: str) -> str:
 
 def _without_public_prefix(prefix: str, path: str) -> str:
     if prefix and path.startswith(f"{prefix}/"):
-        return path[len(prefix):]
+        return path[len(prefix) :]
     if prefix and path == prefix:
         return "/"
     return path
@@ -1626,6 +1651,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
         "description": description,
         "type": source.get("type") or row.get("kb_type") or "document",
         "publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
+        "wiki_index_enabled": row.get("wiki_index_enabled", True),
+        "external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
+        "external_search_updated_at": row.get("external_search_updated_at"),
         "owner_user_id": row.get("owner_user_id") if is_admin else None,
         "is_owner": is_owner,
         "can_write": not is_conversation_deposit and (is_admin or is_owner),
@@ -1816,6 +1844,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
     }
 
 
+async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
+    """Read every Wiki page and its processed Markdown, never raw documents."""
+
+    summaries: dict[str, dict[str, Any]] = {}
+    page_number = 1
+    while True:
+        listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
+        rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
+        for item in rows:
+            slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
+            if slug:
+                summaries[slug] = item
+        total_pages = max(1, int(listing.get("total_pages") or 1))
+        if page_number >= total_pages or not rows:
+            break
+        page_number += 1
+
+    pages: list[dict[str, Any]] = []
+    entries = list(summaries.items())
+    for start in range(0, len(entries), 20):
+        batch = entries[start : start + 20]
+        details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
+        pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
+    return pages
+
+
+def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
+    parent_slug = str(page.get("parent_slug") or "").strip("/")
+    if parent_slug:
+        ancestors: list[str] = []
+        visited: set[str] = set()
+        current = by_slug.get(parent_slug)
+        while current is not None:
+            slug = str(current.get("slug") or "").strip("/")
+            if not slug or slug in visited:
+                break
+            visited.add(slug)
+            ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
+            current_parent = str(current.get("parent_slug") or "").strip("/")
+            current = by_slug.get(current_parent) if current_parent else None
+        if ancestors:
+            return ancestors
+
+    category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
+    if category_path:
+        return category_path
+    wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
+    if len(wiki_path) > 2:
+        return wiki_path[1:-1]
+    slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
+    return slug_parts[:-1]
+
+
+def _safe_excel_text(value: Any) -> str:
+    text = str(value or "")
+    return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
+
+
+def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
+    from openpyxl import Workbook
+    from openpyxl.styles import Alignment, Font, PatternFill
+
+    pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
+    by_slug = {page["slug"]: page for page in pages if page["slug"]}
+    directories = [_wiki_directory_parts(page, by_slug) for page in pages]
+    max_depth = max((len(parts) for parts in directories), default=0)
+    max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
+    columns = [
+        "知识库",
+        "标题",
+        "Slug",
+        "目录完整路径",
+        "所在目录",
+        "父级目录完整路径",
+        *(f"第{level}级目录" for level in range(1, max_depth + 1)),
+        "摘要",
+        *("正文" if index == 1 else f"正文（续{index - 1}）" for index in range(1, max_content_parts + 1)),
+        "页面类型",
+        "状态",
+        "别名",
+        "版本",
+        "创建时间",
+        "更新时间",
+    ]
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.title = "Wiki 全量导出"
+    sheet.append(columns)
+    for cell in sheet[1]:
+        cell.font = Font(bold=True, color="FFFFFF")
+        cell.fill = PatternFill("solid", fgColor="1F4E78")
+        cell.alignment = Alignment(horizontal="center", vertical="center")
+    for page, directory in zip(pages, directories, strict=True):
+        content = str(page.get("content") or "")
+        content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
+        content_parts.extend([""] * (max_content_parts - len(content_parts)))
+        row = [
+            knowledge_base_name,
+            page.get("title"),
+            page.get("slug"),
+            " / ".join(directory),
+            directory[-1] if directory else "根目录",
+            " / ".join(directory[:-1]),
+            *directory,
+            *([""] * (max_depth - len(directory))),
+            page.get("summary"),
+            *content_parts,
+            page.get("page_type"),
+            page.get("status"),
+            "、".join(str(item) for item in page.get("aliases") or []),
+            page.get("version"),
+            page.get("created_at"),
+            page.get("updated_at"),
+        ]
+        sheet.append([_safe_excel_text(value) for value in row])
+    sheet.freeze_panes = "A2"
+    sheet.auto_filter.ref = sheet.dimensions
+    for column in sheet.columns:
+        letter = column[0].column_letter
+        heading = str(column[0].value or "")
+        sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
+        for cell in column[1:]:
+            cell.alignment = Alignment(vertical="top", wrap_text=True)
+    output = BytesIO()
+    workbook.save(output)
+    output.seek(0)
+    return output
+
+
 def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
     target_query = urlencode({"tab": tab}) if tab != "documents" else ""
     target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
@@ -2421,12 +2578,37 @@ async def list_wiki_pages(
         _raise_upstream(exc)
 
 
+@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
+async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
+    """Export every processed Wiki page with its current and parent directories."""
+
+    row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
+    client = _client_or_503()
+    try:
+        pages = await _all_wiki_pages(client, str(row["weknora_id"]))
+    except WeKnoraError as exc:
+        _raise_upstream(exc)
+    try:
+        output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
+    except ImportError as exc:
+        raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
+    timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+    safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
+    filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
+    return StreamingResponse(
+        output,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
+    )
+
+
 @router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
 async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
     row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
     payload = body.model_dump(exclude_none=True)
     try:
         page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
+        _schedule_local_wiki_sync(request, row)
         return _wiki_page_view(page) or page
     except WeKnoraError as exc:
         _raise_upstream(exc)
@@ -2454,6 +2636,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
             _raise_upstream(exc)
     try:
         page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
+        _schedule_local_wiki_sync(request, row)
         return _wiki_page_view(page) or page
     except WeKnoraError as exc:
         _raise_upstream(exc)
@@ -2464,6 +2647,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
     row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
     try:
         await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
+        _schedule_local_wiki_sync(request, row)
     except WeKnoraError as exc:
         _raise_upstream(exc)
 
@@ -2547,32 +2731,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
             rows.append(row)
     else:
         rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
+    service = getattr(request.app.state, "llmwiki_retrieval_service", None)
+    if service is None:
+        raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
+    include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
     try:
-        results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
-    except WeKnoraError as exc:
-        _raise_upstream(exc)
-    local_by_remote = {str(row["weknora_id"]): row for row in rows}
-    visible: list[dict[str, Any]] = []
-    for result in results:
-        mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
-        if mapping is None:
-            continue
-        visible_result = (
-            _sanitize_conversation_deposit_value(result)
-            if _is_conversation_deposit_mapping(mapping)
-            else result
-        )
-        params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
-        visible.append(
-            {
-                **visible_result,
-                "knowledge_base_id": mapping["id"],
-                "knowledge_base_name": mapping.get("name") or "",
-                "source": mapping.get("name") or "LLMWiki",
-                "url": f"/weknora-source-preview?{params}",
-            }
-        )
-    return {"results": visible, "knowledge_base_count": len(rows)}
+        result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
+    except WikiIndexError as exc:
+        raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
+    mapping_by_id = {str(row["id"]): row for row in rows}
+    result["results"] = [
+        _sanitize_conversation_deposit_value(item)
+        if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
+        else item
+        for item in result.get("results") or []
+    ]
+    return result
 
 
 @router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)

offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py

@@ -0,0 +1,290 @@
+"""Authenticated APIs for DeerFlow-local Wiki vector search and administration."""
+
+from __future__ import annotations
+
+from typing import Any
+from uuid import uuid4
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from app.gateway.deps import get_optional_user_from_request
+from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
+from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+
+router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
+
+
+class WikiVectorSearchRequest(BaseModel):
+    query: str = Field(..., min_length=1, max_length=4000)
+    knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+    top_k: int = Field(default=8, ge=1, le=50)
+    include_drafts: bool = False
+    include_content: bool = True
+
+
+class WikiIndexActionRequest(BaseModel):
+    knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+    force: bool = False
+    reason: str = Field(default="manual", max_length=255)
+
+
+class WikiIndexSettingsRequest(BaseModel):
+    wiki_index_enabled: bool | None = None
+    external_search_enabled: bool | None = None
+
+
+async def _actor(request: Request) -> tuple[str, bool]:
+    user = await get_optional_user_from_request(request)
+    if user is None:
+        return "default", True
+    return str(user.id), getattr(user, "system_role", None) == "admin"
+
+
+def _services(request: Request):
+    config = request.app.state.config.llmwiki.local_wiki_index
+    if not config.enabled:
+        raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
+    search = getattr(request.app.state, "llmwiki_vector_search", None)
+    index_store = getattr(request.app.state, "llmwiki_index_store", None)
+    if search is None or index_store is None:
+        raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE", "message": "Local Wiki index is unavailable"})
+    return config, search, index_store
+
+
+async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
+    user_id, is_admin = await _actor(request)
+    mapping_store = request.app.state.llmwiki_store
+    if mapping_ids:
+        rows = []
+        for mapping_id in dict.fromkeys(mapping_ids):
+            row = await mapping_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
+            if row is None or not row.get("wiki_index_enabled", True):
+                raise HTTPException(status_code=404, detail="Knowledge base not found")
+            rows.append(row)
+    else:
+        rows = [row for row in await mapping_store.list_visible(user_id, scope="all", is_admin=is_admin) if row.get("wiki_index_enabled", True)]
+    return rows, user_id, is_admin
+
+
+@router.post("/wiki-vector-search")
+async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
+    config, search, _ = _services(request)
+    if not config.internal_search_enabled:
+        raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
+    rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
+    include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
+    try:
+        result = await search.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
+    except WikiIndexError as exc:
+        raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
+    if not body.include_content:
+        for item in result["results"]:
+            item.pop("content", None)
+            item.pop("snippet", None)
+            for section in item["matched_sections"]:
+                section.pop("content", None)
+    return result
+
+
+@router.get("/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}")
+async def local_wiki_page(
+    request: Request,
+    mapping_id: str,
+    slug: str,
+    include_provenance: bool = Query(default=False),
+    include_archived: bool = Query(default=False),
+) -> dict[str, Any]:
+    _, _, index_store = _services(request)
+    user_id, is_admin = await _actor(request)
+    mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
+    if mapping is None:
+        raise HTTPException(status_code=404, detail="Wiki page not found")
+    page = await index_store.get_page(mapping_id, slug.strip("/"))
+    if page is None or page.get("is_remote_deleted"):
+        raise HTTPException(status_code=404, detail="Wiki page not found")
+    owner_or_admin = is_admin or str(mapping.get("owner_user_id")) == user_id
+    if page.get("status") == "archived" and not (include_archived and owner_or_admin):
+        raise HTTPException(status_code=404, detail="Wiki page not found")
+    if page.get("status") != "published" and not owner_or_admin:
+        raise HTTPException(status_code=404, detail="Wiki page not found")
+    result = {
+        "id": page["id"],
+        "knowledge_base_id": mapping_id,
+        "title": page.get("title") or "",
+        "slug": page["slug"],
+        "summary": page.get("summary") or "",
+        "content": page.get("content_md") or "",
+        "page_type": page.get("page_type") or "article",
+        "status": page.get("status"),
+        "aliases": page.get("aliases") or [],
+        "category_path": page.get("category_path") or [],
+        "parent_slug": page.get("parent_slug"),
+        "wiki_path": page.get("wiki_path"),
+        "page_version": page.get("page_version") or 0,
+        "updated_at": page.get("remote_updated_at") or page.get("updated_at"),
+        "created_at": page.get("remote_created_at") or page.get("created_at"),
+    }
+    if owner_or_admin:
+        result.update({"index_status": page.get("index_status"), "indexed_at": page.get("indexed_at")})
+        if include_provenance:
+            result.update({"source_refs": page.get("source_refs") or [], "chunk_refs": page.get("chunk_refs") or []})
+    return result
+
+
+@router.get("/wiki-index/status")
+async def wiki_index_status(request: Request) -> dict[str, Any]:
+    _, is_admin = await _actor(request)
+    if not is_admin:
+        raise HTTPException(status_code=403, detail="Administrator access required")
+    config = request.app.state.config.llmwiki.local_wiki_index
+    index_store = getattr(request.app.state, "llmwiki_index_store", None)
+    if index_store is None:
+        raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
+    mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+    states = {row["knowledge_base_mapping_id"]: row for row in await index_store.list_index_status()}
+    items = []
+    for mapping in mappings:
+        state = states.get(mapping["id"], {"state": "not_synced", "index_revision": 0})
+        items.append(
+            {
+                **state,
+                "knowledge_base_id": mapping["id"],
+                "knowledge_base_name": mapping.get("name") or "",
+                "wiki_index_enabled": mapping.get("wiki_index_enabled", True),
+                "external_search_enabled": mapping.get("external_search_enabled", False),
+            }
+        )
+    return {
+        "enabled": config.enabled,
+        "sync_enabled": config.sync_enabled,
+        "internal_search_enabled": config.internal_search_enabled,
+        "embedding_model": config.embedding.model,
+        "embedding_fingerprint": (
+            request.app.state.llmwiki_embedding.fingerprint[:12]
+            if getattr(request.app.state, "llmwiki_embedding", None) is not None
+            else None
+        ),
+        "knowledge_bases": items,
+    }
+
+
+async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
+    service = getattr(app.state, "llmwiki_sync_service", None)
+    if service is None:
+        return
+    mappings = await app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+    wanted = set(mapping_ids)
+    for mapping in mappings:
+        if mapping.get("wiki_index_enabled", True) and (not wanted or mapping["id"] in wanted):
+            try:
+                await service.sync_mapping(mapping, force=force)
+            except Exception:
+                continue
+
+
+async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
+    service = getattr(app.state, "llmwiki_sync_service", None)
+    if service is None:
+        return
+    try:
+        await service.sync_mapping(mapping, force=force)
+    except Exception:
+        return
+
+
+@router.post("/wiki-index/sync", status_code=202)
+async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
+    _services(request)
+    _, is_admin = await _actor(request)
+    if not is_admin:
+        raise HTTPException(status_code=403, detail="Administrator access required")
+    job_id = str(uuid4())
+    spawn_local_wiki_index_job(request.app, _run_sync(request.app, body.knowledge_base_ids, body.force))
+    return {"job_id": job_id, "status": "accepted"}
+
+
+@router.post("/wiki-index/rebuild", status_code=202)
+async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
+    _, _, index_store = _services(request)
+    _, is_admin = await _actor(request)
+    if not is_admin:
+        raise HTTPException(status_code=403, detail="Administrator access required")
+    mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+    ids = body.knowledge_base_ids or [row["id"] for row in mappings if row.get("wiki_index_enabled", True)]
+    fingerprint = request.app.state.llmwiki_embedding.fingerprint
+    await index_store.mark_rebuild_required(ids, fingerprint)
+    job_id = str(uuid4())
+    spawn_local_wiki_index_job(request.app, _run_sync(request.app, ids, True))
+    return {"job_id": job_id, "status": "accepted", "reason": body.reason}
+
+
+@router.post("/wiki-index/knowledge-bases/{mapping_id}/sync", status_code=202)
+async def sync_one_wiki_index(request: Request, mapping_id: str, force: bool = Query(default=False)) -> dict[str, Any]:
+    return await sync_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=force))
+
+
+@router.post("/wiki-index/knowledge-bases/{mapping_id}/rebuild", status_code=202)
+async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str = Query(default="manual", max_length=255)) -> dict[str, Any]:
+    return await rebuild_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=True, reason=reason))
+
+
+@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
+async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
+    user_id, is_admin = await _actor(request)
+    mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
+    if mapping is None:
+        raise HTTPException(status_code=404, detail="Knowledge base not found")
+    index_store = getattr(request.app.state, "llmwiki_index_store", None)
+    if index_store is None:
+        raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
+    states = await index_store.list_index_status([mapping_id])
+    state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
+    embedding = getattr(request.app.state, "llmwiki_embedding", None)
+    fingerprint = embedding.fingerprint if embedding is not None else None
+    fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
+    return {
+        **state,
+        "knowledge_base_id": mapping_id,
+        "local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
+        "wiki_index_enabled": mapping.get("wiki_index_enabled", True),
+        "fully_vectorized": fully_vectorized,
+        "retrieval_mode": "vector" if fully_vectorized else "wiki_api",
+    }
+
+
+@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
+async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
+    config = request.app.state.config.llmwiki.local_wiki_index
+    if not config.enabled:
+        raise HTTPException(
+            status_code=409,
+            detail={"code": "WIKI_INDEX_DISABLED", "message": "本地 Wiki 向量化未在配置中启用"},
+        )
+    service = getattr(request.app.state, "llmwiki_sync_service", None)
+    if service is None:
+        raise HTTPException(status_code=503, detail={"code": "WIKI_VECTORIZATION_UNAVAILABLE"})
+    user_id, is_admin = await _actor(request)
+    mapping_store = request.app.state.llmwiki_store
+    mapping = await mapping_store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
+    if mapping is None:
+        raise HTTPException(status_code=404, detail="Knowledge base not found")
+    if not mapping.get("wiki_index_enabled", True):
+        mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
+    job_id = str(uuid4())
+    spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
+    return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
+
+
+@router.patch("/knowledge-bases/{mapping_id}/wiki-index-settings")
+async def update_wiki_index_settings(request: Request, mapping_id: str, body: WikiIndexSettingsRequest) -> dict[str, Any]:
+    user_id, is_admin = await _actor(request)
+    store = request.app.state.llmwiki_store
+    current = await store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
+    if current is None:
+        raise HTTPException(status_code=404, detail="Knowledge base not found")
+    if body.external_search_enabled and str(current.get("owner_user_id")) == "system" and str(current.get("name") or "").strip() == "对话沉淀":
+        raise HTTPException(status_code=403, detail="Conversation deposit can never be exposed externally")
+    updated = await store.update_mapping(mapping_id, wiki_index_enabled=body.wiki_index_enabled, external_search_enabled=body.external_search_enabled)
+    return {"knowledge_base_id": mapping_id, "wiki_index_enabled": updated.get("wiki_index_enabled", True), "external_search_enabled": updated.get("external_search_enabled", False)}

offline-backend-20260512/backend/config.yaml

@@ -534,6 +534,45 @@ llmwiki:
     web_base_url: "http://119.254.86.251:8088"
     admin_email: "lqq@qq.com"
     admin_password: "1qaz@WSX"
+  # DeerFlow-local Wiki mirror/vector index. Keep disabled until the
+  # OpenAI-compatible embedding endpoint below is configured and reachable.
+  local_wiki_index:
+    enabled: false
+    sync_enabled: false
+    internal_search_enabled: false
+    reference_ui_enabled: false
+    sync_interval_seconds: 120
+    chunk_max_chars: 1000
+    chunk_overlap_chars: 120
+    chunk_min_chars: 80
+    embedding:
+      enabled: true
+      provider: openai
+      model: bge-m3
+      base_url: ""
+      api_key: ""
+      dimensions: 1024
+      batch_size: 32
+      timeout_seconds: 30
+      max_retries: 3
+    engine: numpy_exact
+    strict_vector: true
+    top_k_sections: 30
+    top_k_pages: 8
+    max_sections_per_page: 2
+    min_similarity: 0.35
+    cache_max_knowledge_bases: 20
+    cache_revision_check_seconds: 5
+    external_api:
+      enabled: false
+      api_key: ""
+      previous_api_key: ""
+      max_query_chars: 1000
+      max_knowledge_bases: 20
+      max_results: 50
+      default_results: 10
+      include_content_max_chars: 12000
+      requests_per_minute: 60
 
 log_system:
   enabled: true

offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md

@@ -1,11 +1,12 @@
 # WeKnora Wiki 数据同步至 DeerFlow 并建立本地向量索引——实现级设计方案
 
-> 文档状态：待实现（Implementation Ready）  
+> 文档状态：首版已实现（Implemented，2026-08-23；2026-08-24 增补整库未向量化时的 Wiki-only 检索回退、知识库所有者手动向量化及全量 Excel 导出）
 > 编写日期：2026-08-21  
 > 适用代码库：当前 DeerFlow 工作区  
 > WeKnora 适配基线：v0.7.2  
 > 关联文档：`docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`  
 > 核心边界：**不修改 WeKnora 后端；WeKnora 仅作为 Wiki 生成源，DeerFlow 保存完整 Wiki 镜像、章节向量和检索索引，并承接全部内部与外部检索。**
+> 现行产品补充决策：内部检索只有在单个知识库整库完成向量化后才使用本地向量；未完成时只调用 WeKnora Wiki 页面检索接口并读取 Wiki 正文，绝不调用原始文档/切片检索。外部 API 继续严格使用本地向量。Wiki 管理页支持所有者手动触发整库向量化，并将全部 Wiki 导出为包含当前目录、完整父级目录和逐级目录列的 Excel。
 
 ---
 

offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 from urllib.parse import urlsplit, urlunsplit
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 
 def normalize_service_url(value: str | None) -> str:
@@ -37,18 +37,12 @@ class WeKnoraConnectionConfig(BaseModel):
     web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
     admin_email: str = Field(
         default="",
-        description=(
-            "Optional WeKnora administrator email used by DeerFlow's server-side "
-            "WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
-        ),
+        description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
     )
     admin_password: str = Field(
         default="",
         repr=False,
-        description=(
-            "Optional WeKnora administrator password used by DeerFlow's server-side "
-            "WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
-        ),
+        description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
     )
     model_config = ConfigDict(extra="ignore")
 
@@ -58,10 +52,107 @@ class WeKnoraConnectionConfig(BaseModel):
         return normalize_service_url(value)
 
 
+class LocalWikiEmbeddingConfig(BaseModel):
+    enabled: bool = True
+    provider: str = "openai"
+    model: str = "bge-m3"
+    base_url: str = ""
+    api_key: str = Field(default="", repr=False)
+    dimensions: int | None = Field(default=1024, ge=1, le=65536)
+    batch_size: int = Field(default=32, ge=1, le=512)
+    timeout_seconds: float = Field(default=30, gt=0, le=300)
+    max_retries: int = Field(default=3, ge=0, le=10)
+    model_config = ConfigDict(extra="ignore")
+
+    @field_validator("provider")
+    @classmethod
+    def _validate_provider(cls, value: str) -> str:
+        normalized = value.strip().lower()
+        if normalized not in {"openai", "openai_compatible"}:
+            raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
+        return normalized
+
+    @field_validator("base_url")
+    @classmethod
+    def _validate_url(cls, value: str) -> str:
+        return normalize_service_url(value)
+
+
+class LocalWikiExternalApiConfig(BaseModel):
+    enabled: bool = False
+    api_key: str = Field(default="", repr=False)
+    previous_api_key: str = Field(default="", repr=False)
+    max_query_chars: int = Field(default=1000, ge=1, le=10000)
+    max_knowledge_bases: int = Field(default=20, ge=1, le=100)
+    max_results: int = Field(default=50, ge=1, le=100)
+    default_results: int = Field(default=10, ge=1, le=100)
+    include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
+    requests_per_minute: int = Field(default=60, ge=1, le=100000)
+    model_config = ConfigDict(extra="ignore")
+
+    @model_validator(mode="after")
+    def _validate_enabled_key(self):
+        if self.enabled and not self.api_key.strip():
+            raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
+        if self.default_results > self.max_results:
+            raise ValueError("external_api.default_results must not exceed max_results")
+        return self
+
+
+class LocalWikiIndexConfig(BaseModel):
+    """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
+
+    enabled: bool = False
+    sync_enabled: bool = False
+    internal_search_enabled: bool = False
+    reference_ui_enabled: bool = False
+    sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
+    sync_page_size: int = Field(default=100, ge=1, le=500)
+    sync_concurrency: int = Field(default=3, ge=1, le=32)
+    sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
+    sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
+    sync_max_retries: int = Field(default=3, ge=0, le=10)
+    sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
+    chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
+    chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
+    chunk_min_chars: int = Field(default=80, ge=1, le=10000)
+    title_prefix_enabled: bool = True
+    embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
+    engine: str = "numpy_exact"
+    strict_vector: bool = True
+    top_k_sections: int = Field(default=30, ge=1, le=1000)
+    top_k_pages: int = Field(default=8, ge=1, le=100)
+    max_sections_per_page: int = Field(default=2, ge=1, le=20)
+    min_similarity: float = Field(default=0.35, ge=-1, le=1)
+    keyword_boost_enabled: bool = False
+    keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
+    cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
+    cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
+    external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
+    model_config = ConfigDict(extra="ignore")
+
+    @model_validator(mode="after")
+    def _validate_index(self):
+        if self.chunk_overlap_chars >= self.chunk_max_chars:
+            raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
+        if self.chunk_min_chars > self.chunk_max_chars:
+            raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
+        if self.top_k_pages > self.top_k_sections:
+            raise ValueError("top_k_pages must not exceed top_k_sections")
+        if self.engine != "numpy_exact":
+            raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
+        if self.enabled and not self.embedding.enabled:
+            raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
+        if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
+            raise ValueError("local Wiki index embedding base_url and model are required when enabled")
+        return self
+
+
 class LlmWikiConfig(BaseModel):
     """Top-level ``config.yaml`` section for conditional WeKnora mode."""
 
     weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
+    local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
     model_config = ConfigDict(extra="ignore")
 
 

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/__init__.py

@@ -0,0 +1,7 @@
+"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
+
+from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
+from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
+from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+
+__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/chunking.py

@@ -0,0 +1,108 @@
+"""Markdown-aware Wiki section chunking."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+
+_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
+_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
+_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+
+
+@dataclass(frozen=True)
+class WikiSection:
+    section_index: int
+    heading: str | None
+    content: str
+    embedding_text: str
+    content_hash: str
+
+
+def _clean_markdown(value: str) -> str:
+    text = value.replace("\r\n", "\n")
+    text = _FRONTMATTER_RE.sub("", text)
+    text = _SCRIPT_STYLE_RE.sub("", text)
+    text = _IMAGE_RE.sub(lambda match: match.group(1), text)
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
+    paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
+    chunks: list[str] = []
+    buffer = ""
+    for paragraph in paragraphs:
+        if len(buffer) + len(paragraph) + 2 <= max_chars:
+            buffer = f"{buffer}\n\n{paragraph}".strip()
+            continue
+        if buffer:
+            chunks.append(buffer)
+            buffer = ""
+        if len(paragraph) <= max_chars:
+            buffer = paragraph
+            continue
+        step = max(1, max_chars - overlap)
+        chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
+    if buffer:
+        chunks.append(buffer)
+    return chunks
+
+
+def chunk_wiki_page(
+    page: dict,
+    *,
+    max_chars: int,
+    overlap: int,
+    min_chars: int,
+    title_prefix_enabled: bool,
+) -> list[WikiSection]:
+    content = _clean_markdown(str(page.get("content_md") or ""))
+    heading_stack: list[str] = []
+    sections: list[tuple[str | None, str]] = []
+    current_heading: str | None = None
+    body: list[str] = []
+    for line in content.splitlines():
+        match = _HEADING_RE.match(line)
+        if not match:
+            body.append(line)
+            continue
+        if any(item.strip() for item in body):
+            sections.append((current_heading, "\n".join(body).strip()))
+        level = len(match.group(1))
+        heading_stack = heading_stack[: level - 1]
+        heading_stack.append(match.group(2).strip())
+        current_heading = " / ".join(heading_stack)
+        body = []
+    if any(item.strip() for item in body):
+        sections.append((current_heading, "\n".join(body).strip()))
+    if content and not sections:
+        sections.append((None, content))
+    summary = _clean_markdown(str(page.get("summary") or ""))
+    if summary:
+        sections.insert(0, ("页面摘要", summary))
+
+    prefix_lines = [f"页面：{page.get('title') or page.get('slug') or ''}"]
+    aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
+    if aliases:
+        prefix_lines.append(f"别名：{'、'.join(aliases)}")
+    if page.get("page_type"):
+        prefix_lines.append(f"类型：{page['page_type']}")
+    seen: set[str] = set()
+    result: list[WikiSection] = []
+    for heading, section_text in sections:
+        for chunk in _windows(section_text, max_chars, overlap):
+            normalized = re.sub(r"\s+", " ", chunk).strip()
+            if len(normalized) < min_chars and len(sections) > 1:
+                continue
+            digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+            if digest in seen:
+                continue
+            seen.add(digest)
+            context = list(prefix_lines) if title_prefix_enabled else []
+            if heading:
+                context.append(f"章节：{heading}")
+            embedding_text = "\n".join([*context, "", chunk]).strip()
+            result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
+    return result

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py

@@ -0,0 +1,107 @@
+"""Strict OpenAI-compatible embedding client for the Wiki index."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+from time import monotonic
+
+import httpx
+
+from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
+from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
+
+logger = logging.getLogger(__name__)
+
+
+class WikiEmbeddingError(RuntimeError):
+    def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
+        super().__init__(message)
+        self.category = category
+        self.retryable = retryable
+
+
+class StrictWikiEmbeddingClient:
+    def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
+        self.config = config
+        self._client = http_client
+        self._owns_client = http_client is None
+        self._resolved_dimensions = config.dimensions
+
+    @property
+    def dimensions(self) -> int | None:
+        return self._resolved_dimensions
+
+    @property
+    def fingerprint(self) -> str:
+        raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
+        return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+    async def close(self) -> None:
+        if self._owns_client and self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
+        return self._client
+
+    async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+        if not self.config.enabled or not self.config.base_url or not self.config.model:
+            raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
+        if not texts:
+            return []
+        vectors: list[list[float]] = []
+        batch_size = max(1, self.config.batch_size)
+        for start in range(0, len(texts), batch_size):
+            vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
+        if len(vectors) != len(texts):
+            raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
+        return vectors
+
+    async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
+        url = self.config.base_url.rstrip("/") + "/embeddings"
+        headers = {"Content-Type": "application/json"}
+        if self.config.api_key:
+            headers["Authorization"] = f"Bearer {self.config.api_key}"
+        payload: dict = {"model": self.config.model, "input": texts}
+        if self.config.dimensions:
+            payload["dimensions"] = self.config.dimensions
+        last_error: WikiEmbeddingError | None = None
+        for attempt in range(self.config.max_retries + 1):
+            started = monotonic()
+            try:
+                response = await (await self._get_client()).post(url, headers=headers, json=payload)
+                if response.status_code in {401, 403}:
+                    raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
+                if response.status_code == 429 or response.status_code >= 500:
+                    raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
+                response.raise_for_status()
+                data = response.json()
+                items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
+                if len(items) != len(texts):
+                    raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
+                result: list[list[float]] = []
+                for item in items:
+                    vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
+                    if self._resolved_dimensions is None:
+                        self._resolved_dimensions = int(vector.size)
+                    result.append(vector.astype(float).tolist())
+                logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
+                return result
+            except WikiEmbeddingError as exc:
+                last_error = exc
+            except httpx.TimeoutException:
+                last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
+            except (httpx.HTTPError, ValueError, TypeError) as exc:
+                last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
+            if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
+                raise last_error
+            await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
+        raise last_error or WikiEmbeddingError("Embedding request failed")
+
+    async def embed_query(self, text: str) -> list[float]:
+        result = await self.embed_texts([text])
+        return result[0]

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/normalize.py

@@ -0,0 +1,86 @@
+"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import datetime
+from typing import Any
+from urllib.parse import unquote
+
+
+def normalize_slug(value: Any) -> str:
+    slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
+    slug = re.sub(r"/{2,}", "/", slug)
+    if not slug:
+        raise ValueError("Wiki page slug is empty")
+    return slug
+
+
+def slug_hash(slug: str) -> str:
+    return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
+
+
+def _list(value: Any) -> list:
+    if isinstance(value, list):
+        return value
+    if isinstance(value, tuple):
+        return list(value)
+    if isinstance(value, str) and value.strip():
+        return [value.strip()]
+    return []
+
+
+def _datetime(value: Any) -> datetime | None:
+    if isinstance(value, datetime):
+        return value
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
+    slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
+    aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
+    category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
+    title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
+    summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
+    content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
+    status = str(remote.get("status") or "published").strip().lower()
+    if status not in {"draft", "published", "archived"}:
+        status = "published"
+    hash_payload = {
+        "title": title,
+        "aliases": aliases,
+        "summary": summary,
+        "content_md": content,
+        "page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
+        "status": status,
+    }
+    content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
+    return {
+        "knowledge_base_mapping_id": mapping_id,
+        "remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
+        "slug": slug,
+        "slug_hash": slug_hash(slug),
+        "title": title,
+        "summary": summary,
+        "content_md": content,
+        "page_type": hash_payload["page_type"],
+        "status": status,
+        "aliases": aliases,
+        "category_path": category_path,
+        "parent_slug": str(remote.get("parent_slug") or "").strip() or None,
+        "wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
+        "source_refs": _list(remote.get("source_refs") or remote.get("sources")),
+        "chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
+        "page_version": int(remote.get("page_version") or remote.get("version") or 0),
+        "content_hash": content_hash,
+        "remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
+        "remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
+    }

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/retrieval.py

@@ -0,0 +1,248 @@
+"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
+
+The fallback deliberately calls WeKnora's ``/wiki/pages`` API.  It never calls
+the raw knowledge/chunk search endpoint, so callers always receive processed
+Wiki pages even while a whole knowledge base is awaiting vectorization.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from urllib.parse import urlencode
+
+from deerflow.integrations.weknora.client import WeKnoraClient
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+
+def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
+    """Return whether a completed full scan produced one coherent active index."""
+
+    if not state or not fingerprint:
+        return False
+    if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
+        return False
+    if str(state.get("embedding_fingerprint") or "") != fingerprint:
+        return False
+    local_count = int(state.get("local_page_count") or 0)
+    ready_count = int(state.get("ready_page_count") or 0)
+    failed_count = int(state.get("failed_page_count") or 0)
+    vector_count = int(state.get("vector_count") or 0)
+    if failed_count or ready_count != local_count:
+        return False
+    return local_count == 0 or vector_count > 0
+
+
+class WikiRetrievalService:
+    """Search complete indexes locally and incomplete indexes through Wiki API."""
+
+    def __init__(
+        self,
+        store: LlmWikiIndexStore,
+        *,
+        vector_search: WikiVectorSearchService | None,
+        client: WeKnoraClient | None,
+    ) -> None:
+        self.store = store
+        self.vector_search = vector_search
+        self.client = client
+
+    async def search(
+        self,
+        query: str,
+        mappings: list[dict[str, Any]],
+        *,
+        top_k_pages: int = 8,
+        include_drafts: bool = False,
+    ) -> dict[str, Any]:
+        normalized_query = query.strip()
+        if not normalized_query:
+            raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
+        if not mappings:
+            return {
+                "query": normalized_query,
+                "mode": "wiki_api",
+                "results": [],
+                "knowledge_base_count": 0,
+                "partial": False,
+                "warnings": [],
+                "fallback_knowledge_base_ids": [],
+            }
+
+        vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
+        results: list[dict[str, Any]] = []
+        warnings: list[str] = []
+        vector_used = False
+        fallback_used = False
+
+        if vector_mappings and self.vector_search is not None:
+            try:
+                payload = await self.vector_search.search(
+                    normalized_query,
+                    vector_mappings,
+                    top_k_pages=max(top_k_pages, len(vector_mappings)),
+                    include_drafts=include_drafts,
+                )
+                results.extend(payload.get("results") or [])
+                warnings.extend(str(item) for item in payload.get("warnings") or [])
+                vector_used = True
+            except WikiIndexError as exc:
+                # A status row can become stale between classification and
+                # matrix loading. Wiki API remains a safe Wiki-only fallback.
+                warnings.append(f"local_vector:{exc.code}")
+                fallback_mappings.extend(vector_mappings)
+                vector_mappings = []
+
+        fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
+        if fallback_mappings:
+            if self.client is None:
+                warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
+            else:
+                batches = await asyncio.gather(
+                    *(
+                        self._search_remote_mapping(
+                            normalized_query,
+                            mapping,
+                            limit=max(top_k_pages, 1),
+                            include_drafts=include_drafts,
+                        )
+                        for mapping in fallback_mappings
+                    ),
+                    return_exceptions=True,
+                )
+                for mapping, batch in zip(fallback_mappings, batches, strict=True):
+                    if isinstance(batch, Exception):
+                        warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
+                    else:
+                        results.extend(batch)
+                        fallback_used = True
+
+        if warnings and not results and not vector_used and not fallback_used:
+            raise WikiIndexError(
+                "WIKI_API_UNAVAILABLE",
+                "Wiki page retrieval is unavailable",
+                status_code=502 if self.client is not None else 503,
+            )
+
+        # De-duplicate by the public Wiki citation identity. Vector scores and
+        # Wiki-API rank scores are both descending relevance hints.
+        unique: dict[tuple[str, str], dict[str, Any]] = {}
+        for item in results:
+            key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
+            current = unique.get(key)
+            if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
+                unique[key] = item
+        ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
+        mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
+        fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
+        return {
+            "query": normalized_query,
+            "mode": mode,
+            "results": ranked[:top_k_pages],
+            "knowledge_base_count": len(mappings),
+            "index_fingerprint": fingerprint,
+            "partial": bool(warnings),
+            "warnings": warnings,
+            "fallback_knowledge_base_ids": fallback_ids,
+        }
+
+    async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+        if self.vector_search is None:
+            return [], list(mappings)
+        mapping_ids = [str(mapping["id"]) for mapping in mappings]
+        states = {
+            str(state["knowledge_base_mapping_id"]): state
+            for state in await self.store.list_index_status(mapping_ids)
+        }
+        fingerprint = self.vector_search.embedding.fingerprint
+        vector_mappings = [
+            mapping
+            for mapping in mappings
+            if mapping.get("wiki_index_enabled", True)
+            and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
+        ]
+        ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
+        return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
+
+    async def _search_remote_mapping(
+        self,
+        query: str,
+        mapping: dict[str, Any],
+        *,
+        limit: int,
+        include_drafts: bool,
+    ) -> list[dict[str, Any]]:
+        if self.client is None:
+            return []
+        remote_id = str(mapping["weknora_id"])
+        listing = await self.client.list_wiki_pages(
+            remote_id,
+            page=1,
+            page_size=min(max(limit * 3, 20), 100),
+            query=query,
+        )
+        summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
+        slugs = list(
+            dict.fromkeys(
+                str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
+                for item in summaries
+                if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
+            )
+        )[: max(limit * 2, limit)]
+        details = await asyncio.gather(
+            *(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
+            return_exceptions=True,
+        )
+        results: list[dict[str, Any]] = []
+        for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
+            if isinstance(detail, Exception):
+                continue
+            try:
+                page = normalize_wiki_page(str(mapping["id"]), detail)
+            except (TypeError, ValueError):
+                continue
+            if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
+                continue
+            params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
+            content = str(page.get("content_md") or "")
+            score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
+            results.append(
+                {
+                    "kind": "wiki_page",
+                    "id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
+                    "knowledge_base_id": mapping["id"],
+                    "knowledge_base_name": mapping.get("name") or "",
+                    "wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
+                    "wiki_slug": slug,
+                    "title": page.get("title") or slug,
+                    "summary": page.get("summary") or "",
+                    "page_type": page.get("page_type") or "article",
+                    "page_version": int(page.get("page_version") or 0),
+                    "updated_at": page.get("remote_updated_at"),
+                    "score": score,
+                    "content": content,
+                    "snippet": content[:360],
+                    "matched_heading": None,
+                    "matched_sections": [
+                        {
+                            "section_index": 0,
+                            "heading": None,
+                            "content": content,
+                            "score": score,
+                        }
+                    ],
+                    "url": f"/weknora-source-preview?{params}",
+                    "source": mapping.get("name") or "LLMWiki",
+                    "type": "llmwiki_wiki",
+                    "mode": "citation",
+                    "retrieval_mode": "wiki_api",
+                }
+            )
+            if len(results) >= limit:
+                break
+        return results
+
+
+__all__ = ["WikiRetrievalService", "is_fully_vectorized"]

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/runtime.py

@@ -0,0 +1,48 @@
+"""App-to-harness runtime injection for local Wiki index services."""
+
+from __future__ import annotations
+
+from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
+from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
+from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+_store: LlmWikiIndexStore | None = None
+_search: WikiVectorSearchService | None = None
+_sync: WikiSyncService | None = None
+_retrieval: WikiRetrievalService | None = None
+
+
+def register_local_wiki_index_runtime(
+    *,
+    store: LlmWikiIndexStore | None,
+    search: WikiVectorSearchService | None,
+    sync: WikiSyncService | None,
+    retrieval: WikiRetrievalService | None = None,
+) -> None:
+    global _store, _search, _sync, _retrieval
+    _store, _search, _sync, _retrieval = store, search, sync, retrieval
+
+
+def get_local_wiki_index_store() -> LlmWikiIndexStore:
+    if _store is None:
+        raise RuntimeError("Local Wiki index store is not initialized")
+    return _store
+
+
+def get_local_wiki_search_service() -> WikiVectorSearchService:
+    if _search is None:
+        raise RuntimeError("Local Wiki vector search is not initialized")
+    return _search
+
+
+def get_local_wiki_sync_service() -> WikiSyncService:
+    if _sync is None:
+        raise RuntimeError("Local Wiki sync service is not initialized")
+    return _sync
+
+
+def get_local_wiki_retrieval_service() -> WikiRetrievalService:
+    if _retrieval is None:
+        raise RuntimeError("Wiki-only retrieval service is not initialized")
+    return _retrieval

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py

@@ -0,0 +1,176 @@
+"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import re
+from collections import OrderedDict, defaultdict
+from time import monotonic
+from typing import Any
+from urllib.parse import urlencode
+
+import numpy as np
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
+from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
+
+
+class WikiIndexError(RuntimeError):
+    def __init__(self, code: str, message: str, *, status_code: int) -> None:
+        super().__init__(message)
+        self.code = code
+        self.status_code = status_code
+
+
+class WikiVectorSearchService:
+    def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
+        self.config = config
+        self.embedding = embedding
+        self.cache = cache
+        self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
+
+    async def search(
+        self,
+        query: str,
+        mappings: list[dict[str, Any]],
+        *,
+        top_k_pages: int | None = None,
+        include_drafts: bool = False,
+        external: bool = False,
+    ) -> dict[str, Any]:
+        normalized_query = query.strip()
+        if not normalized_query:
+            raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
+        if not mappings:
+            return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
+        try:
+            query_vector = await self._embed_query(normalized_query)
+        except (WikiEmbeddingError, ValueError) as exc:
+            raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
+        fingerprint = self.embedding.fingerprint
+        tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
+        batches = await asyncio.gather(*tasks, return_exceptions=True)
+        candidates: list[dict[str, Any]] = []
+        warnings: list[str] = []
+        for mapping, batch in zip(mappings, batches, strict=True):
+            if isinstance(batch, Exception):
+                warnings.append(f"{mapping.get('id')}: {batch}")
+            else:
+                candidates.extend(batch)
+        if warnings and not candidates and len(warnings) == len(mappings):
+            if any("FINGERPRINT" in item for item in warnings):
+                code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
+            elif any("rebuild" in item.lower() for item in warnings):
+                code = "WIKI_INDEX_REBUILDING"
+            else:
+                code = "WIKI_INDEX_NOT_READY"
+            raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
+        candidates.sort(key=lambda item: item["score"], reverse=True)
+        grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
+        for candidate in candidates:
+            page = candidate["page"]
+            key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
+            if len(grouped[key]) < self.config.max_sections_per_page:
+                grouped[key].append(candidate)
+        results: list[dict[str, Any]] = []
+        for sections in grouped.values():
+            first = sections[0]
+            page = first["page"]
+            mapping = first["mapping"]
+            second_score = sections[1]["score"] if len(sections) > 1 else 0.0
+            page_score = first["score"] + (0.05 * second_score if second_score else 0)
+            params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
+            results.append(
+                {
+                    "kind": "wiki_page",
+                    "id": page["id"],
+                    "knowledge_base_id": mapping["id"],
+                    "knowledge_base_name": mapping.get("name") or "",
+                    "wiki_page_id": page["id"],
+                    "wiki_slug": page["slug"],
+                    "title": page.get("title") or page["slug"],
+                    "summary": page.get("summary") or "",
+                    "page_type": page.get("page_type") or "article",
+                    "page_version": int(page.get("page_version") or 0),
+                    "updated_at": page.get("remote_updated_at") or page.get("updated_at"),
+                    "score": float(page_score),
+                    "content": "\n\n".join(str(item["section_content"]) for item in sections),
+                    "snippet": str(sections[0]["section_content"])[:360],
+                    "matched_heading": sections[0].get("heading"),
+                    "matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
+                    "url": f"/weknora-source-preview?{params}",
+                    "source": mapping.get("name") or "LLMWiki",
+                    "type": "llmwiki_wiki",
+                    "mode": "citation",
+                }
+            )
+        results.sort(key=lambda item: item["score"], reverse=True)
+        limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
+        return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
+
+    async def _embed_query(self, query: str) -> np.ndarray:
+        digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
+        key = f"{self.embedding.fingerprint}:{digest}"
+        cached = self._query_vectors.get(key)
+        if cached is not None and monotonic() - cached[0] <= 300:
+            self._query_vectors.move_to_end(key)
+            return cached[1]
+        vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
+        key = f"{self.embedding.fingerprint}:{digest}"
+        self._query_vectors[key] = (monotonic(), vector)
+        self._query_vectors.move_to_end(key)
+        while len(self._query_vectors) > 256:
+            self._query_vectors.popitem(last=False)
+        return vector
+
+    async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
+        mapping_id = str(mapping["id"])
+        try:
+            snapshot = await self.cache.get(mapping_id, fingerprint)
+        except ValueError as exc:
+            await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
+            self.cache.invalidate(mapping_id)
+            raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
+        if snapshot.matrix.size == 0:
+            return []
+        if snapshot.dimensions != vector.size:
+            raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
+        eligible = np.asarray(
+            [
+                index
+                for index, row in enumerate(snapshot.rows)
+                if row["page"].get("status") != "draft" or (include_drafts and not external)
+            ],
+            dtype=int,
+        )
+        if eligible.size == 0:
+            return []
+        indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
+        terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
+        candidates = []
+        for index, score in zip(indices, scores, strict=True):
+            row = snapshot.rows[int(index)]
+            page = row["page"]
+            vector_score = float(score)
+            if vector_score < self.config.min_similarity:
+                continue
+            final_score = vector_score
+            if self.config.keyword_boost_enabled and terms:
+                haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
+                keyword_score = sum(term in haystack for term in terms) / len(terms)
+                weight = self.config.keyword_boost_weight
+                final_score = (1 - weight) * vector_score + weight * keyword_score
+            candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
+        return candidates
+
+    def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+        scores = snapshot.matrix[eligible] @ vector
+        count = min(self.config.top_k_sections, scores.size)
+        if count <= 0:
+            return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
+        positions = np.argpartition(scores, -count)[-count:]
+        positions = positions[np.argsort(scores[positions])[::-1]]
+        return eligible[positions], scores[positions]

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/sync.py

@@ -0,0 +1,173 @@
+"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import platform
+from typing import Any
+from uuid import uuid4
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.client import WeKnoraClient
+from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
+from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+logger = logging.getLogger(__name__)
+
+
+class WikiSyncLeaseLost(RuntimeError):
+    pass
+
+
+def _safe_error(exc: BaseException) -> str:
+    return f"{type(exc).__name__}: {str(exc)[:1000]}"
+
+
+class WikiSyncService:
+    def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
+        self.config = config
+        self.store = store
+        self.embedding = embedding
+        self.cache = cache
+        self.client = client
+        self.owner = f"{platform.node()}:{uuid4()}"
+
+    async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
+        mapping_id = str(mapping["id"])
+        sync_id = str(uuid4())
+        if self.embedding.dimensions is None:
+            # Auto-dimension configurations need one successful response to
+            # produce a stable fingerprint before lease/rebuild decisions.
+            await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
+        fingerprint = self.embedding.fingerprint
+        revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
+        if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
+            await self.store.mark_rebuild_required([mapping_id], fingerprint)
+            self.cache.invalidate(mapping_id)
+            prior_state = "rebuild_required"
+        rebuilding = prior_state == "rebuild_required"
+        acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
+        if not acquired:
+            return {"mapping_id": mapping_id, "status": "already_running"}
+        lease_lost = asyncio.Event()
+
+        async def _lease_heartbeat() -> None:
+            interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
+            while True:
+                await asyncio.sleep(interval)
+                try:
+                    renewed = await self.store.renew_sync_lease(
+                        mapping_id,
+                        owner=self.owner,
+                        lease_seconds=self.config.sync_lease_seconds,
+                    )
+                except Exception:
+                    renewed = False
+                    logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
+                if not renewed:
+                    lease_lost.set()
+                    return
+
+        heartbeat = asyncio.create_task(_lease_heartbeat())
+
+        def _ensure_lease() -> None:
+            if lease_lost.is_set():
+                raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
+
+        remote_count = 0
+        changed = 0
+        failed = 0
+        try:
+            remote_id = str(mapping["weknora_id"])
+            await self.client.get_knowledge_base(remote_id)
+            page_number = 1
+            while True:
+                batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
+                pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
+                remote_count += len(pages)
+                for summary in pages:
+                    try:
+                        _ensure_lease()
+                        slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
+                        remote_page = await self.client.get_wiki_page(remote_id, slug)
+                        page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
+                        existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
+                        active_fingerprint = self.embedding.fingerprint
+                        if (
+                            not force
+                            and existing
+                            and existing.get("content_hash") == page["content_hash"]
+                            and existing.get("indexed_fingerprint") == active_fingerprint
+                            and existing.get("index_status") == "ready"
+                            and not existing.get("is_remote_deleted")
+                        ):
+                            await self.store.touch_page(str(existing["id"]), sync_id)
+                            continue
+                        sections = (
+                            []
+                            if page["status"] == "archived"
+                            else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
+                        )
+                        embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
+                        fingerprint = self.embedding.fingerprint
+                        vectors = []
+                        for section, vector in zip(sections, embeddings, strict=True):
+                            blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
+                            vectors.append(
+                                {"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
+                            )
+                        _ensure_lease()
+                        await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
+                        self.cache.invalidate(mapping_id)
+                        changed += 1
+                    except Exception as exc:
+                        if isinstance(exc, WikiSyncLeaseLost):
+                            raise
+                        failed += 1
+                        try:
+                            fallback = normalize_wiki_page(mapping_id, summary)
+                            await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
+                        except Exception as record_exc:
+                            logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
+                            raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
+                        logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
+                _ensure_lease()
+                total_pages = int(batch.get("total_pages") or 1)
+                if page_number >= total_pages:
+                    break
+                page_number += 1
+            _ensure_lease()
+            deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
+            if deleted:
+                self.cache.invalidate(mapping_id)
+            await self.store.finish_sync(
+                mapping_id,
+                owner=self.owner,
+                sync_id=sync_id,
+                success=not (rebuilding and failed),
+                remote_page_count=remote_count,
+                error=f"{failed} page(s) failed" if failed else None,
+            )
+            return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
+        except asyncio.CancelledError:
+            await self.store.finish_sync(
+                mapping_id,
+                owner=self.owner,
+                sync_id=sync_id,
+                success=False,
+                remote_page_count=remote_count,
+                error="Sync cancelled",
+            )
+            raise
+        except Exception as exc:
+            await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
+            logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
+            raise
+        finally:
+            heartbeat.cancel()
+            await asyncio.gather(heartbeat, return_exceptions=True)

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_cache.py

@@ -0,0 +1,86 @@
+"""Per-knowledge-base LRU snapshots for NumPy exact search."""
+
+from __future__ import annotations
+
+import asyncio
+from collections import OrderedDict
+from dataclasses import dataclass
+from time import monotonic
+from typing import Any
+
+import numpy as np
+
+from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+
+@dataclass(frozen=True)
+class WikiVectorSnapshot:
+    mapping_id: str
+    index_revision: int
+    embedding_fingerprint: str
+    dimensions: int
+    matrix: np.ndarray
+    rows: list[dict[str, Any]]
+
+
+class WikiVectorCache:
+    def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
+        self.store = store
+        self.max_knowledge_bases = max_knowledge_bases
+        self.revision_check_seconds = revision_check_seconds
+        self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
+        self._checked_at: dict[str, float] = {}
+        self._locks: dict[str, asyncio.Lock] = {}
+
+    def invalidate(self, mapping_id: str) -> None:
+        self._snapshots.pop(mapping_id, None)
+        self._checked_at.pop(mapping_id, None)
+
+    def clear(self) -> None:
+        self._snapshots.clear()
+        self._checked_at.clear()
+
+    async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
+        cached = self._snapshots.get(mapping_id)
+        now = monotonic()
+        if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
+            self._snapshots.move_to_end(mapping_id)
+            return cached
+        lock = self._locks.setdefault(mapping_id, asyncio.Lock())
+        async with lock:
+            revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
+            self._checked_at[mapping_id] = monotonic()
+            cached = self._snapshots.get(mapping_id)
+            if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
+                raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
+            if active_fingerprint != fingerprint:
+                raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
+            if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
+                self._snapshots.move_to_end(mapping_id)
+                return cached
+            raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
+            snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
+            self._snapshots[mapping_id] = snapshot
+            self._snapshots.move_to_end(mapping_id)
+            while len(self._snapshots) > self.max_knowledge_bases:
+                evicted, _ = self._snapshots.popitem(last=False)
+                self._checked_at.pop(evicted, None)
+            return snapshot
+
+    @staticmethod
+    def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
+        rows = list(raw.get("rows") or [])
+        if not rows:
+            return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
+        dimensions = int(rows[0]["embedding_dimensions"])
+        vectors = []
+        valid_rows = []
+        for row in rows:
+            if int(row["embedding_dimensions"]) != dimensions:
+                raise ValueError("Mixed vector dimensions in one Wiki snapshot")
+            vectors.append(decode_vector(row["vector_blob"], dimensions))
+            valid_rows.append(row)
+        matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
+        matrix.setflags(write=False)
+        return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)

offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_codec.py

@@ -0,0 +1,33 @@
+"""Validated little-endian normalized float32 Wiki vector codec."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
+    vector = np.asarray(values, dtype=np.float32)
+    if vector.ndim != 1 or vector.size == 0:
+        raise ValueError("Embedding vector must be a non-empty one-dimensional array")
+    if dimensions is not None and vector.size != dimensions:
+        raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
+    if not np.isfinite(vector).all():
+        raise ValueError("Embedding vector contains NaN or infinity")
+    norm = float(np.linalg.norm(vector))
+    if not np.isfinite(norm) or norm <= 0:
+        raise ValueError("Embedding vector has zero norm")
+    return np.asarray(vector / norm, dtype="<f4")
+
+
+def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
+    vector = normalize_vector(values, dimensions=dimensions)
+    return vector.tobytes(order="C"), int(vector.size)
+
+
+def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
+    if len(blob) != dimensions * 4:
+        raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
+    vector = np.frombuffer(blob, dtype="<f4")
+    if not np.isfinite(vector).all():
+        raise ValueError("Stored vector contains NaN or infinity")
+    return vector.copy()

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/base.py

@@ -22,7 +22,15 @@ class LlmWikiStore(ABC):
     async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
 
     @abstractmethod
-    async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None: ...
+    async def update_mapping(
+        self,
+        mapping_id: str,
+        *,
+        name: str | None = None,
+        description: str | None = None,
+        wiki_index_enabled: bool | None = None,
+        external_search_enabled: bool | None = None,
+    ) -> dict[str, Any] | None: ...
 
     @abstractmethod
     async def adopt_conversation_deposit_mapping(

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/memory.py

@@ -33,6 +33,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
             "published_by": None,
             "reviewed_by": None,
             "published_at": None,
+            "wiki_index_enabled": True,
+            "external_search_enabled": False,
+            "external_search_updated_at": None,
             "created_at": now,
             "updated_at": now,
         }
@@ -65,7 +68,15 @@ class MemoryLlmWikiStore(LlmWikiStore):
                 return dict(row)
         return None
 
-    async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
+    async def update_mapping(
+        self,
+        mapping_id: str,
+        *,
+        name: str | None = None,
+        description: str | None = None,
+        wiki_index_enabled: bool | None = None,
+        external_search_enabled: bool | None = None,
+    ) -> dict[str, Any] | None:
         row = self._rows.get(mapping_id)
         if row is None:
             return None
@@ -73,6 +84,11 @@ class MemoryLlmWikiStore(LlmWikiStore):
             row["name"] = name
         if description is not None:
             row["description"] = description
+        if wiki_index_enabled is not None:
+            row["wiki_index_enabled"] = wiki_index_enabled
+        if external_search_enabled is not None:
+            row["external_search_enabled"] = external_search_enabled
+            row["external_search_updated_at"] = _now()
         row["updated_at"] = _now()
         return dict(row)
 
@@ -143,11 +159,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
 
     async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
         for row in self._rows.values():
-            if (
-                row.get("owner_user_id") == "system"
-                and row.get("name") == "对话沉淀"
-                and row.get("publication_status") == "published"
-            ):
+            if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
                 return dict(row)
         return None
 

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/model.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from datetime import UTC, datetime
 
-from sqlalchemy import String, UniqueConstraint
+from sqlalchemy import Boolean, String, UniqueConstraint, false, true
 from sqlalchemy.orm import Mapped, mapped_column
 
 from deerflow.persistence.base import Base
@@ -24,6 +24,9 @@ class LlmWikiKnowledgeBaseRow(Base):
     published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
     reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
     published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
+    external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
+    external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
     created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
     updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
 

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/sql.py

@@ -19,7 +19,7 @@ def _now() -> datetime:
 
 def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
     data = row.to_dict()
-    for key in ("published_at", "created_at", "updated_at"):
+    for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
         value = data.get(key)
         if isinstance(value, datetime):
             data[key] = value.isoformat()
@@ -82,7 +82,15 @@ class LlmWikiRepository(LlmWikiStore):
             row = (await session.execute(stmt)).scalar_one_or_none()
             return _to_dict(row) if row else None
 
-    async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
+    async def update_mapping(
+        self,
+        mapping_id: str,
+        *,
+        name: str | None = None,
+        description: str | None = None,
+        wiki_index_enabled: bool | None = None,
+        external_search_enabled: bool | None = None,
+    ) -> dict[str, Any] | None:
         async with self._sf() as session:
             row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
             if row is None:
@@ -91,6 +99,11 @@ class LlmWikiRepository(LlmWikiStore):
                 row.name = name
             if description is not None:
                 row.description = description
+            if wiki_index_enabled is not None:
+                row.wiki_index_enabled = wiki_index_enabled
+            if external_search_enabled is not None:
+                row.external_search_enabled = external_search_enabled
+                row.external_search_updated_at = _now()
             row.updated_at = _now()
             await session.commit()
             return _to_dict(row)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py

@@ -0,0 +1,21 @@
+"""Persistence facade for the DeerFlow-local Wiki vector index."""
+
+from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
+from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
+from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
+
+
+def make_llmwiki_index_store(session_factory):
+    return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
+
+
+__all__ = [
+    "LlmWikiIndexStore",
+    "MemoryLlmWikiIndexStore",
+    "SqlLlmWikiIndexStore",
+    "LlmWikiWikiPageRow",
+    "LlmWikiWikiVectorRow",
+    "LlmWikiWikiSyncStateRow",
+    "make_llmwiki_index_store",
+]

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/base.py

@@ -0,0 +1,54 @@
+"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class LlmWikiIndexStore(ABC):
+    @abstractmethod
+    async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
+
+    @abstractmethod
+    async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
+
+    @abstractmethod
+    async def touch_page(self, page_id: str, sync_id: str) -> None: ...
+
+    @abstractmethod
+    async def replace_page_vectors(
+        self,
+        *,
+        page: dict[str, Any],
+        vectors: list[dict[str, Any]],
+        fingerprint: str,
+        sync_id: str,
+    ) -> dict[str, Any]: ...
+
+    @abstractmethod
+    async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
+
+    @abstractmethod
+    async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
+
+    @abstractmethod
+    async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
+
+    @abstractmethod
+    async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
+
+    @abstractmethod
+    async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
+
+    @abstractmethod
+    async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
+
+    @abstractmethod
+    async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
+
+    @abstractmethod
+    async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
+
+    @abstractmethod
+    async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/memory.py

@@ -0,0 +1,160 @@
+"""In-memory local Wiki index store used by tests and DB-less mode."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import uuid4
+
+from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
+    def __init__(self) -> None:
+        self.pages: dict[str, dict[str, Any]] = {}
+        self.vectors: dict[str, list[dict[str, Any]]] = {}
+        self.states: dict[str, dict[str, Any]] = {}
+
+    def _state(self, mapping_id: str) -> dict[str, Any]:
+        return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
+
+    async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
+        row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
+        return dict(row) if row else None
+
+    async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
+        row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
+        return dict(row) if row else None
+
+    async def touch_page(self, page_id: str, sync_id: str) -> None:
+        if page_id in self.pages:
+            self.pages[page_id]["last_seen_sync_id"] = sync_id
+            self.pages[page_id]["last_error"] = None
+
+    async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
+        existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
+        page_id = existing["id"] if existing else str(uuid4())
+        now = _now().isoformat()
+        row = {
+            **(existing or {}),
+            **page,
+            "id": page_id,
+            "index_status": "ready",
+            "indexed_fingerprint": fingerprint,
+            "indexed_at": now,
+            "last_error": None,
+            "is_remote_deleted": False,
+            "last_seen_sync_id": sync_id,
+            "created_at": (existing or {}).get("created_at", now),
+            "updated_at": now,
+        }
+        self.pages[page_id] = row
+        self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
+        state = self._state(page["knowledge_base_mapping_id"])
+        state["index_revision"] += 1
+        state["embedding_fingerprint"] = fingerprint
+        return dict(row)
+
+    async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
+        existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
+        if existing:
+            self.pages[existing["id"]].update(
+                {
+                    "last_seen_sync_id": sync_id,
+                    "last_error": error,
+                    # Preserve the last atomically completed generation.
+                    "index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
+                }
+            )
+            return
+        page_id = str(uuid4())
+        self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
+
+    async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
+        changed = 0
+        for page in self.pages.values():
+            if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
+                page.update({"is_remote_deleted": True, "index_status": "deleted"})
+                self.vectors.pop(page["id"], None)
+                changed += 1
+        if changed:
+            self._state(mapping_id)["index_revision"] += 1
+        return changed
+
+    async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
+        rows = []
+        for page_id, vectors in self.vectors.items():
+            page = self.pages[page_id]
+            if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
+                continue
+            rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
+        state = self._state(mapping_id)
+        return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
+
+    async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
+        state = self._state(mapping_id)
+        return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
+
+    async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
+        state = self._state(mapping_id)
+        now = _now()
+        expires = state.get("lease_expires_at")
+        if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
+            return False
+        state.update(
+            {
+                "state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
+                "active_sync_id": sync_id,
+                "lease_owner": owner,
+                "lease_expires_at": now + timedelta(seconds=lease_seconds),
+                "last_started_at": now,
+            }
+        )
+        return True
+
+    async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
+        state = self._state(mapping_id)
+        if state.get("lease_owner") != owner:
+            return False
+        state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
+        return True
+
+    async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
+        state = self._state(mapping_id)
+        if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
+            return
+        pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
+        was_rebuilding = state.get("state") == "rebuilding"
+        state.update(
+            {
+                "state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
+                "lease_owner": None,
+                "lease_expires_at": None,
+                "active_sync_id": None,
+                "last_error": error,
+                "remote_page_count": remote_page_count,
+                "local_page_count": len(pages),
+                "ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
+                "failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
+                "vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
+                "last_completed_at": _now() if success and not error else state.get("last_completed_at"),
+                "last_success_at": _now() if success else state.get("last_success_at"),
+            }
+        )
+
+    async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
+        wanted = set(mapping_ids or [])
+        return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
+
+    async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
+        for mapping_id in mapping_ids:
+            state = self._state(mapping_id)
+            state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
+            for page in self.pages.values():
+                if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
+                    page["index_status"] = "stale"
+        return len(mapping_ids)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/model.py

@@ -0,0 +1,94 @@
+"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
+from sqlalchemy.orm import Mapped, mapped_column
+
+from deerflow.persistence.base import Base
+from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class LlmWikiWikiPageRow(Base):
+    __tablename__ = "llmwiki_wiki_pages"
+    __table_args__ = (
+        UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
+        Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
+    )
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
+    remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
+    slug: Mapped[str] = mapped_column(String(1024), nullable=False)
+    slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
+    summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
+    content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
+    page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
+    aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+    category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+    parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+    chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+    page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
+    content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
+    index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
+    indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
+    last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
+
+
+class LlmWikiWikiVectorRow(Base):
+    __tablename__ = "llmwiki_wiki_vectors"
+    __table_args__ = (
+        UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
+        Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
+    )
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
+    knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
+    section_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
+    content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
+    vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
+
+
+class LlmWikiWikiSyncStateRow(Base):
+    __tablename__ = "llmwiki_wiki_sync_states"
+
+    knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
+    state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
+    active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py

@@ -0,0 +1,295 @@
+"""SQLAlchemy implementation of the local Wiki index store."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import case, delete, func, or_, select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _dict(row) -> dict[str, Any]:
+    value = row.to_dict()
+    for key, item in tuple(value.items()):
+        if isinstance(item, datetime):
+            value[key] = item.isoformat()
+    return value
+
+
+class SqlLlmWikiIndexStore(LlmWikiIndexStore):
+    def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
+        self._sf = session_factory
+
+    async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
+        row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+        if row is None:
+            row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
+            session.add(row)
+            await session.flush()
+        return row
+
+    async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
+        stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
+        async with self._sf() as session:
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            return _dict(row) if row else None
+
+    async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
+        stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
+        async with self._sf() as session:
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            return _dict(row) if row else None
+
+    async def touch_page(self, page_id: str, sync_id: str) -> None:
+        async with self._sf() as session:
+            await session.execute(
+                update(LlmWikiWikiPageRow)
+                .where(LlmWikiWikiPageRow.id == page_id)
+                .values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
+            )
+            await session.commit()
+
+    async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
+        mapping_id = str(page["knowledge_base_mapping_id"])
+        async with self._sf() as session:
+            stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            if row is not None and row.slug != page["slug"]:
+                raise ValueError("Wiki slug hash collision detected")
+            now = _now()
+            if row is None:
+                row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
+                session.add(row)
+            else:
+                for key, value in page.items():
+                    setattr(row, key, value)
+            row.index_status = "ready"
+            row.indexed_fingerprint = fingerprint
+            row.indexed_at = now
+            row.last_error = None
+            row.is_remote_deleted = False
+            row.last_seen_sync_id = sync_id
+            row.updated_at = now
+            await session.flush()
+            await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
+            session.add_all(
+                [
+                    LlmWikiWikiVectorRow(
+                        id=str(uuid4()),
+                        page_id=row.id,
+                        knowledge_base_mapping_id=mapping_id,
+                        section_index=int(item["section_index"]),
+                        heading=item.get("heading"),
+                        section_content=str(item["section_content"]),
+                        content_hash=str(item["content_hash"]),
+                        embedding_fingerprint=fingerprint,
+                        embedding_dimensions=int(item["embedding_dimensions"]),
+                        vector_blob=bytes(item["vector_blob"]),
+                        created_at=now,
+                    )
+                    for item in vectors
+                ]
+            )
+            state = await self._ensure_state(session, mapping_id)
+            state.index_revision = int(state.index_revision or 0) + 1
+            state.embedding_fingerprint = fingerprint
+            state.updated_at = now
+            await session.commit()
+            return _dict(row)
+
+    async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
+        mapping_id = str(page["knowledge_base_mapping_id"])
+        safe_error = error[:4000]
+        async with self._sf() as session:
+            stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
+            row = (await session.execute(stmt)).scalar_one_or_none()
+            now = _now()
+            if row is None:
+                row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
+                session.add(row)
+                row.index_status = "failed"
+            else:
+                vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
+                # Preserve the last atomically completed generation while the
+                # newer remote version remains pending after an embedding error.
+                row.index_status = "ready" if vector_count else "failed"
+            row.last_seen_sync_id = sync_id
+            row.last_error = safe_error
+            row.updated_at = now
+            await self._ensure_state(session, mapping_id)
+            await session.commit()
+
+    async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
+        async with self._sf() as session:
+            stmt = (
+                select(LlmWikiWikiPageRow)
+                .where(
+                    LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
+                    LlmWikiWikiPageRow.is_remote_deleted.is_(False),
+                    or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
+                )
+                .with_for_update()
+            )
+            rows = list((await session.execute(stmt)).scalars())
+            if not rows:
+                return 0
+            now = _now()
+            page_ids = [row.id for row in rows]
+            await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
+            for row in rows:
+                row.is_remote_deleted = True
+                row.index_status = "deleted"
+                row.updated_at = now
+            state = await self._ensure_state(session, mapping_id)
+            state.index_revision = int(state.index_revision or 0) + 1
+            state.updated_at = now
+            await session.commit()
+            return len(rows)
+
+    async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
+        stmt = (
+            select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
+            .join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
+            .where(
+                LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
+                LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
+                LlmWikiWikiPageRow.index_status == "ready",
+                LlmWikiWikiPageRow.is_remote_deleted.is_(False),
+                LlmWikiWikiPageRow.status.in_(["published", "draft"]),
+            )
+            .order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
+        )
+        async with self._sf() as session:
+            state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+            pairs = list((await session.execute(stmt)).all())
+            rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
+            return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
+
+    async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
+        async with self._sf() as session:
+            state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+            if state is None:
+                return 0, None, "not_synced"
+            return int(state.index_revision or 0), state.embedding_fingerprint, state.state
+
+    async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
+        now = _now()
+        async with self._sf() as session:
+            try:
+                await self._ensure_state(session, mapping_id)
+                await session.commit()
+            except IntegrityError:
+                # Another worker created the one-per-KB state row between our
+                # read and insert. Roll back this isolated bootstrap session;
+                # the conditional UPDATE below is the actual lease election.
+                await session.rollback()
+        async with self._sf() as session:
+            stmt = (
+                update(LlmWikiWikiSyncStateRow)
+                .where(
+                    LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
+                    or_(
+                        LlmWikiWikiSyncStateRow.lease_owner.is_(None),
+                        LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
+                        LlmWikiWikiSyncStateRow.lease_expires_at < now,
+                    ),
+                )
+                .values(
+                    state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
+                    active_sync_id=sync_id,
+                    lease_owner=owner,
+                    lease_expires_at=now + timedelta(seconds=lease_seconds),
+                    last_started_at=now,
+                    last_error=None,
+                    updated_at=now,
+                )
+            )
+            result = await session.execute(stmt)
+            await session.commit()
+            return bool(result.rowcount)
+
+    async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
+        now = _now()
+        async with self._sf() as session:
+            result = await session.execute(
+                update(LlmWikiWikiSyncStateRow)
+                .where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
+                .values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
+            )
+            await session.commit()
+            return bool(result.rowcount)
+
+    async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
+        async with self._sf() as session:
+            state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+            if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
+                return
+            visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
+            local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
+            ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
+            failed_count = int(
+                (
+                    await session.execute(
+                        select(func.count())
+                        .select_from(LlmWikiWikiPageRow)
+                        .where(
+                            *visible,
+                            or_(
+                                LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
+                                LlmWikiWikiPageRow.last_error.is_not(None),
+                            ),
+                        )
+                    )
+                ).scalar_one()
+            )
+            vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
+            now = _now()
+            was_rebuilding = state.state == "rebuilding"
+            state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
+            state.active_sync_id = None
+            state.lease_owner = None
+            state.lease_expires_at = None
+            state.last_error = (error or "")[:4000] or None
+            state.remote_page_count = remote_page_count
+            state.local_page_count = local_count
+            state.ready_page_count = ready_count
+            state.failed_page_count = failed_count
+            state.vector_count = vector_count
+            if success and not error:
+                state.last_completed_at = now
+            if success:
+                state.last_success_at = now
+            state.updated_at = now
+            await session.commit()
+
+    async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
+        stmt = select(LlmWikiWikiSyncStateRow)
+        if mapping_ids:
+            stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
+        stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
+        async with self._sf() as session:
+            return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+    async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
+        if not mapping_ids:
+            return 0
+        now = _now()
+        async with self._sf() as session:
+            for mapping_id in mapping_ids:
+                state = await self._ensure_state(session, mapping_id)
+                state.state = "rebuild_required"
+                state.embedding_fingerprint = fingerprint
+                state.updated_at = now
+            await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
+            await session.commit()
+        return len(mapping_ids)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260823_01_llmwiki_local_wiki_index.py

@@ -0,0 +1,143 @@
+"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
+
+Revision ID: 20260823_01
+Revises: 20260821_02
+Create Date: 2026-08-23
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+revision = "20260823_01"
+down_revision = "20260821_02"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
+    if "wiki_index_enabled" not in columns:
+        op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
+    if "external_search_enabled" not in columns:
+        op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
+    if "external_search_updated_at" not in columns:
+        op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
+    mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
+    if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
+        op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
+    if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
+        op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
+
+    long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
+    portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
+    if not inspector.has_table("llmwiki_wiki_pages"):
+        op.create_table(
+            "llmwiki_wiki_pages",
+            sa.Column("id", sa.String(36), primary_key=True),
+            sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
+            sa.Column("remote_page_id", sa.String(128), nullable=True),
+            sa.Column("slug", sa.String(1024), nullable=False),
+            sa.Column("slug_hash", sa.String(64), nullable=False),
+            sa.Column("title", sa.String(512), nullable=False, server_default=""),
+            sa.Column("summary", long_text, nullable=False),
+            sa.Column("content_md", long_text, nullable=False),
+            sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
+            sa.Column("status", sa.String(32), nullable=False, server_default="published"),
+            sa.Column("aliases", portable_json, nullable=False),
+            sa.Column("category_path", portable_json, nullable=False),
+            sa.Column("parent_slug", sa.String(1024), nullable=True),
+            sa.Column("wiki_path", sa.String(1024), nullable=True),
+            sa.Column("source_refs", portable_json, nullable=False),
+            sa.Column("chunk_refs", portable_json, nullable=False),
+            sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("content_hash", sa.String(64), nullable=False),
+            sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
+            sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
+            sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("last_error", sa.Text(), nullable=True),
+            sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
+            sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
+            sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+            sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+            sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
+        )
+        op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
+        op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
+        op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
+        op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
+        op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
+        op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
+        op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
+        op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
+        op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
+        op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
+        op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
+        op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
+        op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
+
+    if not inspector.has_table("llmwiki_wiki_vectors"):
+        op.create_table(
+            "llmwiki_wiki_vectors",
+            sa.Column("id", sa.String(36), primary_key=True),
+            sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
+            sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
+            sa.Column("section_index", sa.Integer(), nullable=False),
+            sa.Column("heading", sa.String(1024), nullable=True),
+            sa.Column("section_content", long_text, nullable=False),
+            sa.Column("content_hash", sa.String(64), nullable=False),
+            sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
+            sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
+            sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
+            sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+            sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
+        )
+        op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
+        op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
+        op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
+        op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
+        op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
+
+    if not inspector.has_table("llmwiki_wiki_sync_states"):
+        op.create_table(
+            "llmwiki_wiki_sync_states",
+            sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
+            sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
+            sa.Column("active_sync_id", sa.String(36), nullable=True),
+            sa.Column("lease_owner", sa.String(128), nullable=True),
+            sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("last_error", sa.Text(), nullable=True),
+            sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
+            sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
+            sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
+        if inspector.has_table(table):
+            op.drop_table(table)
+    columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
+    indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
+    for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
+        if index in indexes:
+            op.drop_index(index, table_name="llmwiki_knowledge_bases")
+    for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
+        if column in columns:
+            op.drop_column("llmwiki_knowledge_bases", column)

offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py

@@ -52,6 +52,7 @@ from deerflow.persistence.intent_turns.model import IntentTurnRow
 from deerflow.persistence.light_apps.model import LightAppRow
 from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
 from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
 from deerflow.persistence.memory_config.model import UserMemoryConfigRow
 from deerflow.persistence.menu_overrides.model import MenuOverrideRow
 from deerflow.persistence.models.run_event import RunEventRow
@@ -129,6 +130,9 @@ __all__ = [
     "LlmCallMetricRow",
     "LlmWikiKnowledgeBaseRow",
     "LlmWikiConversationDepositRow",
+    "LlmWikiWikiPageRow",
+    "LlmWikiWikiVectorRow",
+    "LlmWikiWikiSyncStateRow",
     "MenuOverrideRow",
     "NotificationRow",
     "ParallelAgentTaskRow",

offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/llmwiki_search_tool.py

@@ -6,28 +6,25 @@ import json
 import re
 from collections.abc import Mapping
 from typing import Any
-from urllib.parse import urlencode
 
 from langchain.tools import ToolRuntime, tool
 from langgraph.typing import ContextT
 
 from deerflow.agents.thread_state import ThreadState
+from deerflow.config import get_app_config
 from deerflow.config.agents_config import load_agent_config
-from deerflow.integrations.weknora.client import WeKnoraError
-from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
+from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
 from deerflow.runtime.user_context import get_current_user, get_effective_user_id
 
-
 _CONVERSATION_DEPOSIT_NAME = "对话沉淀"
 _CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
 _LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
 
 
 def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
-    return (
-        str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID
-        and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
-    )
+    return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
 
 
 def _sanitize_conversation_deposit_value(value: Any) -> Any:
@@ -79,9 +76,10 @@ async def llmwiki_search_tool(
         query: Focused natural-language retrieval query.
     """
 
+    local_config = get_app_config().llmwiki.local_wiki_index
     resolved = get_resolved_llmwiki_runtime()
-    if not resolved.weknora_enabled:
-        return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
+    if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
+        return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
 
     store = get_llmwiki_store()
     user_id = get_effective_user_id()
@@ -111,52 +109,26 @@ async def llmwiki_search_tool(
         return "No authorized LLMWiki knowledge bases are available."
 
     try:
-        results = await build_weknora_client(resolved).search(
-            query.strip(),
-            [str(row["weknora_id"]) for row in rows],
-        )
-    except (RuntimeError, WeKnoraError) as exc:
-        return f"LLMWiki retrieval failed: {exc}"
-
-    local_by_remote = {str(row["weknora_id"]): row for row in rows}
-    citations = []
-    for result in results:
-        mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
-        if mapping is None:
-            continue
-        visible_result = (
-            _sanitize_conversation_deposit_value(result)
-            if _is_conversation_deposit_mapping(mapping)
-            else result
-        )
-        params = urlencode(
-            {
-                "knowledge_base_id": mapping["id"],
-                "chunk_id": visible_result.get("chunk_id") or "",
-                "knowledge_id": visible_result.get("knowledge_id") or "",
-                "document_id": visible_result.get("knowledge_id") or "",
-                "wiki_slug": visible_result.get("wiki_slug") or visible_result.get("slug") or "",
-            }
-        )
-        citations.append(
-            {
-                "title": visible_result.get("title") or visible_result.get("filename") or mapping.get("name") or "LLMWiki source",
-                "content": visible_result.get("content") or "",
-                "score": visible_result.get("score") or 0,
-                "knowledge_base_id": mapping["id"],
-                "knowledge_base_name": mapping.get("name") or "",
-                "source": mapping.get("name") or "LLMWiki",
-                "knowledge_id": visible_result.get("knowledge_id") or "",
-                "chunk_id": visible_result.get("chunk_id") or "",
-                "url": f"/weknora-source-preview?{params}",
-            }
-        )
+        payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
+    except (RuntimeError, WikiIndexError) as exc:
+        code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
+        return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
+    mapping_by_id = {str(row["id"]): row for row in rows}
+    results = []
+    for item in payload["results"]:
+        mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
+        visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
+        results.append(visible)
     return json.dumps(
         {
             "query": query,
             "selection": [row["id"] for row in rows],
             "knowledge_base_count": len(rows),
-            "results": citations,
+            "mode": payload.get("mode") or "wiki_api",
+            "results": results,
+            "partial": payload.get("partial", False),
+            "warnings": payload.get("warnings", []),
+            "fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
         },
         ensure_ascii=False,
     )

offline-backend-20260512/backend/packages/harness/pyproject.toml

@@ -21,7 +21,8 @@ dependencies = [
     "langgraph-cli>=0.4.24",
     "langgraph-runtime-inmem>=0.28.0",
     "markdownify>=1.2.2",
-    "markitdown[all,xlsx]>=0.0.1a2",
+    "markitdown[all,xlsx]>=0.0.1a2",
+    "numpy>=2.0.0",
     "pydantic>=2.12.5",
     "pyyaml>=6.0.3",
     "readabilipy>=0.3.0",

offline-backend-20260512/backend/pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "croniter>=6.2.2",
     "hindsight-client>=0.4.22",
     "asyncmy>=0.2.11",
+    "openpyxl>=3.1.5",
 ]
 
 [project.optional-dependencies]

offline-backend-20260512/backend/tests/test_external_llmwiki_search.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+from starlette.requests import Request
+
+from app.gateway.auth_middleware import _authorize_external_llmwiki
+from app.gateway.routers.external_llmwiki import _external_mappings
+
+
+def _request(key: str) -> Request:
+    return Request({"type": "http", "method": "POST", "path": "/api/external/llmwiki/wiki/vector-search", "headers": [(b"x-api-key", key.encode())]})
+
+
+def test_external_api_key_uses_isolated_fail_closed_auth(monkeypatch) -> None:
+    config = SimpleNamespace(enabled=True, api_key="current", previous_api_key="previous")
+    monkeypatch.setattr("deerflow.config.get_app_config", lambda: SimpleNamespace(llmwiki=SimpleNamespace(local_wiki_index=SimpleNamespace(external_api=config))))
+    assert _authorize_external_llmwiki(_request("current")) is None
+    assert _authorize_external_llmwiki(_request("previous")) is None
+    rejected = _authorize_external_llmwiki(_request("wrong"))
+    assert rejected is not None
+    assert rejected.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_external_mappings_require_explicit_public_external_permission() -> None:
+    rows = [
+        {"id": "allowed", "publication_status": "published", "external_search_enabled": True, "wiki_index_enabled": True},
+        {"id": "private", "publication_status": "private", "external_search_enabled": True, "wiki_index_enabled": True},
+        {"id": "not-opted-in", "publication_status": "published", "external_search_enabled": False, "wiki_index_enabled": True},
+        {
+            "id": "deposit",
+            "name": "对话沉淀",
+            "owner_user_id": "system",
+            "publication_status": "published",
+            "external_search_enabled": True,
+            "wiki_index_enabled": True,
+        },
+    ]
+
+    class Store:
+        async def list_visible(self, *_args, **_kwargs):
+            return rows
+
+    request = Request({"type": "http", "method": "POST", "path": "/", "headers": [], "app": SimpleNamespace(state=SimpleNamespace(llmwiki_store=Store()))})
+    assert [row["id"] for row in await _external_mappings(request, [])] == ["allowed"]
+    with pytest.raises(HTTPException) as exc_info:
+        await _external_mappings(request, ["not-opted-in"])
+    assert exc_info.value.status_code == 404

offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector
+from deerflow.persistence.base import Base
+from deerflow.persistence.llmwiki.model import LlmWikiKnowledgeBaseRow
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
+from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
+
+
+@pytest.mark.asyncio
+async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(tmp_path) -> None:
+    engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'wiki.db'}")
+    async with engine.begin() as connection:
+        await connection.run_sync(
+            lambda sync_connection: Base.metadata.create_all(
+                sync_connection,
+                tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiSyncStateRow.__table__],
+            )
+        )
+    session_factory = async_sessionmaker(engine, expire_on_commit=False)
+    async with session_factory() as session:
+        session.add(
+            LlmWikiKnowledgeBaseRow(
+                id="mapping-1",
+                weknora_id="remote-1",
+                owner_user_id="u1",
+                name="Wiki",
+                description="",
+                kb_type="document",
+                publication_status="private",
+                created_at=datetime.now(UTC),
+                updated_at=datetime.now(UTC),
+            )
+        )
+        await session.commit()
+
+    store = SqlLlmWikiIndexStore(session_factory)
+    assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
+    assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
+    page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
+    blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
+    saved = await store.replace_page_vectors(
+        page=page,
+        vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
+        fingerprint="fp",
+        sync_id="sync-1",
+    )
+    snapshot = await store.load_vector_snapshot("mapping-1", "fp")
+    assert saved["slug"] == "people/alice"
+    assert len(snapshot["rows"]) == 1
+    assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
+
+    assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
+    assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
+    await engine.dispose()
+
+
+def test_vector_codec_rejects_corrupt_and_non_finite_vectors() -> None:
+    with pytest.raises(ValueError, match="NaN"):
+        encode_vector([1.0, float("nan")], dimensions=2)
+    with pytest.raises(ValueError, match="Corrupt"):
+        decode_vector(b"123", 2)

offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
+from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
+
+
+class FakeEmbedding:
+    dimensions = 2
+    fingerprint = "fp"
+
+    async def embed_query(self, text: str) -> list[float]:
+        return [1.0, 0.0]
+
+
+def test_enabled_index_requires_supported_enabled_embedding() -> None:
+    with pytest.raises(ValidationError, match="embedding must be enabled"):
+        LocalWikiIndexConfig(enabled=True, embedding={"enabled": False})
+    with pytest.raises(ValidationError, match="provider must be openai"):
+        LocalWikiIndexConfig(embedding={"provider": "custom"})
+
+
+@pytest.mark.asyncio
+async def test_exact_search_groups_sections_by_wiki_page_and_filters_drafts() -> None:
+    store = MemoryLlmWikiIndexStore()
+    await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
+    published = normalize_wiki_page("kb", {"slug": "alice", "title": "Alice", "summary": "Founder", "content": "Alpha", "status": "published"})
+    draft = normalize_wiki_page("kb", {"slug": "secret", "title": "Secret", "content": "Draft", "status": "draft"})
+    a, _ = encode_vector([1.0, 0.0], dimensions=2)
+    b, _ = encode_vector([0.9, 0.1], dimensions=2)
+    await store.replace_page_vectors(
+        page=published,
+        vectors=[
+            {"section_index": 0, "heading": "A", "section_content": "Alpha", "content_hash": "a", "embedding_dimensions": 2, "vector_blob": a},
+            {"section_index": 1, "heading": "B", "section_content": "Beta", "content_hash": "b", "embedding_dimensions": 2, "vector_blob": b},
+        ],
+        fingerprint="fp",
+        sync_id="s",
+    )
+    await store.replace_page_vectors(page=draft, vectors=[{"section_index": 0, "heading": None, "section_content": "Draft", "content_hash": "d", "embedding_dimensions": 2, "vector_blob": a}], fingerprint="fp", sync_id="s")
+    config = LocalWikiIndexConfig(min_similarity=0, top_k_sections=10, top_k_pages=8, max_sections_per_page=2)
+    service = WikiVectorSearchService(config, FakeEmbedding(), WikiVectorCache(store, revision_check_seconds=0))
+    mapping = {"id": "kb", "name": "People"}
+
+    result = await service.search("alice", [mapping])
+    assert [item["wiki_slug"] for item in result["results"]] == ["alice"]
+    assert len(result["results"][0]["matched_sections"]) == 2
+    assert "wiki_slug=alice" in result["results"][0]["url"]
+
+    with_drafts = await service.search("alice", [mapping], include_drafts=True)
+    assert {item["wiki_slug"] for item in with_drafts["results"]} == {"alice", "secret"}
+
+    narrow_service = WikiVectorSearchService(
+        LocalWikiIndexConfig(min_similarity=0, top_k_sections=1, top_k_pages=1),
+        FakeEmbedding(),
+        WikiVectorCache(store, revision_check_seconds=0),
+    )
+    assert [item["wiki_slug"] for item in (await narrow_service.search("alice", [mapping]))["results"]] == ["alice"]
+
+
+@pytest.mark.asyncio
+async def test_corrupt_blob_marks_index_for_rebuild() -> None:
+    store = MemoryLlmWikiIndexStore()
+    await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
+    page = normalize_wiki_page("kb", {"slug": "broken", "title": "Broken", "content": "Broken", "status": "published"})
+    blob, _ = encode_vector([1.0, 0.0], dimensions=2)
+    saved = await store.replace_page_vectors(
+        page=page,
+        vectors=[{"section_index": 0, "heading": None, "section_content": "Broken", "content_hash": "x", "embedding_dimensions": 2, "vector_blob": blob}],
+        fingerprint="fp",
+        sync_id="s",
+    )
+    store.vectors[saved["id"]][0]["vector_blob"] = b"broken"
+    service = WikiVectorSearchService(
+        LocalWikiIndexConfig(min_similarity=0, top_k_sections=10, top_k_pages=8),
+        FakeEmbedding(),
+        WikiVectorCache(store, revision_check_seconds=0),
+    )
+
+    with pytest.raises(WikiIndexError) as exc_info:
+        await service.search("broken", [{"id": "kb", "name": "Broken"}])
+    assert exc_info.value.code == "WIKI_INDEX_REBUILDING"
+    assert store.states["kb"]["state"] == "rebuild_required"

offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from openpyxl import load_workbook
+
+from app.gateway.routers.llmwiki import _build_wiki_excel
+from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
+
+
+class _Store:
+    def __init__(self, states):
+        self.states = states
+
+    async def list_index_status(self, mapping_ids=None):
+        wanted = set(mapping_ids or [])
+        return [state for state in self.states if not wanted or state["knowledge_base_mapping_id"] in wanted]
+
+
+class _WikiClient:
+    def __init__(self):
+        self.list_calls = []
+        self.raw_search_called = False
+
+    async def list_wiki_pages(self, knowledge_base_id, *, page=1, page_size=50, query=""):
+        self.list_calls.append((knowledge_base_id, query))
+        return {"pages": [{"slug": "guides/install", "title": "安装"}], "total": 1, "total_pages": 1}
+
+    async def get_wiki_page(self, knowledge_base_id, slug):
+        return {
+            "id": f"{knowledge_base_id}:{slug}",
+            "slug": slug,
+            "title": "安装",
+            "summary": "Wiki 摘要",
+            "content": "# 安装\n这是处理后的 Wiki 正文。",
+            "status": "published",
+            "category_path": ["指南"],
+        }
+
+    async def search(self, query, knowledge_base_ids):
+        self.raw_search_called = True
+        raise AssertionError("raw knowledge search must never be called")
+
+
+class _VectorSearch:
+    def __init__(self):
+        self.embedding = SimpleNamespace(fingerprint="fp-current")
+        self.calls = []
+
+    async def search(self, query, mappings, **kwargs):
+        self.calls.append([mapping["id"] for mapping in mappings])
+        mapping = mappings[0]
+        return {
+            "results": [
+                {
+                    "kind": "wiki_page",
+                    "id": "local-page",
+                    "wiki_page_id": "local-page",
+                    "wiki_slug": "local/page",
+                    "knowledge_base_id": mapping["id"],
+                    "title": "本地页面",
+                    "content": "本地 Wiki",
+                    "matched_sections": [],
+                    "score": 0.95,
+                }
+            ],
+            "warnings": [],
+        }
+
+
+def _ready_state(mapping_id: str):
+    return {
+        "knowledge_base_mapping_id": mapping_id,
+        "state": "idle",
+        "last_success_at": "2026-08-24T00:00:00Z",
+        "embedding_fingerprint": "fp-current",
+        "local_page_count": 1,
+        "ready_page_count": 1,
+        "failed_page_count": 0,
+        "vector_count": 3,
+    }
+
+
+def test_fully_vectorized_requires_completed_full_consistent_scan() -> None:
+    state = _ready_state("kb")
+    assert is_fully_vectorized(state, "fp-current")
+    assert not is_fully_vectorized({**state, "state": "partial"}, "fp-current")
+    assert not is_fully_vectorized({**state, "ready_page_count": 0}, "fp-current")
+    assert not is_fully_vectorized(state, "different-fingerprint")
+
+
+@pytest.mark.asyncio
+async def test_unvectorized_library_uses_only_wiki_page_search_api() -> None:
+    client = _WikiClient()
+    service = WikiRetrievalService(_Store([]), vector_search=_VectorSearch(), client=client)
+    mapping = {"id": "local-kb", "weknora_id": "remote-kb", "name": "测试库"}
+
+    result = await service.search("安装", [mapping])
+
+    assert result["mode"] == "wiki_api"
+    assert result["fallback_knowledge_base_ids"] == ["local-kb"]
+    assert result["results"][0]["kind"] == "wiki_page"
+    assert result["results"][0]["content"] == "# 安装\n这是处理后的 Wiki 正文。"
+    assert "chunk_id" not in result["results"][0]
+    assert client.list_calls == [("remote-kb", "安装")]
+    assert not client.raw_search_called
+
+
+@pytest.mark.asyncio
+async def test_mixed_search_vectors_complete_library_and_falls_back_incomplete_library() -> None:
+    client = _WikiClient()
+    vector = _VectorSearch()
+    service = WikiRetrievalService(_Store([_ready_state("ready")]), vector_search=vector, client=client)
+    mappings = [
+        {"id": "ready", "weknora_id": "remote-ready", "name": "已完成"},
+        {"id": "pending", "weknora_id": "remote-pending", "name": "未完成"},
+    ]
+
+    result = await service.search("安装", mappings, top_k_pages=8)
+
+    assert result["mode"] == "mixed"
+    assert vector.calls == [["ready"]]
+    assert client.list_calls == [("remote-pending", "安装")]
+    assert {item["knowledge_base_id"] for item in result["results"]} == {"ready", "pending"}
+
+
+def test_excel_export_contains_current_directory_and_every_parent_directory() -> None:
+    long_content = "正文" * 20_000
+    pages = [
+        {"id": "1", "slug": "root", "title": "根专题", "content": "根", "status": "published"},
+        {"id": "2", "slug": "root/child", "title": "子专题", "parent_slug": "root", "content": "子", "status": "published"},
+        {
+            "id": "3",
+            "slug": "root/child/article",
+            "title": "文章",
+            "parent_slug": "root/child",
+            "content": long_content,
+            "status": "published",
+        },
+    ]
+
+    workbook = load_workbook(_build_wiki_excel("知识库", pages), read_only=True)
+    sheet = workbook["Wiki 全量导出"]
+    rows = list(sheet.iter_rows(values_only=True))
+    headings = list(rows[0])
+    article = dict(zip(headings, rows[3], strict=True))
+
+    assert article["目录完整路径"] == "根专题 / 子专题"
+    assert article["所在目录"] == "子专题"
+    assert article["父级目录完整路径"] == "根专题"
+    assert article["第1级目录"] == "根专题"
+    assert article["第2级目录"] == "子专题"
+    assert article["正文"] + article["正文（续1）"] == long_content

offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import pytest
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
+
+
+class FakeEmbedding:
+    dimensions = 2
+    fingerprint = "fp"
+
+    def __init__(self) -> None:
+        self.calls = 0
+
+    async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+        self.calls += 1
+        return [[1.0, 0.0] for _ in texts]
+
+
+class FailingEmbedding(FakeEmbedding):
+    async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+        raise RuntimeError("embedding unavailable")
+
+
+class NewFingerprintEmbedding(FakeEmbedding):
+    fingerprint = "fp-new"
+
+
+class FakeWeKnora:
+    def __init__(self) -> None:
+        self.pages = {
+            "one": {
+                "id": "p1",
+                "slug": "one",
+                "title": "One",
+                "summary": "A long enough summary for indexing and retrieval.",
+                "content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
+                "status": "published",
+                "version": 1,
+            },
+        }
+
+    async def get_knowledge_base(self, knowledge_base_id: str):
+        return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
+
+    async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
+        return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
+
+    async def get_wiki_page(self, knowledge_base_id: str, slug: str):
+        return dict(self.pages[slug])
+
+
+@pytest.mark.asyncio
+async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
+    config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
+    store = MemoryLlmWikiIndexStore()
+    embedding = FakeEmbedding()
+    client = FakeWeKnora()
+    service = WikiSyncService(config, store, embedding, WikiVectorCache(store), client)
+    mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
+
+    first = await service.sync_mapping(mapping)
+    assert first["changed_pages"] == 1
+    assert embedding.calls == 1
+    second = await service.sync_mapping(mapping)
+    assert second["changed_pages"] == 0
+    assert embedding.calls == 1
+
+    client.pages.clear()
+    deleted = await service.sync_mapping(mapping)
+    assert deleted["deleted_pages"] == 1
+    assert next(iter(store.pages.values()))["is_remote_deleted"] is True
+
+
+@pytest.mark.asyncio
+async def test_page_failure_keeps_previous_generation_and_marks_partial() -> None:
+    config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
+    store = MemoryLlmWikiIndexStore()
+    client = FakeWeKnora()
+    mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
+    await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
+    page = next(iter(store.pages.values()))
+    old_hash = page["content_hash"]
+    old_vectors = list(store.vectors[page["id"]])
+    client.pages["one"]["content"] += " Changed."
+
+    result = await WikiSyncService(config, store, FailingEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
+
+    assert result["status"] == "partial"
+    assert store.states["kb"]["state"] == "partial"
+    assert store.states["kb"]["failed_page_count"] == 1
+    assert store.pages[page["id"]]["content_hash"] == old_hash
+    assert store.pages[page["id"]]["index_status"] == "ready"
+    assert store.vectors[page["id"]] == old_vectors
+
+
+@pytest.mark.asyncio
+async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> None:
+    config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
+    store = MemoryLlmWikiIndexStore()
+    client = FakeWeKnora()
+    mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
+    await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
+
+    embedding = NewFingerprintEmbedding()
+    result = await WikiSyncService(config, store, embedding, WikiVectorCache(store), client).sync_mapping(mapping)
+
+    assert result["status"] == "completed"
+    assert result["changed_pages"] == 1
+    assert embedding.calls == 1
+    assert store.states["kb"]["state"] == "idle"
+    assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
+    assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}