feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -2,18 +2,21 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import re
import time
from dataclasses import replace
from datetime import UTC, datetime
from io import BytesIO
from typing import Any, Literal
from urllib.parse import quote, urlencode, urlsplit
import httpx
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import JSONResponse, RedirectResponse, Response
from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.csrf_middleware import is_secure_request
@@ -27,17 +30,28 @@ from app.gateway.llmwiki_deposit import (
ensure_conversation_deposit_mapping,
)
from app.gateway.weknora_embed import (
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
)
from app.gateway.weknora_embed import (
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
)
from app.gateway.weknora_embed import (
WEKNORA_EMBED_TTL_SECONDS as _WEKNORA_EMBED_TTL_SECONDS,
)
from app.gateway.weknora_embed import (
WeKnoraEmbedContext,
)
from app.gateway.weknora_embed import (
sign_weknora_embed_payload as _sign_embed_payload,
)
from app.gateway.weknora_embed import (
verify_weknora_embed_cookie as _verify_embed_cookie,
)
from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
@@ -168,6 +182,8 @@ class SearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=4000)
scope: Literal["personal", "public", "all"] = "all"
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
top_k: int = Field(default=8, ge=1, le=50)
include_drafts: bool = False
class KnowledgeUrlImport(BaseModel):
@@ -223,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
return value
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
service = getattr(request.app.state, "llmwiki_sync_service", None)
config = request.app.state.config.llmwiki.local_wiki_index
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
@@ -575,12 +600,12 @@ async def _authorize_weknora_api_proxy(
# page to render upload controls and current-tenant state. They do not
# expose other knowledge-base contents.
if read_method and tail and tail[0] in {
"models",
"system",
"storage-backends",
"vector-stores",
"web-search-providers",
"mcp-services",
"models",
"system",
"storage-backends",
"vector-stores",
"web-search-providers",
"mcp-services",
}:
return
@@ -654,7 +679,7 @@ def _with_public_prefix(prefix: str, path: str) -> str:
def _without_public_prefix(prefix: str, path: str) -> str:
if prefix and path.startswith(f"{prefix}/"):
return path[len(prefix):]
return path[len(prefix) :]
if prefix and path == prefix:
return "/"
return path
@@ -1626,6 +1651,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"description": description,
"type": source.get("type") or row.get("kb_type") or "document",
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
"wiki_index_enabled": row.get("wiki_index_enabled", True),
"external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
"external_search_updated_at": row.get("external_search_updated_at"),
"owner_user_id": row.get("owner_user_id") if is_admin else None,
"is_owner": is_owner,
"can_write": not is_conversation_deposit and (is_admin or is_owner),
@@ -1816,6 +1844,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
}
async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
"""Read every Wiki page and its processed Markdown, never raw documents."""
summaries: dict[str, dict[str, Any]] = {}
page_number = 1
while True:
listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
for item in rows:
slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
if slug:
summaries[slug] = item
total_pages = max(1, int(listing.get("total_pages") or 1))
if page_number >= total_pages or not rows:
break
page_number += 1
pages: list[dict[str, Any]] = []
entries = list(summaries.items())
for start in range(0, len(entries), 20):
batch = entries[start : start + 20]
details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
return pages
def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
parent_slug = str(page.get("parent_slug") or "").strip("/")
if parent_slug:
ancestors: list[str] = []
visited: set[str] = set()
current = by_slug.get(parent_slug)
while current is not None:
slug = str(current.get("slug") or "").strip("/")
if not slug or slug in visited:
break
visited.add(slug)
ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
current_parent = str(current.get("parent_slug") or "").strip("/")
current = by_slug.get(current_parent) if current_parent else None
if ancestors:
return ancestors
category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
if category_path:
return category_path
wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
if len(wiki_path) > 2:
return wiki_path[1:-1]
slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
return slug_parts[:-1]
def _safe_excel_text(value: Any) -> str:
text = str(value or "")
return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill
pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
by_slug = {page["slug"]: page for page in pages if page["slug"]}
directories = [_wiki_directory_parts(page, by_slug) for page in pages]
max_depth = max((len(parts) for parts in directories), default=0)
max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
columns = [
"知识库",
"标题",
"Slug",
"目录完整路径",
"所在目录",
"父级目录完整路径",
*(f"第{level}级目录" for level in range(1, max_depth + 1)),
"摘要",
*("正文" if index == 1 else f"正文(续{index - 1})" for index in range(1, max_content_parts + 1)),
"页面类型",
"状态",
"别名",
"版本",
"创建时间",
"更新时间",
]
workbook = Workbook()
sheet = workbook.active
sheet.title = "Wiki 全量导出"
sheet.append(columns)
for cell in sheet[1]:
cell.font = Font(bold=True, color="FFFFFF")
cell.fill = PatternFill("solid", fgColor="1F4E78")
cell.alignment = Alignment(horizontal="center", vertical="center")
for page, directory in zip(pages, directories, strict=True):
content = str(page.get("content") or "")
content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
content_parts.extend([""] * (max_content_parts - len(content_parts)))
row = [
knowledge_base_name,
page.get("title"),
page.get("slug"),
" / ".join(directory),
directory[-1] if directory else "根目录",
" / ".join(directory[:-1]),
*directory,
*([""] * (max_depth - len(directory))),
page.get("summary"),
*content_parts,
page.get("page_type"),
page.get("status"),
"、".join(str(item) for item in page.get("aliases") or []),
page.get("version"),
page.get("created_at"),
page.get("updated_at"),
]
sheet.append([_safe_excel_text(value) for value in row])
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
for column in sheet.columns:
letter = column[0].column_letter
heading = str(column[0].value or "")
sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
for cell in column[1:]:
cell.alignment = Alignment(vertical="top", wrap_text=True)
output = BytesIO()
workbook.save(output)
output.seek(0)
return output
def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
target_query = urlencode({"tab": tab}) if tab != "documents" else ""
target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
@@ -2421,12 +2578,37 @@ async def list_wiki_pages(
_raise_upstream(exc)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
"""Export every processed Wiki page with its current and parent directories."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
client = _client_or_503()
try:
pages = await _all_wiki_pages(client, str(row["weknora_id"]))
except WeKnoraError as exc:
_raise_upstream(exc)
try:
output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
except ImportError as exc:
raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
return StreamingResponse(
output,
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
)
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
payload = body.model_dump(exclude_none=True)
try:
page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
_schedule_local_wiki_sync(request, row)
return _wiki_page_view(page) or page
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2454,6 +2636,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
_raise_upstream(exc)
try:
page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
_schedule_local_wiki_sync(request, row)
return _wiki_page_view(page) or page
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2464,6 +2647,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
try:
await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
_schedule_local_wiki_sync(request, row)
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2547,32 +2731,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
rows.append(row)
else:
rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
service = getattr(request.app.state, "llmwiki_retrieval_service", None)
if service is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
try:
results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
except WeKnoraError as exc:
_raise_upstream(exc)
local_by_remote = {str(row["weknora_id"]): row for row in rows}
visible: list[dict[str, Any]] = []
for result in results:
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
visible_result = (
_sanitize_conversation_deposit_value(result)
if _is_conversation_deposit_mapping(mapping)
else result
)
params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
visible.append(
{
**visible_result,
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"source": mapping.get("name") or "LLMWiki",
"url": f"/weknora-source-preview?{params}",
}
)
return {"results": visible, "knowledge_base_count": len(rows)}
result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
mapping_by_id = {str(row["id"]): row for row in rows}
result["results"] = [
_sanitize_conversation_deposit_value(item)
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
else item
for item in result.get("results") or []
]
return result
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)