feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
@@ -2,18 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime
|
||||
from io import BytesIO
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import quote, urlencode, urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import JSONResponse, RedirectResponse, Response
|
||||
from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.csrf_middleware import is_secure_request
|
||||
@@ -27,17 +30,28 @@ from app.gateway.llmwiki_deposit import (
|
||||
ensure_conversation_deposit_mapping,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
|
||||
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_TTL_SECONDS as _WEKNORA_EMBED_TTL_SECONDS,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WeKnoraEmbedContext,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
sign_weknora_embed_payload as _sign_embed_payload,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_embed_cookie as _verify_embed_cookie,
|
||||
)
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
|
||||
@@ -168,6 +182,8 @@ class SearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=4000)
|
||||
scope: Literal["personal", "public", "all"] = "all"
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
top_k: int = Field(default=8, ge=1, le=50)
|
||||
include_drafts: bool = False
|
||||
|
||||
|
||||
class KnowledgeUrlImport(BaseModel):
|
||||
@@ -223,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
|
||||
return value
|
||||
|
||||
|
||||
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
|
||||
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -575,12 +600,12 @@ async def _authorize_weknora_api_proxy(
|
||||
# page to render upload controls and current-tenant state. They do not
|
||||
# expose other knowledge-base contents.
|
||||
if read_method and tail and tail[0] in {
|
||||
"models",
|
||||
"system",
|
||||
"storage-backends",
|
||||
"vector-stores",
|
||||
"web-search-providers",
|
||||
"mcp-services",
|
||||
"models",
|
||||
"system",
|
||||
"storage-backends",
|
||||
"vector-stores",
|
||||
"web-search-providers",
|
||||
"mcp-services",
|
||||
}:
|
||||
return
|
||||
|
||||
@@ -654,7 +679,7 @@ def _with_public_prefix(prefix: str, path: str) -> str:
|
||||
|
||||
def _without_public_prefix(prefix: str, path: str) -> str:
|
||||
if prefix and path.startswith(f"{prefix}/"):
|
||||
return path[len(prefix):]
|
||||
return path[len(prefix) :]
|
||||
if prefix and path == prefix:
|
||||
return "/"
|
||||
return path
|
||||
@@ -1626,6 +1651,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"description": description,
|
||||
"type": source.get("type") or row.get("kb_type") or "document",
|
||||
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
|
||||
"wiki_index_enabled": row.get("wiki_index_enabled", True),
|
||||
"external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
|
||||
"external_search_updated_at": row.get("external_search_updated_at"),
|
||||
"owner_user_id": row.get("owner_user_id") if is_admin else None,
|
||||
"is_owner": is_owner,
|
||||
"can_write": not is_conversation_deposit and (is_admin or is_owner),
|
||||
@@ -1816,6 +1844,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
|
||||
"""Read every Wiki page and its processed Markdown, never raw documents."""
|
||||
|
||||
summaries: dict[str, dict[str, Any]] = {}
|
||||
page_number = 1
|
||||
while True:
|
||||
listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
|
||||
rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
for item in rows:
|
||||
slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
if slug:
|
||||
summaries[slug] = item
|
||||
total_pages = max(1, int(listing.get("total_pages") or 1))
|
||||
if page_number >= total_pages or not rows:
|
||||
break
|
||||
page_number += 1
|
||||
|
||||
pages: list[dict[str, Any]] = []
|
||||
entries = list(summaries.items())
|
||||
for start in range(0, len(entries), 20):
|
||||
batch = entries[start : start + 20]
|
||||
details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
|
||||
pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
|
||||
return pages
|
||||
|
||||
|
||||
def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
|
||||
parent_slug = str(page.get("parent_slug") or "").strip("/")
|
||||
if parent_slug:
|
||||
ancestors: list[str] = []
|
||||
visited: set[str] = set()
|
||||
current = by_slug.get(parent_slug)
|
||||
while current is not None:
|
||||
slug = str(current.get("slug") or "").strip("/")
|
||||
if not slug or slug in visited:
|
||||
break
|
||||
visited.add(slug)
|
||||
ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
|
||||
current_parent = str(current.get("parent_slug") or "").strip("/")
|
||||
current = by_slug.get(current_parent) if current_parent else None
|
||||
if ancestors:
|
||||
return ancestors
|
||||
|
||||
category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
|
||||
if category_path:
|
||||
return category_path
|
||||
wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
|
||||
if len(wiki_path) > 2:
|
||||
return wiki_path[1:-1]
|
||||
slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
|
||||
return slug_parts[:-1]
|
||||
|
||||
|
||||
def _safe_excel_text(value: Any) -> str:
|
||||
text = str(value or "")
|
||||
return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
|
||||
|
||||
|
||||
def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
|
||||
pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
|
||||
by_slug = {page["slug"]: page for page in pages if page["slug"]}
|
||||
directories = [_wiki_directory_parts(page, by_slug) for page in pages]
|
||||
max_depth = max((len(parts) for parts in directories), default=0)
|
||||
max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
|
||||
columns = [
|
||||
"知识库",
|
||||
"标题",
|
||||
"Slug",
|
||||
"目录完整路径",
|
||||
"所在目录",
|
||||
"父级目录完整路径",
|
||||
*(f"第{level}级目录" for level in range(1, max_depth + 1)),
|
||||
"摘要",
|
||||
*("正文" if index == 1 else f"正文(续{index - 1})" for index in range(1, max_content_parts + 1)),
|
||||
"页面类型",
|
||||
"状态",
|
||||
"别名",
|
||||
"版本",
|
||||
"创建时间",
|
||||
"更新时间",
|
||||
]
|
||||
workbook = Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.title = "Wiki 全量导出"
|
||||
sheet.append(columns)
|
||||
for cell in sheet[1]:
|
||||
cell.font = Font(bold=True, color="FFFFFF")
|
||||
cell.fill = PatternFill("solid", fgColor="1F4E78")
|
||||
cell.alignment = Alignment(horizontal="center", vertical="center")
|
||||
for page, directory in zip(pages, directories, strict=True):
|
||||
content = str(page.get("content") or "")
|
||||
content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
|
||||
content_parts.extend([""] * (max_content_parts - len(content_parts)))
|
||||
row = [
|
||||
knowledge_base_name,
|
||||
page.get("title"),
|
||||
page.get("slug"),
|
||||
" / ".join(directory),
|
||||
directory[-1] if directory else "根目录",
|
||||
" / ".join(directory[:-1]),
|
||||
*directory,
|
||||
*([""] * (max_depth - len(directory))),
|
||||
page.get("summary"),
|
||||
*content_parts,
|
||||
page.get("page_type"),
|
||||
page.get("status"),
|
||||
"、".join(str(item) for item in page.get("aliases") or []),
|
||||
page.get("version"),
|
||||
page.get("created_at"),
|
||||
page.get("updated_at"),
|
||||
]
|
||||
sheet.append([_safe_excel_text(value) for value in row])
|
||||
sheet.freeze_panes = "A2"
|
||||
sheet.auto_filter.ref = sheet.dimensions
|
||||
for column in sheet.columns:
|
||||
letter = column[0].column_letter
|
||||
heading = str(column[0].value or "")
|
||||
sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
|
||||
for cell in column[1:]:
|
||||
cell.alignment = Alignment(vertical="top", wrap_text=True)
|
||||
output = BytesIO()
|
||||
workbook.save(output)
|
||||
output.seek(0)
|
||||
return output
|
||||
|
||||
|
||||
def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
|
||||
target_query = urlencode({"tab": tab}) if tab != "documents" else ""
|
||||
target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
|
||||
@@ -2421,12 +2578,37 @@ async def list_wiki_pages(
|
||||
_raise_upstream(exc)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
|
||||
async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
|
||||
"""Export every processed Wiki page with its current and parent directories."""
|
||||
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
|
||||
client = _client_or_503()
|
||||
try:
|
||||
pages = await _all_wiki_pages(client, str(row["weknora_id"]))
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
try:
|
||||
output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
|
||||
except ImportError as exc:
|
||||
raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
|
||||
filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
|
||||
return StreamingResponse(
|
||||
output,
|
||||
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
|
||||
async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
payload = body.model_dump(exclude_none=True)
|
||||
try:
|
||||
page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _wiki_page_view(page) or page
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
@@ -2454,6 +2636,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
|
||||
_raise_upstream(exc)
|
||||
try:
|
||||
page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _wiki_page_view(page) or page
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
@@ -2464,6 +2647,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
try:
|
||||
await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -2547,32 +2731,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
|
||||
rows.append(row)
|
||||
else:
|
||||
rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
|
||||
service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
|
||||
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
|
||||
try:
|
||||
results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
local_by_remote = {str(row["weknora_id"]): row for row in rows}
|
||||
visible: list[dict[str, Any]] = []
|
||||
for result in results:
|
||||
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
visible_result = (
|
||||
_sanitize_conversation_deposit_value(result)
|
||||
if _is_conversation_deposit_mapping(mapping)
|
||||
else result
|
||||
)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
|
||||
visible.append(
|
||||
{
|
||||
**visible_result,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
}
|
||||
)
|
||||
return {"results": visible, "knowledge_base_count": len(rows)}
|
||||
result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
result["results"] = [
|
||||
_sanitize_conversation_deposit_value(item)
|
||||
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
|
||||
else item
|
||||
for item in result.get("results") or []
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)
|
||||
|
||||
Reference in new issue
Block a user