feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -0,0 +1,133 @@
"""API-key protected public Wiki vector search backed only by local data."""
from __future__ import annotations
import time
from collections import defaultdict, deque
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from deerflow.integrations.weknora.local_index.search import WikiIndexError
router = APIRouter(prefix="/api/external/llmwiki/wiki", tags=["external-llmwiki"])
_requests: dict[str, deque[float]] = defaultdict(deque)
class ExternalWikiSearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=10000)
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
limit: int | None = Field(default=None, ge=1, le=100)
include_content: bool = False
def _config(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index
if not config.enabled or not config.external_api.enabled:
raise HTTPException(status_code=404, detail="Not found")
return config, config.external_api
def _rate_limit(request: Request, rpm: int) -> None:
ip = request.client.host if request.client else "unknown"
key = f"{getattr(request.state, 'external_api_key_fingerprint', 'unknown')}:{ip}"
now = time.monotonic()
queue = _requests[key]
while queue and queue[0] <= now - 60:
queue.popleft()
if len(queue) >= rpm:
raise HTTPException(status_code=429, detail="Rate limit exceeded")
queue.append(now)
def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
async def _external_mappings(request: Request, requested: list[str]) -> list[dict[str, Any]]:
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
allowed = {row["id"]: row for row in mappings if row.get("publication_status") == "published" and row.get("external_search_enabled", False) and row.get("wiki_index_enabled", True) and not _is_conversation_deposit(row)}
if requested:
ids = list(dict.fromkeys(requested))
if any(mapping_id not in allowed for mapping_id in ids):
raise HTTPException(status_code=404, detail="Knowledge base not found")
return [allowed[mapping_id] for mapping_id in ids]
return list(allowed.values())
@router.post("/vector-search")
async def external_vector_search(request: Request, body: ExternalWikiSearchRequest) -> dict[str, Any]:
_, external = _config(request)
_rate_limit(request, external.requests_per_minute)
if len(body.query) > external.max_query_chars:
raise HTTPException(status_code=413, detail="Query is too large")
if len(body.knowledge_base_ids) > external.max_knowledge_bases:
raise HTTPException(status_code=413, detail="Too many knowledge bases")
limit = body.limit or external.default_results
if limit > external.max_results:
raise HTTPException(status_code=400, detail="Result limit is too large")
mappings = await _external_mappings(request, body.knowledge_base_ids)
search = getattr(request.app.state, "llmwiki_vector_search", None)
if search is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
try:
result = await search.search(body.query, mappings, top_k_pages=limit, external=True)
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
# Re-read publication/external flags after vector computation so a switch
# turned off during an in-flight request cannot leak a cached result.
still_allowed = {row["id"] for row in await _external_mappings(request, [])}
result["results"] = [item for item in result["results"] if item.get("knowledge_base_id") in still_allowed]
remaining = external.include_content_max_chars if body.include_content else 0
items = []
for result_item in result["results"]:
section = (result_item.get("matched_sections") or [{}])[0]
snippet = str(section.get("content") or "")[:1000]
content = ""
if remaining > 0:
content = "\n\n".join(str(item.get("content") or "") for item in result_item.get("matched_sections") or [])[:remaining]
remaining -= len(content)
item = {
"knowledge_base_id": result_item["knowledge_base_id"],
"knowledge_base_name": result_item.get("knowledge_base_name") or "",
"wiki_slug": result_item["wiki_slug"],
"title": result_item.get("title") or "",
"summary": result_item.get("summary") or "",
"matched_heading": section.get("heading"),
"snippet": snippet,
"page_type": result_item.get("page_type") or "article",
"page_version": result_item.get("page_version") or 0,
"updated_at": result_item.get("updated_at"),
"score": result_item.get("score") or 0,
}
if body.include_content:
item["content"] = content
items.append(item)
return {"query": body.query, "mode": "vector", "items": items, "count": len(items), "partial": result.get("partial", False)}
@router.get("/pages/{mapping_id}/{slug:path}")
async def external_wiki_page(request: Request, mapping_id: str, slug: str) -> dict[str, Any]:
_, external = _config(request)
_rate_limit(request, external.requests_per_minute)
mappings = await _external_mappings(request, [mapping_id])
if not mappings:
raise HTTPException(status_code=404, detail="Wiki page not found")
page = await request.app.state.llmwiki_index_store.get_page(mapping_id, slug.strip("/"))
if page is None or page.get("status") != "published" or page.get("index_status") != "ready" or page.get("is_remote_deleted"):
raise HTTPException(status_code=404, detail="Wiki page not found")
# Recheck after reading the page so publication/external revocation takes
# effect even when it races this detail request.
await _external_mappings(request, [mapping_id])
return {
"knowledge_base_id": mapping_id,
"knowledge_base_name": mappings[0].get("name") or "",
"wiki_slug": page["slug"],
"title": page.get("title") or "",
"summary": page.get("summary") or "",
"content": str(page.get("content_md") or "")[: external.include_content_max_chars],
"page_type": page.get("page_type") or "article",
"page_version": page.get("page_version") or 0,
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
}
@@ -2,18 +2,21 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import re
import time
from dataclasses import replace
from datetime import UTC, datetime
from io import BytesIO
from typing import Any, Literal
from urllib.parse import quote, urlencode, urlsplit
import httpx
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import JSONResponse, RedirectResponse, Response
from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.csrf_middleware import is_secure_request
@@ -27,17 +30,28 @@ from app.gateway.llmwiki_deposit import (
ensure_conversation_deposit_mapping,
)
from app.gateway.weknora_embed import (
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
)
from app.gateway.weknora_embed import (
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
)
from app.gateway.weknora_embed import (
WEKNORA_EMBED_TTL_SECONDS as _WEKNORA_EMBED_TTL_SECONDS,
)
from app.gateway.weknora_embed import (
WeKnoraEmbedContext,
)
from app.gateway.weknora_embed import (
sign_weknora_embed_payload as _sign_embed_payload,
)
from app.gateway.weknora_embed import (
verify_weknora_embed_cookie as _verify_embed_cookie,
)
from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
@@ -168,6 +182,8 @@ class SearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=4000)
scope: Literal["personal", "public", "all"] = "all"
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
top_k: int = Field(default=8, ge=1, le=50)
include_drafts: bool = False
class KnowledgeUrlImport(BaseModel):
@@ -223,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
return value
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
service = getattr(request.app.state, "llmwiki_sync_service", None)
config = request.app.state.config.llmwiki.local_wiki_index
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
@@ -575,12 +600,12 @@ async def _authorize_weknora_api_proxy(
# page to render upload controls and current-tenant state. They do not
# expose other knowledge-base contents.
if read_method and tail and tail[0] in {
"models",
"system",
"storage-backends",
"vector-stores",
"web-search-providers",
"mcp-services",
"models",
"system",
"storage-backends",
"vector-stores",
"web-search-providers",
"mcp-services",
}:
return
@@ -654,7 +679,7 @@ def _with_public_prefix(prefix: str, path: str) -> str:
def _without_public_prefix(prefix: str, path: str) -> str:
if prefix and path.startswith(f"{prefix}/"):
return path[len(prefix):]
return path[len(prefix) :]
if prefix and path == prefix:
return "/"
return path
@@ -1626,6 +1651,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"description": description,
"type": source.get("type") or row.get("kb_type") or "document",
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
"wiki_index_enabled": row.get("wiki_index_enabled", True),
"external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
"external_search_updated_at": row.get("external_search_updated_at"),
"owner_user_id": row.get("owner_user_id") if is_admin else None,
"is_owner": is_owner,
"can_write": not is_conversation_deposit and (is_admin or is_owner),
@@ -1816,6 +1844,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
}
async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
"""Read every Wiki page and its processed Markdown, never raw documents."""
summaries: dict[str, dict[str, Any]] = {}
page_number = 1
while True:
listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
for item in rows:
slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
if slug:
summaries[slug] = item
total_pages = max(1, int(listing.get("total_pages") or 1))
if page_number >= total_pages or not rows:
break
page_number += 1
pages: list[dict[str, Any]] = []
entries = list(summaries.items())
for start in range(0, len(entries), 20):
batch = entries[start : start + 20]
details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
return pages
def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
parent_slug = str(page.get("parent_slug") or "").strip("/")
if parent_slug:
ancestors: list[str] = []
visited: set[str] = set()
current = by_slug.get(parent_slug)
while current is not None:
slug = str(current.get("slug") or "").strip("/")
if not slug or slug in visited:
break
visited.add(slug)
ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
current_parent = str(current.get("parent_slug") or "").strip("/")
current = by_slug.get(current_parent) if current_parent else None
if ancestors:
return ancestors
category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
if category_path:
return category_path
wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
if len(wiki_path) > 2:
return wiki_path[1:-1]
slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
return slug_parts[:-1]
def _safe_excel_text(value: Any) -> str:
text = str(value or "")
return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill
pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
by_slug = {page["slug"]: page for page in pages if page["slug"]}
directories = [_wiki_directory_parts(page, by_slug) for page in pages]
max_depth = max((len(parts) for parts in directories), default=0)
max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
columns = [
"知识库",
"标题",
"Slug",
"目录完整路径",
"所在目录",
"父级目录完整路径",
*(f"第{level}级目录" for level in range(1, max_depth + 1)),
"摘要",
*("正文" if index == 1 else f"正文(续{index - 1})" for index in range(1, max_content_parts + 1)),
"页面类型",
"状态",
"别名",
"版本",
"创建时间",
"更新时间",
]
workbook = Workbook()
sheet = workbook.active
sheet.title = "Wiki 全量导出"
sheet.append(columns)
for cell in sheet[1]:
cell.font = Font(bold=True, color="FFFFFF")
cell.fill = PatternFill("solid", fgColor="1F4E78")
cell.alignment = Alignment(horizontal="center", vertical="center")
for page, directory in zip(pages, directories, strict=True):
content = str(page.get("content") or "")
content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
content_parts.extend([""] * (max_content_parts - len(content_parts)))
row = [
knowledge_base_name,
page.get("title"),
page.get("slug"),
" / ".join(directory),
directory[-1] if directory else "根目录",
" / ".join(directory[:-1]),
*directory,
*([""] * (max_depth - len(directory))),
page.get("summary"),
*content_parts,
page.get("page_type"),
page.get("status"),
"、".join(str(item) for item in page.get("aliases") or []),
page.get("version"),
page.get("created_at"),
page.get("updated_at"),
]
sheet.append([_safe_excel_text(value) for value in row])
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
for column in sheet.columns:
letter = column[0].column_letter
heading = str(column[0].value or "")
sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
for cell in column[1:]:
cell.alignment = Alignment(vertical="top", wrap_text=True)
output = BytesIO()
workbook.save(output)
output.seek(0)
return output
def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
target_query = urlencode({"tab": tab}) if tab != "documents" else ""
target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
@@ -2421,12 +2578,37 @@ async def list_wiki_pages(
_raise_upstream(exc)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
"""Export every processed Wiki page with its current and parent directories."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
client = _client_or_503()
try:
pages = await _all_wiki_pages(client, str(row["weknora_id"]))
except WeKnoraError as exc:
_raise_upstream(exc)
try:
output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
except ImportError as exc:
raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
return StreamingResponse(
output,
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
)
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
payload = body.model_dump(exclude_none=True)
try:
page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
_schedule_local_wiki_sync(request, row)
return _wiki_page_view(page) or page
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2454,6 +2636,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
_raise_upstream(exc)
try:
page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
_schedule_local_wiki_sync(request, row)
return _wiki_page_view(page) or page
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2464,6 +2647,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
try:
await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
_schedule_local_wiki_sync(request, row)
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2547,32 +2731,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
rows.append(row)
else:
rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
service = getattr(request.app.state, "llmwiki_retrieval_service", None)
if service is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
try:
results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
except WeKnoraError as exc:
_raise_upstream(exc)
local_by_remote = {str(row["weknora_id"]): row for row in rows}
visible: list[dict[str, Any]] = []
for result in results:
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
visible_result = (
_sanitize_conversation_deposit_value(result)
if _is_conversation_deposit_mapping(mapping)
else result
)
params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
visible.append(
{
**visible_result,
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"source": mapping.get("name") or "LLMWiki",
"url": f"/weknora-source-preview?{params}",
}
)
return {"results": visible, "knowledge_base_count": len(rows)}
result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
mapping_by_id = {str(row["id"]): row for row in rows}
result["results"] = [
_sanitize_conversation_deposit_value(item)
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
else item
for item in result.get("results") or []
]
return result
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)
@@ -0,0 +1,290 @@
"""Authenticated APIs for DeerFlow-local Wiki vector search and administration."""
from __future__ import annotations
from typing import Any
from uuid import uuid4
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
class WikiVectorSearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=4000)
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
top_k: int = Field(default=8, ge=1, le=50)
include_drafts: bool = False
include_content: bool = True
class WikiIndexActionRequest(BaseModel):
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
force: bool = False
reason: str = Field(default="manual", max_length=255)
class WikiIndexSettingsRequest(BaseModel):
wiki_index_enabled: bool | None = None
external_search_enabled: bool | None = None
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
return "default", True
return str(user.id), getattr(user, "system_role", None) == "admin"
def _services(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index
if not config.enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
search = getattr(request.app.state, "llmwiki_vector_search", None)
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if search is None or index_store is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE", "message": "Local Wiki index is unavailable"})
return config, search, index_store
async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
user_id, is_admin = await _actor(request)
mapping_store = request.app.state.llmwiki_store
if mapping_ids:
rows = []
for mapping_id in dict.fromkeys(mapping_ids):
row = await mapping_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if row is None or not row.get("wiki_index_enabled", True):
raise HTTPException(status_code=404, detail="Knowledge base not found")
rows.append(row)
else:
rows = [row for row in await mapping_store.list_visible(user_id, scope="all", is_admin=is_admin) if row.get("wiki_index_enabled", True)]
return rows, user_id, is_admin
@router.post("/wiki-vector-search")
async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
config, search, _ = _services(request)
if not config.internal_search_enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
try:
result = await search.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
if not body.include_content:
for item in result["results"]:
item.pop("content", None)
item.pop("snippet", None)
for section in item["matched_sections"]:
section.pop("content", None)
return result
@router.get("/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}")
async def local_wiki_page(
request: Request,
mapping_id: str,
slug: str,
include_provenance: bool = Query(default=False),
include_archived: bool = Query(default=False),
) -> dict[str, Any]:
_, _, index_store = _services(request)
user_id, is_admin = await _actor(request)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if mapping is None:
raise HTTPException(status_code=404, detail="Wiki page not found")
page = await index_store.get_page(mapping_id, slug.strip("/"))
if page is None or page.get("is_remote_deleted"):
raise HTTPException(status_code=404, detail="Wiki page not found")
owner_or_admin = is_admin or str(mapping.get("owner_user_id")) == user_id
if page.get("status") == "archived" and not (include_archived and owner_or_admin):
raise HTTPException(status_code=404, detail="Wiki page not found")
if page.get("status") != "published" and not owner_or_admin:
raise HTTPException(status_code=404, detail="Wiki page not found")
result = {
"id": page["id"],
"knowledge_base_id": mapping_id,
"title": page.get("title") or "",
"slug": page["slug"],
"summary": page.get("summary") or "",
"content": page.get("content_md") or "",
"page_type": page.get("page_type") or "article",
"status": page.get("status"),
"aliases": page.get("aliases") or [],
"category_path": page.get("category_path") or [],
"parent_slug": page.get("parent_slug"),
"wiki_path": page.get("wiki_path"),
"page_version": page.get("page_version") or 0,
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"created_at": page.get("remote_created_at") or page.get("created_at"),
}
if owner_or_admin:
result.update({"index_status": page.get("index_status"), "indexed_at": page.get("indexed_at")})
if include_provenance:
result.update({"source_refs": page.get("source_refs") or [], "chunk_refs": page.get("chunk_refs") or []})
return result
@router.get("/wiki-index/status")
async def wiki_index_status(request: Request) -> dict[str, Any]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
config = request.app.state.config.llmwiki.local_wiki_index
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
states = {row["knowledge_base_mapping_id"]: row for row in await index_store.list_index_status()}
items = []
for mapping in mappings:
state = states.get(mapping["id"], {"state": "not_synced", "index_revision": 0})
items.append(
{
**state,
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
"external_search_enabled": mapping.get("external_search_enabled", False),
}
)
return {
"enabled": config.enabled,
"sync_enabled": config.sync_enabled,
"internal_search_enabled": config.internal_search_enabled,
"embedding_model": config.embedding.model,
"embedding_fingerprint": (
request.app.state.llmwiki_embedding.fingerprint[:12]
if getattr(request.app.state, "llmwiki_embedding", None) is not None
else None
),
"knowledge_bases": items,
}
async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
service = getattr(app.state, "llmwiki_sync_service", None)
if service is None:
return
mappings = await app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
wanted = set(mapping_ids)
for mapping in mappings:
if mapping.get("wiki_index_enabled", True) and (not wanted or mapping["id"] in wanted):
try:
await service.sync_mapping(mapping, force=force)
except Exception:
continue
async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
service = getattr(app.state, "llmwiki_sync_service", None)
if service is None:
return
try:
await service.sync_mapping(mapping, force=force)
except Exception:
return
@router.post("/wiki-index/sync", status_code=202)
async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
_services(request)
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
job_id = str(uuid4())
spawn_local_wiki_index_job(request.app, _run_sync(request.app, body.knowledge_base_ids, body.force))
return {"job_id": job_id, "status": "accepted"}
@router.post("/wiki-index/rebuild", status_code=202)
async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
_, _, index_store = _services(request)
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
ids = body.knowledge_base_ids or [row["id"] for row in mappings if row.get("wiki_index_enabled", True)]
fingerprint = request.app.state.llmwiki_embedding.fingerprint
await index_store.mark_rebuild_required(ids, fingerprint)
job_id = str(uuid4())
spawn_local_wiki_index_job(request.app, _run_sync(request.app, ids, True))
return {"job_id": job_id, "status": "accepted", "reason": body.reason}
@router.post("/wiki-index/knowledge-bases/{mapping_id}/sync", status_code=202)
async def sync_one_wiki_index(request: Request, mapping_id: str, force: bool = Query(default=False)) -> dict[str, Any]:
return await sync_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=force))
@router.post("/wiki-index/knowledge-bases/{mapping_id}/rebuild", status_code=202)
async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str = Query(default="manual", max_length=255)) -> dict[str, Any]:
return await rebuild_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=True, reason=reason))
@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
user_id, is_admin = await _actor(request)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if mapping is None:
raise HTTPException(status_code=404, detail="Knowledge base not found")
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
states = await index_store.list_index_status([mapping_id])
state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
embedding = getattr(request.app.state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
return {
**state,
"knowledge_base_id": mapping_id,
"local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
}
@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
config = request.app.state.config.llmwiki.local_wiki_index
if not config.enabled:
raise HTTPException(
status_code=409,
detail={"code": "WIKI_INDEX_DISABLED", "message": "本地 Wiki 向量化未在配置中启用"},
)
service = getattr(request.app.state, "llmwiki_sync_service", None)
if service is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_VECTORIZATION_UNAVAILABLE"})
user_id, is_admin = await _actor(request)
mapping_store = request.app.state.llmwiki_store
mapping = await mapping_store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
if mapping is None:
raise HTTPException(status_code=404, detail="Knowledge base not found")
if not mapping.get("wiki_index_enabled", True):
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
job_id = str(uuid4())
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
@router.patch("/knowledge-bases/{mapping_id}/wiki-index-settings")
async def update_wiki_index_settings(request: Request, mapping_id: str, body: WikiIndexSettingsRequest) -> dict[str, Any]:
user_id, is_admin = await _actor(request)
store = request.app.state.llmwiki_store
current = await store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
if current is None:
raise HTTPException(status_code=404, detail="Knowledge base not found")
if body.external_search_enabled and str(current.get("owner_user_id")) == "system" and str(current.get("name") or "").strip() == "对话沉淀":
raise HTTPException(status_code=403, detail="Conversation deposit can never be exposed externally")
updated = await store.update_mapping(mapping_id, wiki_index_enabled=body.wiki_index_enabled, external_search_enabled=body.external_search_enabled)
return {"knowledge_base_id": mapping_id, "wiki_index_enabled": updated.get("wiki_index_enabled", True), "external_search_enabled": updated.get("external_search_enabled", False)}