feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
@@ -0,0 +1,133 @@
|
||||
"""API-key protected public Wiki vector search backed only by local data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
router = APIRouter(prefix="/api/external/llmwiki/wiki", tags=["external-llmwiki"])
|
||||
_requests: dict[str, deque[float]] = defaultdict(deque)
|
||||
|
||||
|
||||
class ExternalWikiSearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=10000)
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
limit: int | None = Field(default=None, ge=1, le=100)
|
||||
include_content: bool = False
|
||||
|
||||
|
||||
def _config(request: Request):
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not config.enabled or not config.external_api.enabled:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
return config, config.external_api
|
||||
|
||||
|
||||
def _rate_limit(request: Request, rpm: int) -> None:
|
||||
ip = request.client.host if request.client else "unknown"
|
||||
key = f"{getattr(request.state, 'external_api_key_fingerprint', 'unknown')}:{ip}"
|
||||
now = time.monotonic()
|
||||
queue = _requests[key]
|
||||
while queue and queue[0] <= now - 60:
|
||||
queue.popleft()
|
||||
if len(queue) >= rpm:
|
||||
raise HTTPException(status_code=429, detail="Rate limit exceeded")
|
||||
queue.append(now)
|
||||
|
||||
|
||||
def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
|
||||
return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
|
||||
|
||||
|
||||
async def _external_mappings(request: Request, requested: list[str]) -> list[dict[str, Any]]:
|
||||
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
allowed = {row["id"]: row for row in mappings if row.get("publication_status") == "published" and row.get("external_search_enabled", False) and row.get("wiki_index_enabled", True) and not _is_conversation_deposit(row)}
|
||||
if requested:
|
||||
ids = list(dict.fromkeys(requested))
|
||||
if any(mapping_id not in allowed for mapping_id in ids):
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
return [allowed[mapping_id] for mapping_id in ids]
|
||||
return list(allowed.values())
|
||||
|
||||
|
||||
@router.post("/vector-search")
|
||||
async def external_vector_search(request: Request, body: ExternalWikiSearchRequest) -> dict[str, Any]:
|
||||
_, external = _config(request)
|
||||
_rate_limit(request, external.requests_per_minute)
|
||||
if len(body.query) > external.max_query_chars:
|
||||
raise HTTPException(status_code=413, detail="Query is too large")
|
||||
if len(body.knowledge_base_ids) > external.max_knowledge_bases:
|
||||
raise HTTPException(status_code=413, detail="Too many knowledge bases")
|
||||
limit = body.limit or external.default_results
|
||||
if limit > external.max_results:
|
||||
raise HTTPException(status_code=400, detail="Result limit is too large")
|
||||
mappings = await _external_mappings(request, body.knowledge_base_ids)
|
||||
search = getattr(request.app.state, "llmwiki_vector_search", None)
|
||||
if search is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
try:
|
||||
result = await search.search(body.query, mappings, top_k_pages=limit, external=True)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
# Re-read publication/external flags after vector computation so a switch
|
||||
# turned off during an in-flight request cannot leak a cached result.
|
||||
still_allowed = {row["id"] for row in await _external_mappings(request, [])}
|
||||
result["results"] = [item for item in result["results"] if item.get("knowledge_base_id") in still_allowed]
|
||||
remaining = external.include_content_max_chars if body.include_content else 0
|
||||
items = []
|
||||
for result_item in result["results"]:
|
||||
section = (result_item.get("matched_sections") or [{}])[0]
|
||||
snippet = str(section.get("content") or "")[:1000]
|
||||
content = ""
|
||||
if remaining > 0:
|
||||
content = "\n\n".join(str(item.get("content") or "") for item in result_item.get("matched_sections") or [])[:remaining]
|
||||
remaining -= len(content)
|
||||
item = {
|
||||
"knowledge_base_id": result_item["knowledge_base_id"],
|
||||
"knowledge_base_name": result_item.get("knowledge_base_name") or "",
|
||||
"wiki_slug": result_item["wiki_slug"],
|
||||
"title": result_item.get("title") or "",
|
||||
"summary": result_item.get("summary") or "",
|
||||
"matched_heading": section.get("heading"),
|
||||
"snippet": snippet,
|
||||
"page_type": result_item.get("page_type") or "article",
|
||||
"page_version": result_item.get("page_version") or 0,
|
||||
"updated_at": result_item.get("updated_at"),
|
||||
"score": result_item.get("score") or 0,
|
||||
}
|
||||
if body.include_content:
|
||||
item["content"] = content
|
||||
items.append(item)
|
||||
return {"query": body.query, "mode": "vector", "items": items, "count": len(items), "partial": result.get("partial", False)}
|
||||
|
||||
|
||||
@router.get("/pages/{mapping_id}/{slug:path}")
|
||||
async def external_wiki_page(request: Request, mapping_id: str, slug: str) -> dict[str, Any]:
|
||||
_, external = _config(request)
|
||||
_rate_limit(request, external.requests_per_minute)
|
||||
mappings = await _external_mappings(request, [mapping_id])
|
||||
if not mappings:
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
page = await request.app.state.llmwiki_index_store.get_page(mapping_id, slug.strip("/"))
|
||||
if page is None or page.get("status") != "published" or page.get("index_status") != "ready" or page.get("is_remote_deleted"):
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
# Recheck after reading the page so publication/external revocation takes
|
||||
# effect even when it races this detail request.
|
||||
await _external_mappings(request, [mapping_id])
|
||||
return {
|
||||
"knowledge_base_id": mapping_id,
|
||||
"knowledge_base_name": mappings[0].get("name") or "",
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or "",
|
||||
"summary": page.get("summary") or "",
|
||||
"content": str(page.get("content_md") or "")[: external.include_content_max_chars],
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": page.get("page_version") or 0,
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
}
|
||||
@@ -2,18 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime
|
||||
from io import BytesIO
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import quote, urlencode, urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import JSONResponse, RedirectResponse, Response
|
||||
from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.csrf_middleware import is_secure_request
|
||||
@@ -27,17 +30,28 @@ from app.gateway.llmwiki_deposit import (
|
||||
ensure_conversation_deposit_mapping,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
|
||||
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_TTL_SECONDS as _WEKNORA_EMBED_TTL_SECONDS,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WeKnoraEmbedContext,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
sign_weknora_embed_payload as _sign_embed_payload,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_embed_cookie as _verify_embed_cookie,
|
||||
)
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
|
||||
@@ -168,6 +182,8 @@ class SearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=4000)
|
||||
scope: Literal["personal", "public", "all"] = "all"
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
top_k: int = Field(default=8, ge=1, le=50)
|
||||
include_drafts: bool = False
|
||||
|
||||
|
||||
class KnowledgeUrlImport(BaseModel):
|
||||
@@ -223,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
|
||||
return value
|
||||
|
||||
|
||||
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
|
||||
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -575,12 +600,12 @@ async def _authorize_weknora_api_proxy(
|
||||
# page to render upload controls and current-tenant state. They do not
|
||||
# expose other knowledge-base contents.
|
||||
if read_method and tail and tail[0] in {
|
||||
"models",
|
||||
"system",
|
||||
"storage-backends",
|
||||
"vector-stores",
|
||||
"web-search-providers",
|
||||
"mcp-services",
|
||||
"models",
|
||||
"system",
|
||||
"storage-backends",
|
||||
"vector-stores",
|
||||
"web-search-providers",
|
||||
"mcp-services",
|
||||
}:
|
||||
return
|
||||
|
||||
@@ -654,7 +679,7 @@ def _with_public_prefix(prefix: str, path: str) -> str:
|
||||
|
||||
def _without_public_prefix(prefix: str, path: str) -> str:
|
||||
if prefix and path.startswith(f"{prefix}/"):
|
||||
return path[len(prefix):]
|
||||
return path[len(prefix) :]
|
||||
if prefix and path == prefix:
|
||||
return "/"
|
||||
return path
|
||||
@@ -1626,6 +1651,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"description": description,
|
||||
"type": source.get("type") or row.get("kb_type") or "document",
|
||||
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
|
||||
"wiki_index_enabled": row.get("wiki_index_enabled", True),
|
||||
"external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
|
||||
"external_search_updated_at": row.get("external_search_updated_at"),
|
||||
"owner_user_id": row.get("owner_user_id") if is_admin else None,
|
||||
"is_owner": is_owner,
|
||||
"can_write": not is_conversation_deposit and (is_admin or is_owner),
|
||||
@@ -1816,6 +1844,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
|
||||
"""Read every Wiki page and its processed Markdown, never raw documents."""
|
||||
|
||||
summaries: dict[str, dict[str, Any]] = {}
|
||||
page_number = 1
|
||||
while True:
|
||||
listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
|
||||
rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
for item in rows:
|
||||
slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
if slug:
|
||||
summaries[slug] = item
|
||||
total_pages = max(1, int(listing.get("total_pages") or 1))
|
||||
if page_number >= total_pages or not rows:
|
||||
break
|
||||
page_number += 1
|
||||
|
||||
pages: list[dict[str, Any]] = []
|
||||
entries = list(summaries.items())
|
||||
for start in range(0, len(entries), 20):
|
||||
batch = entries[start : start + 20]
|
||||
details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
|
||||
pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
|
||||
return pages
|
||||
|
||||
|
||||
def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
|
||||
parent_slug = str(page.get("parent_slug") or "").strip("/")
|
||||
if parent_slug:
|
||||
ancestors: list[str] = []
|
||||
visited: set[str] = set()
|
||||
current = by_slug.get(parent_slug)
|
||||
while current is not None:
|
||||
slug = str(current.get("slug") or "").strip("/")
|
||||
if not slug or slug in visited:
|
||||
break
|
||||
visited.add(slug)
|
||||
ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
|
||||
current_parent = str(current.get("parent_slug") or "").strip("/")
|
||||
current = by_slug.get(current_parent) if current_parent else None
|
||||
if ancestors:
|
||||
return ancestors
|
||||
|
||||
category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
|
||||
if category_path:
|
||||
return category_path
|
||||
wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
|
||||
if len(wiki_path) > 2:
|
||||
return wiki_path[1:-1]
|
||||
slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
|
||||
return slug_parts[:-1]
|
||||
|
||||
|
||||
def _safe_excel_text(value: Any) -> str:
|
||||
text = str(value or "")
|
||||
return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
|
||||
|
||||
|
||||
def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
|
||||
pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
|
||||
by_slug = {page["slug"]: page for page in pages if page["slug"]}
|
||||
directories = [_wiki_directory_parts(page, by_slug) for page in pages]
|
||||
max_depth = max((len(parts) for parts in directories), default=0)
|
||||
max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
|
||||
columns = [
|
||||
"知识库",
|
||||
"标题",
|
||||
"Slug",
|
||||
"目录完整路径",
|
||||
"所在目录",
|
||||
"父级目录完整路径",
|
||||
*(f"第{level}级目录" for level in range(1, max_depth + 1)),
|
||||
"摘要",
|
||||
*("正文" if index == 1 else f"正文(续{index - 1})" for index in range(1, max_content_parts + 1)),
|
||||
"页面类型",
|
||||
"状态",
|
||||
"别名",
|
||||
"版本",
|
||||
"创建时间",
|
||||
"更新时间",
|
||||
]
|
||||
workbook = Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.title = "Wiki 全量导出"
|
||||
sheet.append(columns)
|
||||
for cell in sheet[1]:
|
||||
cell.font = Font(bold=True, color="FFFFFF")
|
||||
cell.fill = PatternFill("solid", fgColor="1F4E78")
|
||||
cell.alignment = Alignment(horizontal="center", vertical="center")
|
||||
for page, directory in zip(pages, directories, strict=True):
|
||||
content = str(page.get("content") or "")
|
||||
content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
|
||||
content_parts.extend([""] * (max_content_parts - len(content_parts)))
|
||||
row = [
|
||||
knowledge_base_name,
|
||||
page.get("title"),
|
||||
page.get("slug"),
|
||||
" / ".join(directory),
|
||||
directory[-1] if directory else "根目录",
|
||||
" / ".join(directory[:-1]),
|
||||
*directory,
|
||||
*([""] * (max_depth - len(directory))),
|
||||
page.get("summary"),
|
||||
*content_parts,
|
||||
page.get("page_type"),
|
||||
page.get("status"),
|
||||
"、".join(str(item) for item in page.get("aliases") or []),
|
||||
page.get("version"),
|
||||
page.get("created_at"),
|
||||
page.get("updated_at"),
|
||||
]
|
||||
sheet.append([_safe_excel_text(value) for value in row])
|
||||
sheet.freeze_panes = "A2"
|
||||
sheet.auto_filter.ref = sheet.dimensions
|
||||
for column in sheet.columns:
|
||||
letter = column[0].column_letter
|
||||
heading = str(column[0].value or "")
|
||||
sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
|
||||
for cell in column[1:]:
|
||||
cell.alignment = Alignment(vertical="top", wrap_text=True)
|
||||
output = BytesIO()
|
||||
workbook.save(output)
|
||||
output.seek(0)
|
||||
return output
|
||||
|
||||
|
||||
def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
|
||||
target_query = urlencode({"tab": tab}) if tab != "documents" else ""
|
||||
target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
|
||||
@@ -2421,12 +2578,37 @@ async def list_wiki_pages(
|
||||
_raise_upstream(exc)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
|
||||
async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
|
||||
"""Export every processed Wiki page with its current and parent directories."""
|
||||
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
|
||||
client = _client_or_503()
|
||||
try:
|
||||
pages = await _all_wiki_pages(client, str(row["weknora_id"]))
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
try:
|
||||
output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
|
||||
except ImportError as exc:
|
||||
raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
|
||||
filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
|
||||
return StreamingResponse(
|
||||
output,
|
||||
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
|
||||
async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
payload = body.model_dump(exclude_none=True)
|
||||
try:
|
||||
page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _wiki_page_view(page) or page
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
@@ -2454,6 +2636,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
|
||||
_raise_upstream(exc)
|
||||
try:
|
||||
page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _wiki_page_view(page) or page
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
@@ -2464,6 +2647,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
try:
|
||||
await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -2547,32 +2731,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
|
||||
rows.append(row)
|
||||
else:
|
||||
rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
|
||||
service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
|
||||
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
|
||||
try:
|
||||
results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
local_by_remote = {str(row["weknora_id"]): row for row in rows}
|
||||
visible: list[dict[str, Any]] = []
|
||||
for result in results:
|
||||
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
visible_result = (
|
||||
_sanitize_conversation_deposit_value(result)
|
||||
if _is_conversation_deposit_mapping(mapping)
|
||||
else result
|
||||
)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
|
||||
visible.append(
|
||||
{
|
||||
**visible_result,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
}
|
||||
)
|
||||
return {"results": visible, "knowledge_base_count": len(rows)}
|
||||
result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
result["results"] = [
|
||||
_sanitize_conversation_deposit_value(item)
|
||||
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
|
||||
else item
|
||||
for item in result.get("results") or []
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Authenticated APIs for DeerFlow-local Wiki vector search and administration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
|
||||
|
||||
|
||||
class WikiVectorSearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=4000)
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
top_k: int = Field(default=8, ge=1, le=50)
|
||||
include_drafts: bool = False
|
||||
include_content: bool = True
|
||||
|
||||
|
||||
class WikiIndexActionRequest(BaseModel):
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
force: bool = False
|
||||
reason: str = Field(default="manual", max_length=255)
|
||||
|
||||
|
||||
class WikiIndexSettingsRequest(BaseModel):
|
||||
wiki_index_enabled: bool | None = None
|
||||
external_search_enabled: bool | None = None
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return "default", True
|
||||
return str(user.id), getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
def _services(request: Request):
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
|
||||
search = getattr(request.app.state, "llmwiki_vector_search", None)
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if search is None or index_store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE", "message": "Local Wiki index is unavailable"})
|
||||
return config, search, index_store
|
||||
|
||||
|
||||
async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping_store = request.app.state.llmwiki_store
|
||||
if mapping_ids:
|
||||
rows = []
|
||||
for mapping_id in dict.fromkeys(mapping_ids):
|
||||
row = await mapping_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None or not row.get("wiki_index_enabled", True):
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
rows.append(row)
|
||||
else:
|
||||
rows = [row for row in await mapping_store.list_visible(user_id, scope="all", is_admin=is_admin) if row.get("wiki_index_enabled", True)]
|
||||
return rows, user_id, is_admin
|
||||
|
||||
|
||||
@router.post("/wiki-vector-search")
|
||||
async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
|
||||
config, search, _ = _services(request)
|
||||
if not config.internal_search_enabled:
|
||||
raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
|
||||
rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
|
||||
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
|
||||
try:
|
||||
result = await search.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
if not body.include_content:
|
||||
for item in result["results"]:
|
||||
item.pop("content", None)
|
||||
item.pop("snippet", None)
|
||||
for section in item["matched_sections"]:
|
||||
section.pop("content", None)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}")
|
||||
async def local_wiki_page(
|
||||
request: Request,
|
||||
mapping_id: str,
|
||||
slug: str,
|
||||
include_provenance: bool = Query(default=False),
|
||||
include_archived: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
_, _, index_store = _services(request)
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
page = await index_store.get_page(mapping_id, slug.strip("/"))
|
||||
if page is None or page.get("is_remote_deleted"):
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
owner_or_admin = is_admin or str(mapping.get("owner_user_id")) == user_id
|
||||
if page.get("status") == "archived" and not (include_archived and owner_or_admin):
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
if page.get("status") != "published" and not owner_or_admin:
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
result = {
|
||||
"id": page["id"],
|
||||
"knowledge_base_id": mapping_id,
|
||||
"title": page.get("title") or "",
|
||||
"slug": page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"content": page.get("content_md") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"status": page.get("status"),
|
||||
"aliases": page.get("aliases") or [],
|
||||
"category_path": page.get("category_path") or [],
|
||||
"parent_slug": page.get("parent_slug"),
|
||||
"wiki_path": page.get("wiki_path"),
|
||||
"page_version": page.get("page_version") or 0,
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"created_at": page.get("remote_created_at") or page.get("created_at"),
|
||||
}
|
||||
if owner_or_admin:
|
||||
result.update({"index_status": page.get("index_status"), "indexed_at": page.get("indexed_at")})
|
||||
if include_provenance:
|
||||
result.update({"source_refs": page.get("source_refs") or [], "chunk_refs": page.get("chunk_refs") or []})
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/wiki-index/status")
|
||||
async def wiki_index_status(request: Request) -> dict[str, Any]:
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
states = {row["knowledge_base_mapping_id"]: row for row in await index_store.list_index_status()}
|
||||
items = []
|
||||
for mapping in mappings:
|
||||
state = states.get(mapping["id"], {"state": "not_synced", "index_revision": 0})
|
||||
items.append(
|
||||
{
|
||||
**state,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
|
||||
"external_search_enabled": mapping.get("external_search_enabled", False),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"enabled": config.enabled,
|
||||
"sync_enabled": config.sync_enabled,
|
||||
"internal_search_enabled": config.internal_search_enabled,
|
||||
"embedding_model": config.embedding.model,
|
||||
"embedding_fingerprint": (
|
||||
request.app.state.llmwiki_embedding.fingerprint[:12]
|
||||
if getattr(request.app.state, "llmwiki_embedding", None) is not None
|
||||
else None
|
||||
),
|
||||
"knowledge_bases": items,
|
||||
}
|
||||
|
||||
|
||||
async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
if service is None:
|
||||
return
|
||||
mappings = await app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
wanted = set(mapping_ids)
|
||||
for mapping in mappings:
|
||||
if mapping.get("wiki_index_enabled", True) and (not wanted or mapping["id"] in wanted):
|
||||
try:
|
||||
await service.sync_mapping(mapping, force=force)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
|
||||
async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
if service is None:
|
||||
return
|
||||
try:
|
||||
await service.sync_mapping(mapping, force=force)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
@router.post("/wiki-index/sync", status_code=202)
|
||||
async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
|
||||
_services(request)
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_sync(request.app, body.knowledge_base_ids, body.force))
|
||||
return {"job_id": job_id, "status": "accepted"}
|
||||
|
||||
|
||||
@router.post("/wiki-index/rebuild", status_code=202)
|
||||
async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
|
||||
_, _, index_store = _services(request)
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
ids = body.knowledge_base_ids or [row["id"] for row in mappings if row.get("wiki_index_enabled", True)]
|
||||
fingerprint = request.app.state.llmwiki_embedding.fingerprint
|
||||
await index_store.mark_rebuild_required(ids, fingerprint)
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_sync(request.app, ids, True))
|
||||
return {"job_id": job_id, "status": "accepted", "reason": body.reason}
|
||||
|
||||
|
||||
@router.post("/wiki-index/knowledge-bases/{mapping_id}/sync", status_code=202)
|
||||
async def sync_one_wiki_index(request: Request, mapping_id: str, force: bool = Query(default=False)) -> dict[str, Any]:
|
||||
return await sync_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=force))
|
||||
|
||||
|
||||
@router.post("/wiki-index/knowledge-bases/{mapping_id}/rebuild", status_code=202)
|
||||
async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str = Query(default="manual", max_length=255)) -> dict[str, Any]:
|
||||
return await rebuild_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=True, reason=reason))
|
||||
|
||||
|
||||
@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
|
||||
async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
states = await index_store.list_index_status([mapping_id])
|
||||
state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
|
||||
embedding = getattr(request.app.state, "llmwiki_embedding", None)
|
||||
fingerprint = embedding.fingerprint if embedding is not None else None
|
||||
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
|
||||
return {
|
||||
**state,
|
||||
"knowledge_base_id": mapping_id,
|
||||
"local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
|
||||
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
|
||||
"fully_vectorized": fully_vectorized,
|
||||
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
|
||||
async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not config.enabled:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WIKI_INDEX_DISABLED", "message": "本地 Wiki 向量化未在配置中启用"},
|
||||
)
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_VECTORIZATION_UNAVAILABLE"})
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping_store = request.app.state.llmwiki_store
|
||||
mapping = await mapping_store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
if not mapping.get("wiki_index_enabled", True):
|
||||
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
|
||||
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
|
||||
|
||||
|
||||
@router.patch("/knowledge-bases/{mapping_id}/wiki-index-settings")
|
||||
async def update_wiki_index_settings(request: Request, mapping_id: str, body: WikiIndexSettingsRequest) -> dict[str, Any]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
store = request.app.state.llmwiki_store
|
||||
current = await store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
|
||||
if current is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
if body.external_search_enabled and str(current.get("owner_user_id")) == "system" and str(current.get("name") or "").strip() == "对话沉淀":
|
||||
raise HTTPException(status_code=403, detail="Conversation deposit can never be exposed externally")
|
||||
updated = await store.update_mapping(mapping_id, wiki_index_enabled=body.wiki_index_enabled, external_search_enabled=body.external_search_enabled)
|
||||
return {"knowledge_base_id": mapping_id, "wiki_index_enabled": updated.get("wiki_index_enabled", True), "external_search_enabled": updated.get("external_search_enabled", False)}
|
||||
Reference in new issue
Block a user