feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
@@ -49,6 +49,8 @@ from app.gateway.routers import (
|
||||
light_apps,
|
||||
llm_metrics,
|
||||
llmwiki,
|
||||
llmwiki_index,
|
||||
external_llmwiki,
|
||||
mcp,
|
||||
memory,
|
||||
menu_overrides,
|
||||
@@ -989,6 +991,22 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Run Alembic migrations before anything else touches the schema.
|
||||
await _run_db_migrations()
|
||||
|
||||
# Local Wiki mirror/index scans are opt-in and run outside request
|
||||
# handling. Each knowledge base still uses a database lease, so this is
|
||||
# safe when multiple Gateway workers start the same scheduler.
|
||||
app.state.llmwiki_index_scheduler_task = None
|
||||
local_wiki_index = app.state.config.llmwiki.local_wiki_index
|
||||
if (
|
||||
local_wiki_index.enabled
|
||||
and local_wiki_index.sync_enabled
|
||||
and getattr(app.state, "llmwiki_sync_service", None) is not None
|
||||
):
|
||||
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
|
||||
|
||||
app.state.llmwiki_index_scheduler_task = asyncio.create_task(
|
||||
local_wiki_index_scheduler_loop(app)
|
||||
)
|
||||
|
||||
# ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
|
||||
try:
|
||||
from deerflow.config.memory_config import load_recall_budget_cache
|
||||
@@ -1171,6 +1189,21 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.exception("Scheduled task service failed to start")
|
||||
|
||||
yield
|
||||
|
||||
llmwiki_index_scheduler_task = getattr(app.state, "llmwiki_index_scheduler_task", None)
|
||||
if llmwiki_index_scheduler_task is not None:
|
||||
llmwiki_index_scheduler_task.cancel()
|
||||
try:
|
||||
await llmwiki_index_scheduler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Local Wiki index scheduler shutdown error")
|
||||
llmwiki_index_jobs = list(getattr(app.state, "llmwiki_index_jobs", set()))
|
||||
for job in llmwiki_index_jobs:
|
||||
job.cancel()
|
||||
if llmwiki_index_jobs:
|
||||
await asyncio.gather(*llmwiki_index_jobs, return_exceptions=True)
|
||||
|
||||
# ?? curator ???
|
||||
curator_task = getattr(app.state, "curator_scheduler_task", None)
|
||||
@@ -1457,6 +1490,8 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# DeerFlow-native LLMWiki management/search BFF (conditionally backed by WeKnora).
|
||||
app.include_router(llmwiki.router)
|
||||
app.include_router(llmwiki_index.router)
|
||||
app.include_router(external_llmwiki.router)
|
||||
|
||||
# Memory API is mounted at /api/memory
|
||||
app.include_router(memory.router)
|
||||
@@ -1654,9 +1689,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# --- ?????????? /page/workspace/roundtable/planning?---
|
||||
# ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
|
||||
# ??? frontend-web/docs/multi-agent-backend-dev.md
|
||||
app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
|
||||
app.include_router(intent.router) # Step 1?/api/intent/init + /stream
|
||||
app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
|
||||
app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
|
||||
app.include_router(intent.router) # Step 1?/api/intent/init + /stream
|
||||
app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
|
||||
# Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
|
||||
|
||||
# Health probes are always unauthenticated (see
|
||||
|
||||
@@ -9,6 +9,8 @@ owner filtering works automatically via the sentinel pattern.
|
||||
Fine-grained permission checks remain in authz.py decorators.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
|
||||
@@ -81,6 +83,30 @@ def _is_public(path: str) -> bool:
|
||||
return any(path.startswith(prefix) for prefix in _PUBLIC_PATH_PREFIXES)
|
||||
|
||||
|
||||
def _is_external_llmwiki(path: str) -> bool:
|
||||
return path.startswith("/api/external/llmwiki/")
|
||||
|
||||
|
||||
def _authorize_external_llmwiki(request: Request) -> JSONResponse | None:
|
||||
"""Authenticate the isolated service-to-service namespace before routing."""
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
config = get_app_config().llmwiki.local_wiki_index.external_api
|
||||
supplied = request.headers.get("X-API-Key", "")
|
||||
valid = bool(config.enabled and supplied) and any(
|
||||
hmac.compare_digest(supplied, candidate)
|
||||
for candidate in (config.api_key, config.previous_api_key)
|
||||
if candidate
|
||||
)
|
||||
if not valid:
|
||||
return JSONResponse(status_code=401, content={"detail": "Invalid API key"})
|
||||
request.state.external_api_key_fingerprint = hashlib.sha256(
|
||||
supplied.encode("utf-8")
|
||||
).hexdigest()[:8]
|
||||
return None
|
||||
|
||||
|
||||
def _is_weknora_embed_proxy_request(request: Request) -> bool:
|
||||
"""Let the WeKnora embed proxy verify its own signed iframe cookie.
|
||||
|
||||
@@ -137,7 +163,11 @@ class AuthMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
# Match the application-relative path so a reverse-proxy prefix
|
||||
# (e.g. /xxx/api/public/...) does not defeat the public whitelist.
|
||||
if _is_public(app_relative_path(request)):
|
||||
relative_path = app_relative_path(request)
|
||||
if _is_external_llmwiki(relative_path):
|
||||
rejected = _authorize_external_llmwiki(request)
|
||||
return rejected if rejected is not None else await call_next(request)
|
||||
if _is_public(relative_path):
|
||||
return await call_next(request)
|
||||
if _is_weknora_embed_proxy_request(request):
|
||||
return await call_next(request)
|
||||
|
||||
@@ -308,8 +308,61 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from deerflow.integrations.weknora.runtime import register_llmwiki_store
|
||||
from deerflow.persistence.llmwiki import make_llmwiki_store
|
||||
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
|
||||
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
|
||||
# migrations and status inspection work while feature flags are off;
|
||||
# network/search services only exist when explicitly enabled.
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
|
||||
|
||||
app.state.llmwiki_index_store = make_llmwiki_index_store(sf)
|
||||
app.state.llmwiki_embedding = None
|
||||
app.state.llmwiki_vector_cache = None
|
||||
app.state.llmwiki_vector_search = None
|
||||
app.state.llmwiki_sync_service = None
|
||||
app.state.llmwiki_retrieval_service = None
|
||||
local_index_config = config.llmwiki.local_wiki_index
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
|
||||
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
||||
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
|
||||
if local_index_config.enabled:
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
|
||||
app.state.llmwiki_vector_cache = WikiVectorCache(
|
||||
app.state.llmwiki_index_store,
|
||||
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
|
||||
revision_check_seconds=local_index_config.cache_revision_check_seconds,
|
||||
)
|
||||
app.state.llmwiki_vector_search = WikiVectorSearchService(local_index_config, app.state.llmwiki_embedding, app.state.llmwiki_vector_cache)
|
||||
if weknora_client is not None:
|
||||
app.state.llmwiki_sync_service = WikiSyncService(
|
||||
local_index_config,
|
||||
app.state.llmwiki_index_store,
|
||||
app.state.llmwiki_embedding,
|
||||
app.state.llmwiki_vector_cache,
|
||||
weknora_client,
|
||||
)
|
||||
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
|
||||
app.state.llmwiki_retrieval_service = WikiRetrievalService(
|
||||
app.state.llmwiki_index_store,
|
||||
vector_search=(app.state.llmwiki_vector_search if local_index_config.internal_search_enabled else None),
|
||||
client=weknora_client,
|
||||
)
|
||||
register_local_wiki_index_runtime(
|
||||
store=app.state.llmwiki_index_store,
|
||||
search=app.state.llmwiki_vector_search,
|
||||
sync=app.state.llmwiki_sync_service,
|
||||
retrieval=app.state.llmwiki_retrieval_service,
|
||||
)
|
||||
|
||||
from deerflow.persistence.taskcop_tasks import make_taskcop_task_store
|
||||
|
||||
@@ -499,8 +552,14 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
from deerflow.persistence.notifications import set_notification_store
|
||||
finally:
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
|
||||
register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
|
||||
llmwiki_embedding = getattr(app.state, "llmwiki_embedding", None)
|
||||
if llmwiki_embedding is not None:
|
||||
await llmwiki_embedding.close()
|
||||
from deerflow.persistence.notifications import set_notification_store
|
||||
|
||||
set_notification_store(None)
|
||||
dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Best-effort periodic full scans for the DeerFlow-local Wiki index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Coroutine
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
|
||||
"""Keep fire-and-forget index jobs strongly referenced until completion."""
|
||||
|
||||
jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None)
|
||||
if jobs is None:
|
||||
jobs = set()
|
||||
app.state.llmwiki_index_jobs = jobs
|
||||
task = asyncio.create_task(coroutine)
|
||||
jobs.add(task)
|
||||
task.add_done_callback(jobs.discard)
|
||||
return task
|
||||
|
||||
|
||||
async def _sync_once(app: FastAPI) -> None:
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
mapping_store = getattr(app.state, "llmwiki_store", None)
|
||||
if service is None or mapping_store is None:
|
||||
return
|
||||
config = app.state.config.llmwiki.local_wiki_index
|
||||
mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
|
||||
semaphore = asyncio.Semaphore(config.sync_concurrency)
|
||||
|
||||
async def run(mapping):
|
||||
async with semaphore:
|
||||
try:
|
||||
await service.sync_mapping(mapping)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True)
|
||||
|
||||
await asyncio.gather(*(run(mapping) for mapping in mappings))
|
||||
|
||||
|
||||
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
|
||||
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
|
||||
while True:
|
||||
await _sync_once(app)
|
||||
await asyncio.sleep(interval)
|
||||
@@ -26,8 +26,8 @@ from langchain_core.messages import HumanMessage
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.agents_config import load_agent_config, validate_agent_id
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
@@ -146,6 +146,24 @@ def _truncate(text: str, max_chars: int) -> str:
|
||||
|
||||
|
||||
def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
if result.get("kind") == "wiki_page":
|
||||
sections = list(result.get("matched_sections") or [])
|
||||
content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
|
||||
raw = {**result, "source": mapping.get("name") or "LLMWiki"}
|
||||
return {
|
||||
"index": index,
|
||||
"id": result.get("wiki_page_id") or result.get("id"),
|
||||
"title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
|
||||
"type": "llmwiki_wiki",
|
||||
"snippet": _truncate(content or str(result.get("summary") or ""), 360),
|
||||
"content": content,
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"score": float(result.get("score") or 0),
|
||||
"url": result.get("url") or "",
|
||||
"skillName": "llmwiki_search",
|
||||
"mode": "citation",
|
||||
"raw": raw,
|
||||
}
|
||||
chunk_id = str(result.get("chunk_id") or "")
|
||||
title = str(result.get("title") or result.get("filename") or mapping.get("name") or f"LLMWiki 片段 {index}")
|
||||
content = str(result.get("content") or "")
|
||||
@@ -175,10 +193,7 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
|
||||
|
||||
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
if not sources:
|
||||
return (
|
||||
"当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。"
|
||||
"请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
)
|
||||
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
|
||||
lines = [
|
||||
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
@@ -187,7 +202,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
"",
|
||||
f"用户问题:{query}",
|
||||
"",
|
||||
f"<llmwiki_reference_batch source_count=\"{len(sources)}\">",
|
||||
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
|
||||
]
|
||||
for source in sources:
|
||||
meta = " / ".join(
|
||||
@@ -203,7 +218,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
lines.append(f"来源:{meta}")
|
||||
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
|
||||
lines.append("")
|
||||
lines.append("</llmwiki_reference_batch>")
|
||||
lines.append("</llmwiki_wiki_reference_batch>")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
@@ -218,7 +233,8 @@ async def build_llmwiki_rag_context(
|
||||
return LlmWikiRagContext([], 0, "", [], "")
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
if not runtime.weknora_enabled:
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
store = _store(request)
|
||||
if store is None:
|
||||
@@ -235,29 +251,25 @@ async def build_llmwiki_rag_context(
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
remote_to_local = {str(row["weknora_id"]): row for row in rows}
|
||||
|
||||
try:
|
||||
results = await build_weknora_client(runtime).search(query, [str(row["weknora_id"]) for row in rows])
|
||||
except WeKnoraError as exc:
|
||||
logger.warning(
|
||||
"LLMWiki pre-retrieval failed for user=%s kb_count=%s: %s",
|
||||
sanitize_log_param(user_id),
|
||||
len(rows),
|
||||
exc,
|
||||
)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
0,
|
||||
f"当前用户选择了 LLMWiki 知识库,但自动检索失败:{exc}。请说明暂时无法检索知识库背景资料,不要编造引用。",
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
|
||||
try:
|
||||
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
|
||||
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
sources: list[dict[str, Any]] = []
|
||||
for result in results:
|
||||
mapping = remote_to_local.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None or not str(result.get("content") or "").strip():
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""API-key protected public Wiki vector search backed only by local data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
router = APIRouter(prefix="/api/external/llmwiki/wiki", tags=["external-llmwiki"])
|
||||
_requests: dict[str, deque[float]] = defaultdict(deque)
|
||||
|
||||
|
||||
class ExternalWikiSearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=10000)
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
limit: int | None = Field(default=None, ge=1, le=100)
|
||||
include_content: bool = False
|
||||
|
||||
|
||||
def _config(request: Request):
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not config.enabled or not config.external_api.enabled:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
return config, config.external_api
|
||||
|
||||
|
||||
def _rate_limit(request: Request, rpm: int) -> None:
|
||||
ip = request.client.host if request.client else "unknown"
|
||||
key = f"{getattr(request.state, 'external_api_key_fingerprint', 'unknown')}:{ip}"
|
||||
now = time.monotonic()
|
||||
queue = _requests[key]
|
||||
while queue and queue[0] <= now - 60:
|
||||
queue.popleft()
|
||||
if len(queue) >= rpm:
|
||||
raise HTTPException(status_code=429, detail="Rate limit exceeded")
|
||||
queue.append(now)
|
||||
|
||||
|
||||
def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
|
||||
return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
|
||||
|
||||
|
||||
async def _external_mappings(request: Request, requested: list[str]) -> list[dict[str, Any]]:
|
||||
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
allowed = {row["id"]: row for row in mappings if row.get("publication_status") == "published" and row.get("external_search_enabled", False) and row.get("wiki_index_enabled", True) and not _is_conversation_deposit(row)}
|
||||
if requested:
|
||||
ids = list(dict.fromkeys(requested))
|
||||
if any(mapping_id not in allowed for mapping_id in ids):
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
return [allowed[mapping_id] for mapping_id in ids]
|
||||
return list(allowed.values())
|
||||
|
||||
|
||||
@router.post("/vector-search")
|
||||
async def external_vector_search(request: Request, body: ExternalWikiSearchRequest) -> dict[str, Any]:
|
||||
_, external = _config(request)
|
||||
_rate_limit(request, external.requests_per_minute)
|
||||
if len(body.query) > external.max_query_chars:
|
||||
raise HTTPException(status_code=413, detail="Query is too large")
|
||||
if len(body.knowledge_base_ids) > external.max_knowledge_bases:
|
||||
raise HTTPException(status_code=413, detail="Too many knowledge bases")
|
||||
limit = body.limit or external.default_results
|
||||
if limit > external.max_results:
|
||||
raise HTTPException(status_code=400, detail="Result limit is too large")
|
||||
mappings = await _external_mappings(request, body.knowledge_base_ids)
|
||||
search = getattr(request.app.state, "llmwiki_vector_search", None)
|
||||
if search is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
try:
|
||||
result = await search.search(body.query, mappings, top_k_pages=limit, external=True)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
# Re-read publication/external flags after vector computation so a switch
|
||||
# turned off during an in-flight request cannot leak a cached result.
|
||||
still_allowed = {row["id"] for row in await _external_mappings(request, [])}
|
||||
result["results"] = [item for item in result["results"] if item.get("knowledge_base_id") in still_allowed]
|
||||
remaining = external.include_content_max_chars if body.include_content else 0
|
||||
items = []
|
||||
for result_item in result["results"]:
|
||||
section = (result_item.get("matched_sections") or [{}])[0]
|
||||
snippet = str(section.get("content") or "")[:1000]
|
||||
content = ""
|
||||
if remaining > 0:
|
||||
content = "\n\n".join(str(item.get("content") or "") for item in result_item.get("matched_sections") or [])[:remaining]
|
||||
remaining -= len(content)
|
||||
item = {
|
||||
"knowledge_base_id": result_item["knowledge_base_id"],
|
||||
"knowledge_base_name": result_item.get("knowledge_base_name") or "",
|
||||
"wiki_slug": result_item["wiki_slug"],
|
||||
"title": result_item.get("title") or "",
|
||||
"summary": result_item.get("summary") or "",
|
||||
"matched_heading": section.get("heading"),
|
||||
"snippet": snippet,
|
||||
"page_type": result_item.get("page_type") or "article",
|
||||
"page_version": result_item.get("page_version") or 0,
|
||||
"updated_at": result_item.get("updated_at"),
|
||||
"score": result_item.get("score") or 0,
|
||||
}
|
||||
if body.include_content:
|
||||
item["content"] = content
|
||||
items.append(item)
|
||||
return {"query": body.query, "mode": "vector", "items": items, "count": len(items), "partial": result.get("partial", False)}
|
||||
|
||||
|
||||
@router.get("/pages/{mapping_id}/{slug:path}")
|
||||
async def external_wiki_page(request: Request, mapping_id: str, slug: str) -> dict[str, Any]:
|
||||
_, external = _config(request)
|
||||
_rate_limit(request, external.requests_per_minute)
|
||||
mappings = await _external_mappings(request, [mapping_id])
|
||||
if not mappings:
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
page = await request.app.state.llmwiki_index_store.get_page(mapping_id, slug.strip("/"))
|
||||
if page is None or page.get("status") != "published" or page.get("index_status") != "ready" or page.get("is_remote_deleted"):
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
# Recheck after reading the page so publication/external revocation takes
|
||||
# effect even when it races this detail request.
|
||||
await _external_mappings(request, [mapping_id])
|
||||
return {
|
||||
"knowledge_base_id": mapping_id,
|
||||
"knowledge_base_name": mappings[0].get("name") or "",
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or "",
|
||||
"summary": page.get("summary") or "",
|
||||
"content": str(page.get("content_md") or "")[: external.include_content_max_chars],
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": page.get("page_version") or 0,
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
}
|
||||
@@ -2,18 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime
|
||||
from io import BytesIO
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import quote, urlencode, urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import JSONResponse, RedirectResponse, Response
|
||||
from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.csrf_middleware import is_secure_request
|
||||
@@ -27,17 +30,28 @@ from app.gateway.llmwiki_deposit import (
|
||||
ensure_conversation_deposit_mapping,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
|
||||
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE as _WEKNORA_EMBED_COOKIE,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_TTL_SECONDS as _WEKNORA_EMBED_TTL_SECONDS,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
WeKnoraEmbedContext,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
sign_weknora_embed_payload as _sign_embed_payload,
|
||||
)
|
||||
from app.gateway.weknora_embed import (
|
||||
verify_weknora_embed_cookie as _verify_embed_cookie,
|
||||
)
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
|
||||
@@ -168,6 +182,8 @@ class SearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=4000)
|
||||
scope: Literal["personal", "public", "all"] = "all"
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
top_k: int = Field(default=8, ge=1, le=50)
|
||||
include_drafts: bool = False
|
||||
|
||||
|
||||
class KnowledgeUrlImport(BaseModel):
|
||||
@@ -223,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
|
||||
return value
|
||||
|
||||
|
||||
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
|
||||
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -575,12 +600,12 @@ async def _authorize_weknora_api_proxy(
|
||||
# page to render upload controls and current-tenant state. They do not
|
||||
# expose other knowledge-base contents.
|
||||
if read_method and tail and tail[0] in {
|
||||
"models",
|
||||
"system",
|
||||
"storage-backends",
|
||||
"vector-stores",
|
||||
"web-search-providers",
|
||||
"mcp-services",
|
||||
"models",
|
||||
"system",
|
||||
"storage-backends",
|
||||
"vector-stores",
|
||||
"web-search-providers",
|
||||
"mcp-services",
|
||||
}:
|
||||
return
|
||||
|
||||
@@ -654,7 +679,7 @@ def _with_public_prefix(prefix: str, path: str) -> str:
|
||||
|
||||
def _without_public_prefix(prefix: str, path: str) -> str:
|
||||
if prefix and path.startswith(f"{prefix}/"):
|
||||
return path[len(prefix):]
|
||||
return path[len(prefix) :]
|
||||
if prefix and path == prefix:
|
||||
return "/"
|
||||
return path
|
||||
@@ -1626,6 +1651,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"description": description,
|
||||
"type": source.get("type") or row.get("kb_type") or "document",
|
||||
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
|
||||
"wiki_index_enabled": row.get("wiki_index_enabled", True),
|
||||
"external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
|
||||
"external_search_updated_at": row.get("external_search_updated_at"),
|
||||
"owner_user_id": row.get("owner_user_id") if is_admin else None,
|
||||
"is_owner": is_owner,
|
||||
"can_write": not is_conversation_deposit and (is_admin or is_owner),
|
||||
@@ -1816,6 +1844,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
|
||||
"""Read every Wiki page and its processed Markdown, never raw documents."""
|
||||
|
||||
summaries: dict[str, dict[str, Any]] = {}
|
||||
page_number = 1
|
||||
while True:
|
||||
listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
|
||||
rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
for item in rows:
|
||||
slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
if slug:
|
||||
summaries[slug] = item
|
||||
total_pages = max(1, int(listing.get("total_pages") or 1))
|
||||
if page_number >= total_pages or not rows:
|
||||
break
|
||||
page_number += 1
|
||||
|
||||
pages: list[dict[str, Any]] = []
|
||||
entries = list(summaries.items())
|
||||
for start in range(0, len(entries), 20):
|
||||
batch = entries[start : start + 20]
|
||||
details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
|
||||
pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
|
||||
return pages
|
||||
|
||||
|
||||
def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
|
||||
parent_slug = str(page.get("parent_slug") or "").strip("/")
|
||||
if parent_slug:
|
||||
ancestors: list[str] = []
|
||||
visited: set[str] = set()
|
||||
current = by_slug.get(parent_slug)
|
||||
while current is not None:
|
||||
slug = str(current.get("slug") or "").strip("/")
|
||||
if not slug or slug in visited:
|
||||
break
|
||||
visited.add(slug)
|
||||
ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
|
||||
current_parent = str(current.get("parent_slug") or "").strip("/")
|
||||
current = by_slug.get(current_parent) if current_parent else None
|
||||
if ancestors:
|
||||
return ancestors
|
||||
|
||||
category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
|
||||
if category_path:
|
||||
return category_path
|
||||
wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
|
||||
if len(wiki_path) > 2:
|
||||
return wiki_path[1:-1]
|
||||
slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
|
||||
return slug_parts[:-1]
|
||||
|
||||
|
||||
def _safe_excel_text(value: Any) -> str:
|
||||
text = str(value or "")
|
||||
return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
|
||||
|
||||
|
||||
def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
|
||||
pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
|
||||
by_slug = {page["slug"]: page for page in pages if page["slug"]}
|
||||
directories = [_wiki_directory_parts(page, by_slug) for page in pages]
|
||||
max_depth = max((len(parts) for parts in directories), default=0)
|
||||
max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
|
||||
columns = [
|
||||
"知识库",
|
||||
"标题",
|
||||
"Slug",
|
||||
"目录完整路径",
|
||||
"所在目录",
|
||||
"父级目录完整路径",
|
||||
*(f"第{level}级目录" for level in range(1, max_depth + 1)),
|
||||
"摘要",
|
||||
*("正文" if index == 1 else f"正文(续{index - 1})" for index in range(1, max_content_parts + 1)),
|
||||
"页面类型",
|
||||
"状态",
|
||||
"别名",
|
||||
"版本",
|
||||
"创建时间",
|
||||
"更新时间",
|
||||
]
|
||||
workbook = Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.title = "Wiki 全量导出"
|
||||
sheet.append(columns)
|
||||
for cell in sheet[1]:
|
||||
cell.font = Font(bold=True, color="FFFFFF")
|
||||
cell.fill = PatternFill("solid", fgColor="1F4E78")
|
||||
cell.alignment = Alignment(horizontal="center", vertical="center")
|
||||
for page, directory in zip(pages, directories, strict=True):
|
||||
content = str(page.get("content") or "")
|
||||
content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
|
||||
content_parts.extend([""] * (max_content_parts - len(content_parts)))
|
||||
row = [
|
||||
knowledge_base_name,
|
||||
page.get("title"),
|
||||
page.get("slug"),
|
||||
" / ".join(directory),
|
||||
directory[-1] if directory else "根目录",
|
||||
" / ".join(directory[:-1]),
|
||||
*directory,
|
||||
*([""] * (max_depth - len(directory))),
|
||||
page.get("summary"),
|
||||
*content_parts,
|
||||
page.get("page_type"),
|
||||
page.get("status"),
|
||||
"、".join(str(item) for item in page.get("aliases") or []),
|
||||
page.get("version"),
|
||||
page.get("created_at"),
|
||||
page.get("updated_at"),
|
||||
]
|
||||
sheet.append([_safe_excel_text(value) for value in row])
|
||||
sheet.freeze_panes = "A2"
|
||||
sheet.auto_filter.ref = sheet.dimensions
|
||||
for column in sheet.columns:
|
||||
letter = column[0].column_letter
|
||||
heading = str(column[0].value or "")
|
||||
sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
|
||||
for cell in column[1:]:
|
||||
cell.alignment = Alignment(vertical="top", wrap_text=True)
|
||||
output = BytesIO()
|
||||
workbook.save(output)
|
||||
output.seek(0)
|
||||
return output
|
||||
|
||||
|
||||
def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
|
||||
target_query = urlencode({"tab": tab}) if tab != "documents" else ""
|
||||
target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
|
||||
@@ -2421,12 +2578,37 @@ async def list_wiki_pages(
|
||||
_raise_upstream(exc)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
|
||||
async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
|
||||
"""Export every processed Wiki page with its current and parent directories."""
|
||||
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
|
||||
client = _client_or_503()
|
||||
try:
|
||||
pages = await _all_wiki_pages(client, str(row["weknora_id"]))
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
try:
|
||||
output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
|
||||
except ImportError as exc:
|
||||
raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
|
||||
filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
|
||||
return StreamingResponse(
|
||||
output,
|
||||
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
|
||||
async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
payload = body.model_dump(exclude_none=True)
|
||||
try:
|
||||
page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _wiki_page_view(page) or page
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
@@ -2454,6 +2636,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
|
||||
_raise_upstream(exc)
|
||||
try:
|
||||
page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _wiki_page_view(page) or page
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
@@ -2464,6 +2647,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
try:
|
||||
await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -2547,32 +2731,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
|
||||
rows.append(row)
|
||||
else:
|
||||
rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
|
||||
service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
|
||||
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
|
||||
try:
|
||||
results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
local_by_remote = {str(row["weknora_id"]): row for row in rows}
|
||||
visible: list[dict[str, Any]] = []
|
||||
for result in results:
|
||||
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
visible_result = (
|
||||
_sanitize_conversation_deposit_value(result)
|
||||
if _is_conversation_deposit_mapping(mapping)
|
||||
else result
|
||||
)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
|
||||
visible.append(
|
||||
{
|
||||
**visible_result,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
}
|
||||
)
|
||||
return {"results": visible, "knowledge_base_count": len(rows)}
|
||||
result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
result["results"] = [
|
||||
_sanitize_conversation_deposit_value(item)
|
||||
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
|
||||
else item
|
||||
for item in result.get("results") or []
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Authenticated APIs for DeerFlow-local Wiki vector search and administration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
|
||||
|
||||
|
||||
class WikiVectorSearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=4000)
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
top_k: int = Field(default=8, ge=1, le=50)
|
||||
include_drafts: bool = False
|
||||
include_content: bool = True
|
||||
|
||||
|
||||
class WikiIndexActionRequest(BaseModel):
|
||||
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
|
||||
force: bool = False
|
||||
reason: str = Field(default="manual", max_length=255)
|
||||
|
||||
|
||||
class WikiIndexSettingsRequest(BaseModel):
|
||||
wiki_index_enabled: bool | None = None
|
||||
external_search_enabled: bool | None = None
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return "default", True
|
||||
return str(user.id), getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
def _services(request: Request):
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
|
||||
search = getattr(request.app.state, "llmwiki_vector_search", None)
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if search is None or index_store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE", "message": "Local Wiki index is unavailable"})
|
||||
return config, search, index_store
|
||||
|
||||
|
||||
async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping_store = request.app.state.llmwiki_store
|
||||
if mapping_ids:
|
||||
rows = []
|
||||
for mapping_id in dict.fromkeys(mapping_ids):
|
||||
row = await mapping_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None or not row.get("wiki_index_enabled", True):
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
rows.append(row)
|
||||
else:
|
||||
rows = [row for row in await mapping_store.list_visible(user_id, scope="all", is_admin=is_admin) if row.get("wiki_index_enabled", True)]
|
||||
return rows, user_id, is_admin
|
||||
|
||||
|
||||
@router.post("/wiki-vector-search")
|
||||
async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
|
||||
config, search, _ = _services(request)
|
||||
if not config.internal_search_enabled:
|
||||
raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
|
||||
rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
|
||||
include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
|
||||
try:
|
||||
result = await search.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
if not body.include_content:
|
||||
for item in result["results"]:
|
||||
item.pop("content", None)
|
||||
item.pop("snippet", None)
|
||||
for section in item["matched_sections"]:
|
||||
section.pop("content", None)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}")
|
||||
async def local_wiki_page(
|
||||
request: Request,
|
||||
mapping_id: str,
|
||||
slug: str,
|
||||
include_provenance: bool = Query(default=False),
|
||||
include_archived: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
_, _, index_store = _services(request)
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
page = await index_store.get_page(mapping_id, slug.strip("/"))
|
||||
if page is None or page.get("is_remote_deleted"):
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
owner_or_admin = is_admin or str(mapping.get("owner_user_id")) == user_id
|
||||
if page.get("status") == "archived" and not (include_archived and owner_or_admin):
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
if page.get("status") != "published" and not owner_or_admin:
|
||||
raise HTTPException(status_code=404, detail="Wiki page not found")
|
||||
result = {
|
||||
"id": page["id"],
|
||||
"knowledge_base_id": mapping_id,
|
||||
"title": page.get("title") or "",
|
||||
"slug": page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"content": page.get("content_md") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"status": page.get("status"),
|
||||
"aliases": page.get("aliases") or [],
|
||||
"category_path": page.get("category_path") or [],
|
||||
"parent_slug": page.get("parent_slug"),
|
||||
"wiki_path": page.get("wiki_path"),
|
||||
"page_version": page.get("page_version") or 0,
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"created_at": page.get("remote_created_at") or page.get("created_at"),
|
||||
}
|
||||
if owner_or_admin:
|
||||
result.update({"index_status": page.get("index_status"), "indexed_at": page.get("indexed_at")})
|
||||
if include_provenance:
|
||||
result.update({"source_refs": page.get("source_refs") or [], "chunk_refs": page.get("chunk_refs") or []})
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/wiki-index/status")
|
||||
async def wiki_index_status(request: Request) -> dict[str, Any]:
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
states = {row["knowledge_base_mapping_id"]: row for row in await index_store.list_index_status()}
|
||||
items = []
|
||||
for mapping in mappings:
|
||||
state = states.get(mapping["id"], {"state": "not_synced", "index_revision": 0})
|
||||
items.append(
|
||||
{
|
||||
**state,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
|
||||
"external_search_enabled": mapping.get("external_search_enabled", False),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"enabled": config.enabled,
|
||||
"sync_enabled": config.sync_enabled,
|
||||
"internal_search_enabled": config.internal_search_enabled,
|
||||
"embedding_model": config.embedding.model,
|
||||
"embedding_fingerprint": (
|
||||
request.app.state.llmwiki_embedding.fingerprint[:12]
|
||||
if getattr(request.app.state, "llmwiki_embedding", None) is not None
|
||||
else None
|
||||
),
|
||||
"knowledge_bases": items,
|
||||
}
|
||||
|
||||
|
||||
async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
if service is None:
|
||||
return
|
||||
mappings = await app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
wanted = set(mapping_ids)
|
||||
for mapping in mappings:
|
||||
if mapping.get("wiki_index_enabled", True) and (not wanted or mapping["id"] in wanted):
|
||||
try:
|
||||
await service.sync_mapping(mapping, force=force)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
|
||||
async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
if service is None:
|
||||
return
|
||||
try:
|
||||
await service.sync_mapping(mapping, force=force)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
@router.post("/wiki-index/sync", status_code=202)
|
||||
async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
|
||||
_services(request)
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_sync(request.app, body.knowledge_base_ids, body.force))
|
||||
return {"job_id": job_id, "status": "accepted"}
|
||||
|
||||
|
||||
@router.post("/wiki-index/rebuild", status_code=202)
|
||||
async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
|
||||
_, _, index_store = _services(request)
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
|
||||
ids = body.knowledge_base_ids or [row["id"] for row in mappings if row.get("wiki_index_enabled", True)]
|
||||
fingerprint = request.app.state.llmwiki_embedding.fingerprint
|
||||
await index_store.mark_rebuild_required(ids, fingerprint)
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_sync(request.app, ids, True))
|
||||
return {"job_id": job_id, "status": "accepted", "reason": body.reason}
|
||||
|
||||
|
||||
@router.post("/wiki-index/knowledge-bases/{mapping_id}/sync", status_code=202)
|
||||
async def sync_one_wiki_index(request: Request, mapping_id: str, force: bool = Query(default=False)) -> dict[str, Any]:
|
||||
return await sync_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=force))
|
||||
|
||||
|
||||
@router.post("/wiki-index/knowledge-bases/{mapping_id}/rebuild", status_code=202)
|
||||
async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str = Query(default="manual", max_length=255)) -> dict[str, Any]:
|
||||
return await rebuild_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=True, reason=reason))
|
||||
|
||||
|
||||
@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
|
||||
async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
|
||||
states = await index_store.list_index_status([mapping_id])
|
||||
state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
|
||||
embedding = getattr(request.app.state, "llmwiki_embedding", None)
|
||||
fingerprint = embedding.fingerprint if embedding is not None else None
|
||||
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
|
||||
return {
|
||||
**state,
|
||||
"knowledge_base_id": mapping_id,
|
||||
"local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
|
||||
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
|
||||
"fully_vectorized": fully_vectorized,
|
||||
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
|
||||
async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not config.enabled:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WIKI_INDEX_DISABLED", "message": "本地 Wiki 向量化未在配置中启用"},
|
||||
)
|
||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "WIKI_VECTORIZATION_UNAVAILABLE"})
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping_store = request.app.state.llmwiki_store
|
||||
mapping = await mapping_store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
if not mapping.get("wiki_index_enabled", True):
|
||||
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
|
||||
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
|
||||
|
||||
|
||||
@router.patch("/knowledge-bases/{mapping_id}/wiki-index-settings")
|
||||
async def update_wiki_index_settings(request: Request, mapping_id: str, body: WikiIndexSettingsRequest) -> dict[str, Any]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
store = request.app.state.llmwiki_store
|
||||
current = await store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
|
||||
if current is None:
|
||||
raise HTTPException(status_code=404, detail="Knowledge base not found")
|
||||
if body.external_search_enabled and str(current.get("owner_user_id")) == "system" and str(current.get("name") or "").strip() == "对话沉淀":
|
||||
raise HTTPException(status_code=403, detail="Conversation deposit can never be exposed externally")
|
||||
updated = await store.update_mapping(mapping_id, wiki_index_enabled=body.wiki_index_enabled, external_search_enabled=body.external_search_enabled)
|
||||
return {"knowledge_base_id": mapping_id, "wiki_index_enabled": updated.get("wiki_index_enabled", True), "external_search_enabled": updated.get("external_search_enabled", False)}
|
||||
Reference in new issue
Block a user