sync update from LAN repo (source master-yx @ cf505220)

This commit is contained in:
gozeros committed 2026-10-21 15:11:11 +08:00
1 parent e243ec3a46
commit 19d3cb96dc
114 files changed
+14787 -451

No files matched your search

@@ -13,6 +13,10 @@ from fastapi import APIRouter, File, Form, HTTPException, Query, Request, Upload
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from app.gateway.llmwiki_index_runtime import (
local_encoder_runtime_status,
rebuild_local_wiki_index_runtime,
)
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from deerflow.config.runtime_paths import resolve_path, runtime_home
from deerflow.config.system_settings import load_system_settings, save_system_settings
@@ -53,8 +57,8 @@ async def _actor(request: Request) -> tuple[str, bool]:
return str(user.id), getattr(user, "system_role", None) == "admin"
def _services(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index
async def _services(request: Request):
config = await rebuild_local_wiki_index_runtime(request.app)
if not config.enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
search = getattr(request.app.state, "llmwiki_vector_search", None)
@@ -65,12 +69,12 @@ def _services(request: Request):
def _local_encoder(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index.embedding
embedding = getattr(request.app.state, "llmwiki_embedding", None)
if config.provider != "local":
effective = getattr(request.app.state, "llmwiki_effective_config", None)
if effective is None or effective.embedding.provider != "local":
raise HTTPException(status_code=409, detail={"code": "EMBEDDING_NOT_LOCAL", "message": "当前 embedding provider 不是 local"})
embedding = getattr(request.app.state, "llmwiki_embedding", None)
if embedding is None:
raise HTTPException(status_code=503, detail={"code": "LOCAL_ENCODER_UNAVAILABLE", "message": "本地编码器未配置或未启用"})
raise HTTPException(status_code=503, detail={"code": "LOCAL_ENCODER_UNAVAILABLE", "message": "本地编码器目录未配置、无效或尚未就绪"})
return embedding
@@ -107,33 +111,35 @@ def _copy_encoder_uploads(files: list[UploadFile], paths: list[str], staging: Pa
return total
def _activate_uploaded_encoder(staging: Path) -> Path:
model_root = (runtime_home() / "models").resolve()
def _activate_uploaded_encoder(staging: Path, model_revision: str) -> Path:
model_root = (runtime_home() / "models" / "bge-m3-revisions").resolve()
model_root.mkdir(parents=True, exist_ok=True)
active = (model_root / "bge-m3").resolve()
active = (model_root / model_revision).resolve()
if active.parent != model_root:
raise RuntimeError("编码器目标目录越界")
backup = (model_root / f".bge-m3-backup-{uuid4().hex}").resolve()
if active.exists():
active.replace(backup)
try:
staging.replace(active)
except Exception:
if backup.exists() and not active.exists():
backup.replace(active)
raise
if backup.exists():
shutil.rmtree(backup)
raise RuntimeError("编码器版本目录已存在")
staging.replace(active)
return active
async def _save_encoder_path(request: Request, path: Path) -> dict[str, object]:
embedding = _local_encoder(request)
async def _save_encoder_path(
request: Request,
path: Path,
*,
model_revision: str | None = None,
) -> dict[str, object]:
current = load_system_settings()
encoder_settings = current.local_wiki_encoder.model_copy(update={"model_path": str(path)})
encoder_settings = current.local_wiki_encoder.model_copy(
update={
"mode": "local",
"model_path": str(path),
"model_revision": model_revision or uuid4().hex,
}
)
save_system_settings(current.model_copy(update={"local_wiki_encoder": encoder_settings}))
await embedding.reset_local_encoder(model_path=str(path))
return embedding.local_encoder_status()
await rebuild_local_wiki_index_runtime(request.app, force=True)
return local_encoder_runtime_status(request.app)
@router.get("/local-encoder/status")
@@ -141,7 +147,8 @@ async def local_encoder_status(request: Request) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
return _local_encoder(request).local_encoder_status()
await rebuild_local_wiki_index_runtime(request.app)
return local_encoder_runtime_status(request.app)
@router.post("/local-encoder/start", status_code=202)
@@ -149,8 +156,10 @@ async def start_local_encoder(request: Request) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
await rebuild_local_wiki_index_runtime(request.app)
try:
return _local_encoder(request).start_local_encoder()
_local_encoder(request).start_local_encoder()
return local_encoder_runtime_status(request.app)
except ValueError as exc:
raise HTTPException(status_code=409, detail={"code": "LOCAL_ENCODER_START_REJECTED", "message": str(exc)}) from None
@@ -163,6 +172,20 @@ async def configure_local_encoder(request: Request, body: LocalEncoderConfigRequ
return await _save_encoder_path(request, _validated_encoder_path(body.model_path))
@router.post("/local-encoder/use-configured")
async def use_configured_encoder(request: Request) -> dict[str, object]:
"""Restore the embedding provider declared in config.yaml without restart."""
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
current = load_system_settings()
encoder_settings = current.local_wiki_encoder.model_copy(update={"mode": "configured"})
save_system_settings(current.model_copy(update={"local_wiki_encoder": encoder_settings}))
await rebuild_local_wiki_index_runtime(request.app, force=True)
return local_encoder_runtime_status(request.app)
@router.post("/local-encoder/upload")
async def upload_local_encoder(
request: Request,
@@ -191,9 +214,17 @@ async def upload_local_encoder(
try:
uploaded_bytes = await asyncio.to_thread(_copy_encoder_uploads, files, paths, staging)
_validated_encoder_path(str(staging))
await _local_encoder(request).reset_local_encoder()
active = await asyncio.to_thread(_activate_uploaded_encoder, staging)
status = await _save_encoder_path(request, active)
model_revision = uuid4().hex
active = await asyncio.to_thread(
_activate_uploaded_encoder,
staging,
model_revision,
)
status = await _save_encoder_path(
request,
active,
model_revision=model_revision,
)
return {**status, "uploaded_files": len(files), "uploaded_bytes": uploaded_bytes}
except HTTPException:
raise
@@ -224,7 +255,7 @@ async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tupl
@router.post("/wiki-vector-search")
async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
config, search, _ = _services(request)
config, search, _ = await _services(request)
if not config.internal_search_enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
@@ -250,7 +281,7 @@ async def local_wiki_page(
include_provenance: bool = Query(default=False),
include_archived: bool = Query(default=False),
) -> dict[str, Any]:
_, _, index_store = _services(request)
_, _, index_store = await _services(request)
user_id, is_admin = await _actor(request)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if mapping is None:
@@ -292,7 +323,7 @@ async def wiki_index_status(request: Request) -> dict[str, Any]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
config = request.app.state.config.llmwiki.local_wiki_index
config = await rebuild_local_wiki_index_runtime(request.app)
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
@@ -352,7 +383,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
@router.post("/wiki-index/sync", status_code=202)
async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
_services(request)
await _services(request)
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
@@ -363,7 +394,7 @@ async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dic
@router.post("/wiki-index/rebuild", status_code=202)
async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
_, _, index_store = _services(request)
_, _, index_store = await _services(request)
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
@@ -388,6 +419,7 @@ async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str
@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
config = await rebuild_local_wiki_index_runtime(request.app)
user_id, is_admin = await _actor(request)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if mapping is None:
@@ -430,7 +462,7 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
return {
**state,
"knowledge_base_id": mapping_id,
"local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
"local_index_enabled": bool(config.enabled),
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
@@ -448,7 +480,7 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
config = request.app.state.config.llmwiki.local_wiki_index
config = await rebuild_local_wiki_index_runtime(request.app)
if not config.enabled:
raise HTTPException(
status_code=409,