sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -13,9 +13,14 @@ from fastapi import APIRouter, File, Form, HTTPException, Query, Request, Upload
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from app.gateway.llmwiki_index_runtime import (
local_encoder_runtime_status,
rebuild_local_wiki_index_runtime,
)
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from deerflow.config.runtime_paths import resolve_path, runtime_home
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.local_index.advanced import advanced_search
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
@@ -31,6 +36,15 @@ class WikiVectorSearchRequest(BaseModel):
include_content: bool = True
class WikiRetrievalDebugRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=4000)
date_from: str | None = Field(default=None, max_length=32)
date_to: str | None = Field(default=None, max_length=32)
top_k: int = Field(default=20, ge=1, le=100)
expand_entities: bool | None = None
include_drafts: bool = False
class WikiIndexActionRequest(BaseModel):
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
force: bool = False
@@ -53,8 +67,8 @@ async def _actor(request: Request) -> tuple[str, bool]:
return str(user.id), getattr(user, "system_role", None) == "admin"
def _services(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index
async def _services(request: Request):
config = await rebuild_local_wiki_index_runtime(request.app)
if not config.enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
search = getattr(request.app.state, "llmwiki_vector_search", None)
@@ -65,12 +79,12 @@ def _services(request: Request):
def _local_encoder(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index.embedding
embedding = getattr(request.app.state, "llmwiki_embedding", None)
if config.provider != "local":
effective = getattr(request.app.state, "llmwiki_effective_config", None)
if effective is None or effective.embedding.provider != "local":
raise HTTPException(status_code=409, detail={"code": "EMBEDDING_NOT_LOCAL", "message": "当前 embedding provider 不是 local"})
embedding = getattr(request.app.state, "llmwiki_embedding", None)
if embedding is None:
raise HTTPException(status_code=503, detail={"code": "LOCAL_ENCODER_UNAVAILABLE", "message": "本地编码器未配置或未启用"})
raise HTTPException(status_code=503, detail={"code": "LOCAL_ENCODER_UNAVAILABLE", "message": "本地编码器目录未配置、无效或尚未就绪"})
return embedding
@@ -107,33 +121,35 @@ def _copy_encoder_uploads(files: list[UploadFile], paths: list[str], staging: Pa
return total
def _activate_uploaded_encoder(staging: Path) -> Path:
model_root = (runtime_home() / "models").resolve()
def _activate_uploaded_encoder(staging: Path, model_revision: str) -> Path:
model_root = (runtime_home() / "models" / "bge-m3-revisions").resolve()
model_root.mkdir(parents=True, exist_ok=True)
active = (model_root / "bge-m3").resolve()
active = (model_root / model_revision).resolve()
if active.parent != model_root:
raise RuntimeError("编码器目标目录越界")
backup = (model_root / f".bge-m3-backup-{uuid4().hex}").resolve()
if active.exists():
active.replace(backup)
try:
staging.replace(active)
except Exception:
if backup.exists() and not active.exists():
backup.replace(active)
raise
if backup.exists():
shutil.rmtree(backup)
raise RuntimeError("编码器版本目录已存在")
staging.replace(active)
return active
async def _save_encoder_path(request: Request, path: Path) -> dict[str, object]:
embedding = _local_encoder(request)
async def _save_encoder_path(
request: Request,
path: Path,
*,
model_revision: str | None = None,
) -> dict[str, object]:
current = load_system_settings()
encoder_settings = current.local_wiki_encoder.model_copy(update={"model_path": str(path)})
encoder_settings = current.local_wiki_encoder.model_copy(
update={
"mode": "local",
"model_path": str(path),
"model_revision": model_revision or uuid4().hex,
}
)
save_system_settings(current.model_copy(update={"local_wiki_encoder": encoder_settings}))
await embedding.reset_local_encoder(model_path=str(path))
return embedding.local_encoder_status()
await rebuild_local_wiki_index_runtime(request.app, force=True)
return local_encoder_runtime_status(request.app)
@router.get("/local-encoder/status")
@@ -141,7 +157,8 @@ async def local_encoder_status(request: Request) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
return _local_encoder(request).local_encoder_status()
await rebuild_local_wiki_index_runtime(request.app)
return local_encoder_runtime_status(request.app)
@router.post("/local-encoder/start", status_code=202)
@@ -149,8 +166,10 @@ async def start_local_encoder(request: Request) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
await rebuild_local_wiki_index_runtime(request.app)
try:
return _local_encoder(request).start_local_encoder()
_local_encoder(request).start_local_encoder()
return local_encoder_runtime_status(request.app)
except ValueError as exc:
raise HTTPException(status_code=409, detail={"code": "LOCAL_ENCODER_START_REJECTED", "message": str(exc)}) from None
@@ -163,6 +182,20 @@ async def configure_local_encoder(request: Request, body: LocalEncoderConfigRequ
return await _save_encoder_path(request, _validated_encoder_path(body.model_path))
@router.post("/local-encoder/use-configured")
async def use_configured_encoder(request: Request) -> dict[str, object]:
"""Restore the embedding provider declared in config.yaml without restart."""
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
current = load_system_settings()
encoder_settings = current.local_wiki_encoder.model_copy(update={"mode": "configured"})
save_system_settings(current.model_copy(update={"local_wiki_encoder": encoder_settings}))
await rebuild_local_wiki_index_runtime(request.app, force=True)
return local_encoder_runtime_status(request.app)
@router.post("/local-encoder/upload")
async def upload_local_encoder(
request: Request,
@@ -191,9 +224,17 @@ async def upload_local_encoder(
try:
uploaded_bytes = await asyncio.to_thread(_copy_encoder_uploads, files, paths, staging)
_validated_encoder_path(str(staging))
await _local_encoder(request).reset_local_encoder()
active = await asyncio.to_thread(_activate_uploaded_encoder, staging)
status = await _save_encoder_path(request, active)
model_revision = uuid4().hex
active = await asyncio.to_thread(
_activate_uploaded_encoder,
staging,
model_revision,
)
status = await _save_encoder_path(
request,
active,
model_revision=model_revision,
)
return {**status, "uploaded_files": len(files), "uploaded_bytes": uploaded_bytes}
except HTTPException:
raise
@@ -224,7 +265,7 @@ async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tupl
@router.post("/wiki-vector-search")
async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
config, search, _ = _services(request)
config, search, _ = await _services(request)
if not config.internal_search_enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
@@ -242,6 +283,45 @@ async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) ->
return result
@router.post("/knowledge-bases/{mapping_id}/retrieval-debug")
async def retrieval_debug(
request: Request,
mapping_id: str,
body: WikiRetrievalDebugRequest,
) -> dict[str, Any]:
"""Run the same auditable retrieval path used by Q&A for one library.
The response intentionally contains the query plan, deterministic counts,
concrete Wiki/source references and channel diagnostics so the browser's
Network panel can be used to diagnose a false "no data" answer.
"""
config, _, index_store = await _services(request)
if not config.internal_search_enabled:
raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED"})
rows, user_id, is_admin = await _authorized_mappings(request, [mapping_id])
include_drafts = body.include_drafts and (is_admin or str(rows[0].get("owner_user_id")) == user_id)
retrieval = getattr(request.app.state, "llmwiki_retrieval_service", None)
if retrieval is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
request_id = str(uuid4())
try:
result = await advanced_search(
query=body.query,
mappings=rows,
retrieval_service=retrieval,
index_store=index_store,
top_k=body.top_k,
date_from=body.date_from,
date_to=body.date_to,
expand_entities=body.expand_entities,
include_drafts=include_drafts,
)
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc), "request_id": request_id}) from None
return {"request_id": request_id, **result}
@router.get("/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}")
async def local_wiki_page(
request: Request,
@@ -250,7 +330,7 @@ async def local_wiki_page(
include_provenance: bool = Query(default=False),
include_archived: bool = Query(default=False),
) -> dict[str, Any]:
_, _, index_store = _services(request)
_, _, index_store = await _services(request)
user_id, is_admin = await _actor(request)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if mapping is None:
@@ -292,7 +372,7 @@ async def wiki_index_status(request: Request) -> dict[str, Any]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
config = request.app.state.config.llmwiki.local_wiki_index
config = await rebuild_local_wiki_index_runtime(request.app)
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
@@ -339,12 +419,18 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
continue
async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
async def _run_one_sync(
app,
mapping: dict[str, Any],
*,
force: bool,
sync_id: str | None = None,
) -> None:
service = getattr(app.state, "llmwiki_sync_service", None)
if service is None:
return
try:
await service.sync_mapping(mapping, force=force)
await service.sync_mapping(mapping, force=force, sync_id=sync_id)
except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
return
@@ -352,7 +438,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
@router.post("/wiki-index/sync", status_code=202)
async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
_services(request)
await _services(request)
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
@@ -363,7 +449,7 @@ async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dic
@router.post("/wiki-index/rebuild", status_code=202)
async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
_, _, index_store = _services(request)
_, _, index_store = await _services(request)
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
@@ -387,7 +473,12 @@ async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str
@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
async def knowledge_base_wiki_index_status(
request: Request,
mapping_id: str,
include_pages: bool = Query(default=True),
) -> dict[str, Any]:
config = await rebuild_local_wiki_index_runtime(request.app)
user_id, is_admin = await _actor(request)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if mapping is None:
@@ -404,13 +495,24 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
and not state.get("last_success_at")
):
state = {**state, "state": "not_synced"}
pages = await index_store.list_page_index_status(mapping_id)
# The page table is useful in the management view, but loading every row
# on each short polling interval is needless work for the small sync
# progress dialog. Keep the existing response as the default and expose a
# summary-only variant for frequent polling.
pages = (
await index_store.list_page_index_status(mapping_id)
if include_pages
else []
)
active_sync_id = str(state.get("active_sync_id") or "")
is_running = str(state.get("state") or "") in {"running", "rebuilding"}
for page in pages:
page["vectorized"] = page.get("index_status") == "ready" and int(page.get("vector_count") or 0) > 0
page["processed_in_current_sync"] = bool(active_sync_id) and page.get("last_seen_sync_id") == active_sync_id
total_pages = max(int(state.get("remote_page_count") or 0), len(pages))
total_pages = max(
int(state.get("remote_page_count") or 0),
len(pages) if include_pages else int(state.get("local_page_count") or 0),
)
processed_pages = (
sum(bool(page["processed_in_current_sync"]) for page in pages)
if is_running
@@ -427,28 +529,71 @@ async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) ->
embedding = getattr(request.app.state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
return {
serving_stale_snapshot = fully_vectorized and str(state.get("state") or "") != "idle"
last_sync = state.get("last_sync_result")
response = {
**state,
"last_sync": dict(last_sync) if isinstance(last_sync, dict) else None,
"knowledge_base_id": mapping_id,
"local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
"local_index_enabled": bool(config.enabled),
"wiki_index_enabled": mapping.get("wiki_index_enabled", True),
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
"serving_snapshot_available": fully_vectorized,
"serving_stale_snapshot": serving_stale_snapshot,
"retrieval_mode": "hybrid" if fully_vectorized else "keyword_wiki_api",
"diagnostics": {
"configured_embedding_fingerprint": fingerprint,
"indexed_embedding_fingerprint": state.get("embedding_fingerprint"),
"embedding_fingerprint_matches": bool(fingerprint)
and state.get("embedding_fingerprint") == fingerprint,
"last_error": state.get("last_error"),
},
"progress": {
"total_pages": total_pages,
"processed_pages": processed_pages,
"vectorized_pages": sum(bool(page["vectorized"]) for page in pages),
"failed_pages": sum(bool(page.get("last_error")) or page.get("index_status") in {"failed", "stale"} for page in pages),
"vector_count": sum(int(page.get("vector_count") or 0) for page in pages),
"vectorized_pages": (
sum(bool(page["vectorized"]) for page in pages)
if include_pages
else int(state.get("ready_page_count") or 0)
),
"failed_pages": (
sum(
bool(page.get("last_error"))
or page.get("index_status") in {"failed", "stale"}
for page in pages
)
if include_pages
else int(state.get("failed_page_count") or 0)
),
"vector_count": (
sum(int(page.get("vector_count") or 0) for page in pages)
if include_pages
else int(state.get("vector_count") or 0)
),
"completion_percent": completion_percent,
},
"pages": pages,
}
if include_pages:
response["pages"] = pages
return response
@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
config = request.app.state.config.llmwiki.local_wiki_index
async def _queue_incremental_mapping_sync(
request: Request,
mapping_id: str,
) -> dict[str, Any]:
user_id, is_admin = await _actor(request)
mapping_store = request.app.state.llmwiki_store
mapping = await mapping_store.get_authorized(
mapping_id,
user_id,
write=True,
is_admin=is_admin,
)
if mapping is None:
raise HTTPException(status_code=404, detail="Knowledge base not found")
config = await rebuild_local_wiki_index_runtime(request.app)
if not config.enabled:
raise HTTPException(
status_code=409,
@@ -457,16 +602,59 @@ async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[st
service = getattr(request.app.state, "llmwiki_sync_service", None)
if service is None:
raise HTTPException(status_code=503, detail={"code": "WIKI_VECTORIZATION_UNAVAILABLE"})
user_id, is_admin = await _actor(request)
mapping_store = request.app.state.llmwiki_store
mapping = await mapping_store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
if mapping is None:
raise HTTPException(status_code=404, detail="Knowledge base not found")
embedding = getattr(service, "embedding", None)
if embedding is not None and getattr(embedding, "dimensions", None) is None:
# Do not acknowledge a manual job that cannot even initialize its
# encoder: otherwise the caller could poll an old idle result forever.
try:
await embedding.embed_query(
"DeerFlow local Wiki embedding dimension probe"
)
except Exception as exc:
logger.warning("Manual Wiki sync embedding probe failed", exc_info=True)
raise HTTPException(
status_code=503,
detail={
"code": "WIKI_EMBEDDING_UNAVAILABLE",
"message": str(exc)[:1000] or "Embedding initialization failed",
},
) from None
if not mapping.get("wiki_index_enabled", True):
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
job_id = str(uuid4())
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=False))
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
spawn_local_wiki_index_job(
request.app,
_run_one_sync(
request.app,
mapping,
force=False,
sync_id=job_id,
),
)
return {
"job_id": job_id,
"status": "accepted",
"mode": "incremental",
"knowledge_base_id": mapping_id,
"status_url": f"/api/llmwiki/knowledge-bases/{mapping_id}/wiki-index-status",
}
@router.post("/knowledge-bases/{mapping_id}/sync", status_code=202)
async def sync_knowledge_base_incrementally(
request: Request,
mapping_id: str,
) -> dict[str, Any]:
"""Queue an owner-authorized metadata-first WeKnora Wiki refresh."""
return await _queue_incremental_mapping_sync(request, mapping_id)
@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
"""Backward-compatible alias for the canonical incremental sync action."""
return await _queue_incremental_mapping_sync(request, mapping_id)
@router.patch("/knowledge-bases/{mapping_id}/wiki-index-settings")