feat: improve knowledge sync and entity alignment

This commit is contained in:
吉鑫升 committed 2026-09-04 18:43:12 +08:00
1 parent 7dbac9cc56
commit dea4697ea0
38 files changed
+3742 -634

No files matched your search

@@ -78,6 +78,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
api_key: str = Field(default="", repr=False)
local_model_path: str = ""
local_cache_dir: str = ""
local_process_isolation: bool = True
local_threads: int = Field(default=2, ge=1, le=32)
dimensions: int | None = Field(
default=None,
@@ -133,8 +134,8 @@ class LocalWikiIndexConfig(BaseModel):
default="http://127.0.0.1:5174",
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
)
auto_sync: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
auto_sync: bool = True
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32)
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
@@ -505,6 +505,17 @@ class ModelConcurrencySettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class LocalWikiEncoderSettings(BaseModel):
"""Runtime override for the administrator-managed offline Wiki encoder."""
model_path: str = Field(
default="",
max_length=4096,
description="Absolute or project-relative directory containing the offline BGE-M3 ONNX model.",
)
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -520,6 +531,7 @@ class SystemSettings(BaseModel):
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
local_wiki_encoder: LocalWikiEncoderSettings = Field(default_factory=LocalWikiEncoderSettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
knowledge_base_product_version: Literal["v1", "v2"] = Field(
@@ -4,7 +4,12 @@ from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path
from time import monotonic
@@ -45,7 +50,13 @@ class StrictWikiEmbeddingClient:
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
self._local_model = None
self._local_process: subprocess.Popen[str] | None = None
self._local_lock = asyncio.Lock()
self._local_state = "idle"
self._local_error: str | None = None
self._local_started_at: str | None = None
self._local_ready_at: str | None = None
self._local_warmup_task: asyncio.Task[None] | None = None
@property
def dimensions(self) -> int | None:
@@ -57,10 +68,35 @@ class StrictWikiEmbeddingClient:
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
await self.reset_local_encoder()
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
"""Stop only the embedded worker and optionally switch its model path.
This is intentionally independent from the Gateway lifecycle: an
administrator can replace the offline model without restarting or
stopping the rest of DeerFlow.
"""
warmup = self._local_warmup_task
if warmup is not None and not warmup.done():
warmup.cancel()
await asyncio.gather(warmup, return_exceptions=True)
self._local_warmup_task = None
process = self._local_process
self._local_process = None
if process is not None:
await asyncio.to_thread(self._stop_local_process, process)
self._local_model = None
self._local_state = "idle"
self._local_error = None
self._local_started_at = None
self._local_ready_at = None
if model_path is not None:
self.config = self.config.model_copy(update={"local_model_path": model_path})
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
@@ -71,7 +107,18 @@ class StrictWikiEmbeddingClient:
if not texts:
return []
async with self._local_lock:
return await asyncio.to_thread(self._embed_local, texts)
self._local_state = "starting"
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
try:
values = await asyncio.to_thread(self._embed_local, texts)
except Exception as exc:
self._local_state = "failed"
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
raise
self._local_state = "ready"
self._local_error = None
self._local_ready_at = datetime.now(UTC).isoformat()
return values
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
@@ -87,11 +134,36 @@ class StrictWikiEmbeddingClient:
def _embed_local(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled:
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
model_path = self._validated_local_model_path()
if self.config.local_process_isolation:
return self._embed_local_process(texts, model_path)
return self._embed_local_in_process(texts, model_path)
def _validated_local_model_path(self) -> Path:
model_path = self._local_model_path()
if model_path is None:
raise WikiEmbeddingError(
"离线本地编码器目录未配置;请设置 local_model_path 或 DEERFLOW_BGE_M3_MODEL_PATH",
category="configuration",
retryable=False,
)
if not model_path.is_dir() or not (
(model_path / "onnx" / "model.onnx").is_file()
or (model_path / "model.onnx").is_file()
):
raise WikiEmbeddingError(
f"离线本地编码器目录无效或缺少 ONNX 模型:{model_path}",
category="configuration",
retryable=False,
)
return model_path
def _embed_local_in_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
if self._local_model is None:
try:
from fastembed import TextEmbedding
except ImportError as exc:
raise WikiEmbeddingError("请安装本地编码依赖:uv sync --extra local-embedding", retryable=False) from exc
raise WikiEmbeddingError("标准后端依赖缺少 fastembed,请检查离线 wheelhouse 和安装结果", retryable=False) from exc
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
from fastembed.common.model_description import ModelSource, PoolingType
@@ -108,10 +180,14 @@ class StrictWikiEmbeddingClient:
size_in_gb=2.29,
)
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
options = {"model_name": name, "threads": self.config.local_threads, "cache_dir": str(cache_dir), "providers": ["CPUExecutionProvider"]}
if self.config.local_model_path:
options["specific_model_path"] = str(Path(self.config.local_model_path).expanduser())
options["local_files_only"] = True
options = {
"model_name": name,
"threads": self.config.local_threads,
"cache_dir": str(cache_dir),
"providers": ["CPUExecutionProvider"],
"specific_model_path": str(model_path),
"local_files_only": True,
}
self._local_model = TextEmbedding(**options)
values = []
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
@@ -123,6 +199,146 @@ class StrictWikiEmbeddingClient:
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
return values
def _embed_local_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
process = self._local_process
if process is None or process.poll() is not None:
process = self._start_local_process(model_path)
self._local_process = process
if process.stdin is None or process.stdout is None:
raise WikiEmbeddingError("本地编码器子进程通信通道不可用", retryable=False)
request_id = hashlib.sha256(f"{monotonic()}:{len(texts)}".encode()).hexdigest()[:16]
try:
process.stdin.write(
json.dumps({"id": request_id, "texts": texts}, ensure_ascii=False) + "\n"
)
process.stdin.flush()
response = self._read_local_process_message(process, request_id=request_id)
except (BrokenPipeError, OSError, ValueError, json.JSONDecodeError) as exc:
self._stop_local_process(process)
self._local_process = None
raise WikiEmbeddingError(
f"本地编码器子进程异常退出:{str(exc)[:500]}",
category="local_process",
retryable=True,
) from exc
if response.get("status") != "ok":
error = str(response.get("error") or "本地编码失败")[:1000]
if process.poll() is not None:
self._local_process = None
raise WikiEmbeddingError(error, category="local_process", retryable=False)
raw_vectors = response.get("vectors")
if not isinstance(raw_vectors, list) or len(raw_vectors) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
values: list[list[float]] = []
for raw in raw_vectors:
normalized = normalize_vector(raw, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
return values
def _start_local_process(self, model_path: Path) -> subprocess.Popen[str]:
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
cache_dir = (
Path(self.config.local_cache_dir).expanduser()
if self.config.local_cache_dir
else Path.home() / ".cache" / "deerflow" / "embeddings"
)
command = [
sys.executable,
"-m",
"deerflow.integrations.weknora.local_index.embedding_worker",
"--model-name",
name,
"--model-path",
str(model_path),
"--cache-dir",
str(cache_dir),
"--threads",
str(self.config.local_threads),
"--batch-size",
str(min(self.config.batch_size, 4)),
]
environment = os.environ.copy()
environment.update(
{
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"HF_DATASETS_OFFLINE": "1",
"PYTHONIOENCODING": "utf-8",
"PYTHONUTF8": "1",
}
)
creation_flags = 0
if os.name == "nt":
creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) | int(
getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0)
)
try:
process = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
encoding="utf-8",
bufsize=1,
env=environment,
creationflags=creation_flags,
)
response = self._read_local_process_message(process, ready=True)
except (OSError, ValueError, json.JSONDecodeError) as exc:
if "process" in locals():
self._stop_local_process(process)
raise WikiEmbeddingError(
f"无法启动本地编码器子进程:{str(exc)[:500]}",
category="local_process",
retryable=False,
) from exc
if response.get("status") != "ready":
error = str(response.get("error") or "本地编码器初始化失败")[:1000]
self._stop_local_process(process)
raise WikiEmbeddingError(error, category="local_process", retryable=False)
return process
@staticmethod
def _read_local_process_message(
process: subprocess.Popen[str],
*,
request_id: str | None = None,
ready: bool = False,
) -> dict[str, object]:
if process.stdout is None:
raise ValueError("本地编码器没有输出通道")
for _ in range(20):
line = process.stdout.readline()
if not line:
raise ValueError(f"本地编码器已退出(exit={process.poll()})")
payload = json.loads(line)
if not isinstance(payload, dict):
continue
if ready and payload.get("status") in {"ready", "error"}:
return payload
if request_id is not None and payload.get("id") == request_id:
return payload
raise ValueError("本地编码器返回了无法识别的响应")
@staticmethod
def _stop_local_process(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
try:
if process.stdin is not None:
process.stdin.write('{"command":"shutdown"}\n')
process.stdin.flush()
process.wait(timeout=2)
except Exception:
process.terminate()
try:
process.wait(timeout=2)
except Exception:
process.kill()
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
@@ -185,3 +401,80 @@ class StrictWikiEmbeddingClient:
async def embed_query(self, text: str) -> list[float]:
result = await self.embed_texts([text])
return result[0]
def local_encoder_status(self) -> dict[str, object]:
running = self._local_warmup_task is not None and not self._local_warmup_task.done()
process_ready = self._local_process is not None and self._local_process.poll() is None
process_failed = self._local_process is not None and not process_ready
state = (
"starting"
if running
else "ready"
if (self._local_model is not None or process_ready)
else "failed"
if process_failed
else self._local_state
)
last_error = self._local_error
if process_failed and not last_error:
last_error = f"本地编码器子进程已退出(exit={self._local_process.poll()})"
model_path = self._local_model_path()
model_files_ready = bool(
model_path
and model_path.is_dir()
and ((model_path / "onnx" / "model.onnx").is_file() or (model_path / "model.onnx").is_file())
)
return {
"provider": self.config.provider,
"model": self.config.model,
"dimensions": self.dimensions,
"state": state,
"running": running,
"last_error": last_error,
"started_at": self._local_started_at,
"ready_at": self._local_ready_at,
"offline_only": self.config.provider == "local",
"model_path_configured": model_path is not None,
"model_path_exists": bool(model_path and model_path.is_dir()),
"model_files_ready": model_files_ready,
"process_isolation": self.config.local_process_isolation,
"model_path": str(model_path) if model_path is not None else "",
}
def _local_model_path(self) -> Path | None:
raw = self.config.local_model_path.strip() or os.environ.get(
"DEERFLOW_BGE_M3_MODEL_PATH", ""
).strip()
if not raw:
return None
path = Path(raw).expanduser()
return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve()
def start_local_encoder(self) -> dict[str, object]:
"""Warm the embedded CPU encoder without coupling it to app startup."""
if self.config.provider != "local":
raise ValueError("当前 embedding provider 不是 local")
if not self.config.enabled:
raise ValueError("本地编码器未启用")
process_ready = self._local_process is not None and self._local_process.poll() is None
if self._local_model is not None or process_ready:
self._local_state = "ready"
return self.local_encoder_status()
if self._local_warmup_task is None or self._local_warmup_task.done():
self._local_state = "starting"
self._local_error = None
self._local_started_at = datetime.now(UTC).isoformat()
self._local_warmup_task = asyncio.create_task(
self._warm_local_encoder(),
name="llmwiki-local-encoder-warmup",
)
return self.local_encoder_status()
async def _warm_local_encoder(self) -> None:
try:
await self.embed_query("本地编码器启动检查")
except asyncio.CancelledError:
self._local_state = "idle"
raise
except Exception:
logger.exception("Local Wiki encoder warmup failed model=%s", self.config.model)
@@ -0,0 +1,98 @@
"""Offline FastEmbed worker isolated from the DeerFlow gateway process.
The protocol is UTF-8 JSON Lines over stdin/stdout. Model weights must already
exist on disk; every offline flag is enforced before FastEmbed is imported.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
def _write(payload: dict[str, object]) -> None:
sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def _build_model(args: argparse.Namespace):
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
os.environ["HF_DATASETS_OFFLINE"] = "1"
from fastembed import TextEmbedding
if args.model_name == "BAAI/bge-m3" and not any(
item["model"] == args.model_name for item in TextEmbedding.list_supported_models()
):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=args.model_name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
return TextEmbedding(
model_name=args.model_name,
threads=args.threads,
cache_dir=args.cache_dir,
providers=["CPUExecutionProvider"],
specific_model_path=args.model_path,
local_files_only=True,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--model-name", required=True)
parser.add_argument("--model-path", required=True)
parser.add_argument("--cache-dir", required=True)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--batch-size", type=int, default=2)
args = parser.parse_args()
model_path = Path(args.model_path)
if not model_path.is_dir():
_write({"status": "error", "error": f"离线模型目录不存在:{model_path}"})
return 2
try:
model = _build_model(args)
except Exception as exc: # noqa: BLE001
_write({"status": "error", "error": f"离线模型加载失败:{type(exc).__name__}: {exc}"})
return 2
_write({"status": "ready"})
for line in sys.stdin:
request: dict[str, object] = {}
try:
request = json.loads(line)
if request.get("command") == "shutdown":
return 0
request_id = str(request.get("id") or "")
texts = request.get("texts")
if not request_id or not isinstance(texts, list) or not all(isinstance(value, str) for value in texts):
raise ValueError("编码请求格式无效")
vectors = [
value.tolist()
for value in model.embed(texts, batch_size=max(1, min(args.batch_size, 4)))
]
_write({"id": request_id, "status": "ok", "vectors": vectors})
except Exception as exc: # noqa: BLE001
_write(
{
"id": str(request.get("id") or "") if isinstance(request, dict) else "",
"status": "error",
"error": f"本地编码失败:{type(exc).__name__}: {exc}",
}
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -163,7 +163,10 @@ class WikiSyncService:
error=f"{failed} page(s) failed" if failed else None,
)
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
if not failed and remote_count and notify and self.on_completed:
# Notify even when the remote library is now empty. The global
# assistant base must then remove stale contributions from this
# source instead of retaining deleted Wiki pages forever.
if not failed and notify and self.on_completed:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
@@ -158,6 +158,11 @@ class SkillKnowledgeRepository:
binding.remote_business_key = target.get("remote_business_key")
binding.status = "queued"
binding.active = True
# The current manual submitter owns the relationship UI
# from this point onward. Target write authorization is
# enforced by the router before this method is called.
if trigger == "manual":
binding.created_by = created_by
binding.updated_at = now
item = SkillKnowledgeSyncItemRow(
id=str(uuid4()),
@@ -187,7 +192,14 @@ class SkillKnowledgeRepository:
await session.commit()
return result
async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
async def list_bindings(
self,
*,
skill_names: list[str] | None = None,
target_id: str | None = None,
include_detached: bool = False,
created_by: str | None = None,
) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeBindingRow)
if skill_names:
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
@@ -195,6 +207,8 @@ class SkillKnowledgeRepository:
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
if not include_detached:
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
if created_by is not None:
stmt = stmt.where(SkillKnowledgeBindingRow.created_by == created_by)
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
@@ -247,10 +261,33 @@ class SkillKnowledgeRepository:
result["items"] = [_dict(item) for item in items]
return result
async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
async def list_jobs(self, *, limit: int = 50, created_by: str | None = None) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
if created_by is not None:
stmt = stmt.where(SkillKnowledgeSyncJobRow.created_by == created_by)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
jobs = list((await session.execute(stmt)).scalars())
if not jobs:
return []
job_ids = [row.id for row in jobs]
item_rows = list(
(
await session.execute(
select(SkillKnowledgeSyncItemRow)
.where(SkillKnowledgeSyncItemRow.job_id.in_(job_ids))
.order_by(SkillKnowledgeSyncItemRow.created_at)
)
).scalars()
)
items_by_job: dict[str, list[dict[str, Any]]] = {job_id: [] for job_id in job_ids}
for item in item_rows:
items_by_job[item.job_id].append(_dict(item))
result = []
for job in jobs:
data = _dict(job)
data["items"] = items_by_job[job.id]
result.append(data)
return result
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
now = _now()