feat: improve knowledge sync and entity alignment
This commit is contained in:
1 parent
7dbac9cc56
commit
dea4697ea0
38 files changed
+3742
-634
No files matched your search
@@ -78,6 +78,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
|
||||
api_key: str = Field(default="", repr=False)
|
||||
local_model_path: str = ""
|
||||
local_cache_dir: str = ""
|
||||
local_process_isolation: bool = True
|
||||
local_threads: int = Field(default=2, ge=1, le=32)
|
||||
dimensions: int | None = Field(
|
||||
default=None,
|
||||
@@ -133,8 +134,8 @@ class LocalWikiIndexConfig(BaseModel):
|
||||
default="http://127.0.0.1:5174",
|
||||
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
|
||||
)
|
||||
auto_sync: bool = False
|
||||
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
|
||||
auto_sync: bool = True
|
||||
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
|
||||
sync_page_size: int = Field(default=100, ge=1, le=500)
|
||||
sync_concurrency: int = Field(default=3, ge=1, le=32)
|
||||
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
|
||||
|
||||
@@ -505,6 +505,17 @@ class ModelConcurrencySettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class LocalWikiEncoderSettings(BaseModel):
|
||||
"""Runtime override for the administrator-managed offline Wiki encoder."""
|
||||
|
||||
model_path: str = Field(
|
||||
default="",
|
||||
max_length=4096,
|
||||
description="Absolute or project-relative directory containing the offline BGE-M3 ONNX model.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SystemSettings(BaseModel):
|
||||
"""Top-level system settings document."""
|
||||
|
||||
@@ -520,6 +531,7 @@ class SystemSettings(BaseModel):
|
||||
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
|
||||
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
|
||||
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
|
||||
local_wiki_encoder: LocalWikiEncoderSettings = Field(default_factory=LocalWikiEncoderSettings)
|
||||
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
|
||||
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
|
||||
knowledge_base_product_version: Literal["v1", "v2"] = Field(
|
||||
|
||||
+299
-6
@@ -4,7 +4,12 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
|
||||
@@ -45,7 +50,13 @@ class StrictWikiEmbeddingClient:
|
||||
self._owns_client = http_client is None
|
||||
self._resolved_dimensions = config.dimensions
|
||||
self._local_model = None
|
||||
self._local_process: subprocess.Popen[str] | None = None
|
||||
self._local_lock = asyncio.Lock()
|
||||
self._local_state = "idle"
|
||||
self._local_error: str | None = None
|
||||
self._local_started_at: str | None = None
|
||||
self._local_ready_at: str | None = None
|
||||
self._local_warmup_task: asyncio.Task[None] | None = None
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
@@ -57,10 +68,35 @@ class StrictWikiEmbeddingClient:
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.reset_local_encoder()
|
||||
if self._owns_client and self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
|
||||
"""Stop only the embedded worker and optionally switch its model path.
|
||||
|
||||
This is intentionally independent from the Gateway lifecycle: an
|
||||
administrator can replace the offline model without restarting or
|
||||
stopping the rest of DeerFlow.
|
||||
"""
|
||||
warmup = self._local_warmup_task
|
||||
if warmup is not None and not warmup.done():
|
||||
warmup.cancel()
|
||||
await asyncio.gather(warmup, return_exceptions=True)
|
||||
self._local_warmup_task = None
|
||||
process = self._local_process
|
||||
self._local_process = None
|
||||
if process is not None:
|
||||
await asyncio.to_thread(self._stop_local_process, process)
|
||||
self._local_model = None
|
||||
self._local_state = "idle"
|
||||
self._local_error = None
|
||||
self._local_started_at = None
|
||||
self._local_ready_at = None
|
||||
if model_path is not None:
|
||||
self.config = self.config.model_copy(update={"local_model_path": model_path})
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
|
||||
@@ -71,7 +107,18 @@ class StrictWikiEmbeddingClient:
|
||||
if not texts:
|
||||
return []
|
||||
async with self._local_lock:
|
||||
return await asyncio.to_thread(self._embed_local, texts)
|
||||
self._local_state = "starting"
|
||||
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
|
||||
try:
|
||||
values = await asyncio.to_thread(self._embed_local, texts)
|
||||
except Exception as exc:
|
||||
self._local_state = "failed"
|
||||
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
|
||||
raise
|
||||
self._local_state = "ready"
|
||||
self._local_error = None
|
||||
self._local_ready_at = datetime.now(UTC).isoformat()
|
||||
return values
|
||||
if not self.config.enabled or not self.config.base_url or not self.config.model:
|
||||
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
|
||||
if not texts:
|
||||
@@ -87,11 +134,36 @@ class StrictWikiEmbeddingClient:
|
||||
def _embed_local(self, texts: list[str]) -> list[list[float]]:
|
||||
if not self.config.enabled:
|
||||
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
|
||||
model_path = self._validated_local_model_path()
|
||||
if self.config.local_process_isolation:
|
||||
return self._embed_local_process(texts, model_path)
|
||||
return self._embed_local_in_process(texts, model_path)
|
||||
|
||||
def _validated_local_model_path(self) -> Path:
|
||||
model_path = self._local_model_path()
|
||||
if model_path is None:
|
||||
raise WikiEmbeddingError(
|
||||
"离线本地编码器目录未配置;请设置 local_model_path 或 DEERFLOW_BGE_M3_MODEL_PATH",
|
||||
category="configuration",
|
||||
retryable=False,
|
||||
)
|
||||
if not model_path.is_dir() or not (
|
||||
(model_path / "onnx" / "model.onnx").is_file()
|
||||
or (model_path / "model.onnx").is_file()
|
||||
):
|
||||
raise WikiEmbeddingError(
|
||||
f"离线本地编码器目录无效或缺少 ONNX 模型:{model_path}",
|
||||
category="configuration",
|
||||
retryable=False,
|
||||
)
|
||||
return model_path
|
||||
|
||||
def _embed_local_in_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
|
||||
if self._local_model is None:
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except ImportError as exc:
|
||||
raise WikiEmbeddingError("请安装本地编码依赖:uv sync --extra local-embedding", retryable=False) from exc
|
||||
raise WikiEmbeddingError("标准后端依赖缺少 fastembed,请检查离线 wheelhouse 和安装结果", retryable=False) from exc
|
||||
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
|
||||
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
|
||||
from fastembed.common.model_description import ModelSource, PoolingType
|
||||
@@ -108,10 +180,14 @@ class StrictWikiEmbeddingClient:
|
||||
size_in_gb=2.29,
|
||||
)
|
||||
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
|
||||
options = {"model_name": name, "threads": self.config.local_threads, "cache_dir": str(cache_dir), "providers": ["CPUExecutionProvider"]}
|
||||
if self.config.local_model_path:
|
||||
options["specific_model_path"] = str(Path(self.config.local_model_path).expanduser())
|
||||
options["local_files_only"] = True
|
||||
options = {
|
||||
"model_name": name,
|
||||
"threads": self.config.local_threads,
|
||||
"cache_dir": str(cache_dir),
|
||||
"providers": ["CPUExecutionProvider"],
|
||||
"specific_model_path": str(model_path),
|
||||
"local_files_only": True,
|
||||
}
|
||||
self._local_model = TextEmbedding(**options)
|
||||
values = []
|
||||
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
|
||||
@@ -123,6 +199,146 @@ class StrictWikiEmbeddingClient:
|
||||
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
|
||||
return values
|
||||
|
||||
def _embed_local_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
|
||||
process = self._local_process
|
||||
if process is None or process.poll() is not None:
|
||||
process = self._start_local_process(model_path)
|
||||
self._local_process = process
|
||||
if process.stdin is None or process.stdout is None:
|
||||
raise WikiEmbeddingError("本地编码器子进程通信通道不可用", retryable=False)
|
||||
request_id = hashlib.sha256(f"{monotonic()}:{len(texts)}".encode()).hexdigest()[:16]
|
||||
try:
|
||||
process.stdin.write(
|
||||
json.dumps({"id": request_id, "texts": texts}, ensure_ascii=False) + "\n"
|
||||
)
|
||||
process.stdin.flush()
|
||||
response = self._read_local_process_message(process, request_id=request_id)
|
||||
except (BrokenPipeError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
self._stop_local_process(process)
|
||||
self._local_process = None
|
||||
raise WikiEmbeddingError(
|
||||
f"本地编码器子进程异常退出:{str(exc)[:500]}",
|
||||
category="local_process",
|
||||
retryable=True,
|
||||
) from exc
|
||||
if response.get("status") != "ok":
|
||||
error = str(response.get("error") or "本地编码失败")[:1000]
|
||||
if process.poll() is not None:
|
||||
self._local_process = None
|
||||
raise WikiEmbeddingError(error, category="local_process", retryable=False)
|
||||
raw_vectors = response.get("vectors")
|
||||
if not isinstance(raw_vectors, list) or len(raw_vectors) != len(texts):
|
||||
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
|
||||
values: list[list[float]] = []
|
||||
for raw in raw_vectors:
|
||||
normalized = normalize_vector(raw, dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(normalized.size)
|
||||
values.append(normalized.tolist())
|
||||
return values
|
||||
|
||||
def _start_local_process(self, model_path: Path) -> subprocess.Popen[str]:
|
||||
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
|
||||
cache_dir = (
|
||||
Path(self.config.local_cache_dir).expanduser()
|
||||
if self.config.local_cache_dir
|
||||
else Path.home() / ".cache" / "deerflow" / "embeddings"
|
||||
)
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"deerflow.integrations.weknora.local_index.embedding_worker",
|
||||
"--model-name",
|
||||
name,
|
||||
"--model-path",
|
||||
str(model_path),
|
||||
"--cache-dir",
|
||||
str(cache_dir),
|
||||
"--threads",
|
||||
str(self.config.local_threads),
|
||||
"--batch-size",
|
||||
str(min(self.config.batch_size, 4)),
|
||||
]
|
||||
environment = os.environ.copy()
|
||||
environment.update(
|
||||
{
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"TRANSFORMERS_OFFLINE": "1",
|
||||
"HF_DATASETS_OFFLINE": "1",
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONUTF8": "1",
|
||||
}
|
||||
)
|
||||
creation_flags = 0
|
||||
if os.name == "nt":
|
||||
creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) | int(
|
||||
getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0)
|
||||
)
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
bufsize=1,
|
||||
env=environment,
|
||||
creationflags=creation_flags,
|
||||
)
|
||||
response = self._read_local_process_message(process, ready=True)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
if "process" in locals():
|
||||
self._stop_local_process(process)
|
||||
raise WikiEmbeddingError(
|
||||
f"无法启动本地编码器子进程:{str(exc)[:500]}",
|
||||
category="local_process",
|
||||
retryable=False,
|
||||
) from exc
|
||||
if response.get("status") != "ready":
|
||||
error = str(response.get("error") or "本地编码器初始化失败")[:1000]
|
||||
self._stop_local_process(process)
|
||||
raise WikiEmbeddingError(error, category="local_process", retryable=False)
|
||||
return process
|
||||
|
||||
@staticmethod
|
||||
def _read_local_process_message(
|
||||
process: subprocess.Popen[str],
|
||||
*,
|
||||
request_id: str | None = None,
|
||||
ready: bool = False,
|
||||
) -> dict[str, object]:
|
||||
if process.stdout is None:
|
||||
raise ValueError("本地编码器没有输出通道")
|
||||
for _ in range(20):
|
||||
line = process.stdout.readline()
|
||||
if not line:
|
||||
raise ValueError(f"本地编码器已退出(exit={process.poll()})")
|
||||
payload = json.loads(line)
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
if ready and payload.get("status") in {"ready", "error"}:
|
||||
return payload
|
||||
if request_id is not None and payload.get("id") == request_id:
|
||||
return payload
|
||||
raise ValueError("本地编码器返回了无法识别的响应")
|
||||
|
||||
@staticmethod
|
||||
def _stop_local_process(process: subprocess.Popen[str]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
if process.stdin is not None:
|
||||
process.stdin.write('{"command":"shutdown"}\n')
|
||||
process.stdin.flush()
|
||||
process.wait(timeout=2)
|
||||
except Exception:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except Exception:
|
||||
process.kill()
|
||||
|
||||
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
@@ -185,3 +401,80 @@ class StrictWikiEmbeddingClient:
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
result = await self.embed_texts([text])
|
||||
return result[0]
|
||||
|
||||
def local_encoder_status(self) -> dict[str, object]:
|
||||
running = self._local_warmup_task is not None and not self._local_warmup_task.done()
|
||||
process_ready = self._local_process is not None and self._local_process.poll() is None
|
||||
process_failed = self._local_process is not None and not process_ready
|
||||
state = (
|
||||
"starting"
|
||||
if running
|
||||
else "ready"
|
||||
if (self._local_model is not None or process_ready)
|
||||
else "failed"
|
||||
if process_failed
|
||||
else self._local_state
|
||||
)
|
||||
last_error = self._local_error
|
||||
if process_failed and not last_error:
|
||||
last_error = f"本地编码器子进程已退出(exit={self._local_process.poll()})"
|
||||
model_path = self._local_model_path()
|
||||
model_files_ready = bool(
|
||||
model_path
|
||||
and model_path.is_dir()
|
||||
and ((model_path / "onnx" / "model.onnx").is_file() or (model_path / "model.onnx").is_file())
|
||||
)
|
||||
return {
|
||||
"provider": self.config.provider,
|
||||
"model": self.config.model,
|
||||
"dimensions": self.dimensions,
|
||||
"state": state,
|
||||
"running": running,
|
||||
"last_error": last_error,
|
||||
"started_at": self._local_started_at,
|
||||
"ready_at": self._local_ready_at,
|
||||
"offline_only": self.config.provider == "local",
|
||||
"model_path_configured": model_path is not None,
|
||||
"model_path_exists": bool(model_path and model_path.is_dir()),
|
||||
"model_files_ready": model_files_ready,
|
||||
"process_isolation": self.config.local_process_isolation,
|
||||
"model_path": str(model_path) if model_path is not None else "",
|
||||
}
|
||||
|
||||
def _local_model_path(self) -> Path | None:
|
||||
raw = self.config.local_model_path.strip() or os.environ.get(
|
||||
"DEERFLOW_BGE_M3_MODEL_PATH", ""
|
||||
).strip()
|
||||
if not raw:
|
||||
return None
|
||||
path = Path(raw).expanduser()
|
||||
return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve()
|
||||
|
||||
def start_local_encoder(self) -> dict[str, object]:
|
||||
"""Warm the embedded CPU encoder without coupling it to app startup."""
|
||||
if self.config.provider != "local":
|
||||
raise ValueError("当前 embedding provider 不是 local")
|
||||
if not self.config.enabled:
|
||||
raise ValueError("本地编码器未启用")
|
||||
process_ready = self._local_process is not None and self._local_process.poll() is None
|
||||
if self._local_model is not None or process_ready:
|
||||
self._local_state = "ready"
|
||||
return self.local_encoder_status()
|
||||
if self._local_warmup_task is None or self._local_warmup_task.done():
|
||||
self._local_state = "starting"
|
||||
self._local_error = None
|
||||
self._local_started_at = datetime.now(UTC).isoformat()
|
||||
self._local_warmup_task = asyncio.create_task(
|
||||
self._warm_local_encoder(),
|
||||
name="llmwiki-local-encoder-warmup",
|
||||
)
|
||||
return self.local_encoder_status()
|
||||
|
||||
async def _warm_local_encoder(self) -> None:
|
||||
try:
|
||||
await self.embed_query("本地编码器启动检查")
|
||||
except asyncio.CancelledError:
|
||||
self._local_state = "idle"
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Local Wiki encoder warmup failed model=%s", self.config.model)
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
"""Offline FastEmbed worker isolated from the DeerFlow gateway process.
|
||||
|
||||
The protocol is UTF-8 JSON Lines over stdin/stdout. Model weights must already
|
||||
exist on disk; every offline flag is enforced before FastEmbed is imported.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _write(payload: dict[str, object]) -> None:
|
||||
sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _build_model(args: argparse.Namespace):
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
||||
os.environ["HF_DATASETS_OFFLINE"] = "1"
|
||||
from fastembed import TextEmbedding
|
||||
|
||||
if args.model_name == "BAAI/bge-m3" and not any(
|
||||
item["model"] == args.model_name for item in TextEmbedding.list_supported_models()
|
||||
):
|
||||
from fastembed.common.model_description import ModelSource, PoolingType
|
||||
|
||||
TextEmbedding.add_custom_model(
|
||||
model=args.model_name,
|
||||
pooling=PoolingType.CLS,
|
||||
normalization=True,
|
||||
dim=1024,
|
||||
sources=ModelSource(hf="BAAI/bge-m3"),
|
||||
model_file="onnx/model.onnx",
|
||||
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
|
||||
license="mit",
|
||||
size_in_gb=2.29,
|
||||
)
|
||||
return TextEmbedding(
|
||||
model_name=args.model_name,
|
||||
threads=args.threads,
|
||||
cache_dir=args.cache_dir,
|
||||
providers=["CPUExecutionProvider"],
|
||||
specific_model_path=args.model_path,
|
||||
local_files_only=True,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-name", required=True)
|
||||
parser.add_argument("--model-path", required=True)
|
||||
parser.add_argument("--cache-dir", required=True)
|
||||
parser.add_argument("--threads", type=int, default=2)
|
||||
parser.add_argument("--batch-size", type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
model_path = Path(args.model_path)
|
||||
if not model_path.is_dir():
|
||||
_write({"status": "error", "error": f"离线模型目录不存在:{model_path}"})
|
||||
return 2
|
||||
try:
|
||||
model = _build_model(args)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_write({"status": "error", "error": f"离线模型加载失败:{type(exc).__name__}: {exc}"})
|
||||
return 2
|
||||
_write({"status": "ready"})
|
||||
for line in sys.stdin:
|
||||
request: dict[str, object] = {}
|
||||
try:
|
||||
request = json.loads(line)
|
||||
if request.get("command") == "shutdown":
|
||||
return 0
|
||||
request_id = str(request.get("id") or "")
|
||||
texts = request.get("texts")
|
||||
if not request_id or not isinstance(texts, list) or not all(isinstance(value, str) for value in texts):
|
||||
raise ValueError("编码请求格式无效")
|
||||
vectors = [
|
||||
value.tolist()
|
||||
for value in model.embed(texts, batch_size=max(1, min(args.batch_size, 4)))
|
||||
]
|
||||
_write({"id": request_id, "status": "ok", "vectors": vectors})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_write(
|
||||
{
|
||||
"id": str(request.get("id") or "") if isinstance(request, dict) else "",
|
||||
"status": "error",
|
||||
"error": f"本地编码失败:{type(exc).__name__}: {exc}",
|
||||
}
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+4
-1
@@ -163,7 +163,10 @@ class WikiSyncService:
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
)
|
||||
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
if not failed and remote_count and notify and self.on_completed:
|
||||
# Notify even when the remote library is now empty. The global
|
||||
# assistant base must then remove stale contributions from this
|
||||
# source instead of retaining deleted Wiki pages forever.
|
||||
if not failed and notify and self.on_completed:
|
||||
result["assistant_sync"] = await self.on_completed(mapping)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
|
||||
+937
-32
File diff suppressed because it is too large.
Load diff
+40
-3
@@ -158,6 +158,11 @@ class SkillKnowledgeRepository:
|
||||
binding.remote_business_key = target.get("remote_business_key")
|
||||
binding.status = "queued"
|
||||
binding.active = True
|
||||
# The current manual submitter owns the relationship UI
|
||||
# from this point onward. Target write authorization is
|
||||
# enforced by the router before this method is called.
|
||||
if trigger == "manual":
|
||||
binding.created_by = created_by
|
||||
binding.updated_at = now
|
||||
item = SkillKnowledgeSyncItemRow(
|
||||
id=str(uuid4()),
|
||||
@@ -187,7 +192,14 @@ class SkillKnowledgeRepository:
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
|
||||
async def list_bindings(
|
||||
self,
|
||||
*,
|
||||
skill_names: list[str] | None = None,
|
||||
target_id: str | None = None,
|
||||
include_detached: bool = False,
|
||||
created_by: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(SkillKnowledgeBindingRow)
|
||||
if skill_names:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
|
||||
@@ -195,6 +207,8 @@ class SkillKnowledgeRepository:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
|
||||
if not include_detached:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
|
||||
if created_by is not None:
|
||||
stmt = stmt.where(SkillKnowledgeBindingRow.created_by == created_by)
|
||||
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
@@ -247,10 +261,33 @@ class SkillKnowledgeRepository:
|
||||
result["items"] = [_dict(item) for item in items]
|
||||
return result
|
||||
|
||||
async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async def list_jobs(self, *, limit: int = 50, created_by: str | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
|
||||
if created_by is not None:
|
||||
stmt = stmt.where(SkillKnowledgeSyncJobRow.created_by == created_by)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
jobs = list((await session.execute(stmt)).scalars())
|
||||
if not jobs:
|
||||
return []
|
||||
job_ids = [row.id for row in jobs]
|
||||
item_rows = list(
|
||||
(
|
||||
await session.execute(
|
||||
select(SkillKnowledgeSyncItemRow)
|
||||
.where(SkillKnowledgeSyncItemRow.job_id.in_(job_ids))
|
||||
.order_by(SkillKnowledgeSyncItemRow.created_at)
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
items_by_job: dict[str, list[dict[str, Any]]] = {job_id: [] for job_id in job_ids}
|
||||
for item in item_rows:
|
||||
items_by_job[item.job_id].append(_dict(item))
|
||||
result = []
|
||||
for job in jobs:
|
||||
data = _dict(job)
|
||||
data["items"] = items_by_job[job.id]
|
||||
result.append(data)
|
||||
return result
|
||||
|
||||
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
now = _now()
|
||||
|
||||
Reference in new issue
Block a user