Co-authored-by: Cursor <cursoragent@cursor.com>

# Conflicts:
#	frontend-web/src/components/workspace/workspace-nav-chat-list.tsx
#	frontend-web/src/pages/WorkspaceRoutes.tsx
#	offline-backend-20260512/backend/app/gateway/app.py
#	offline-backend-20260512/backend/app/gateway/deps.py
#	offline-backend-20260512/backend/pyproject.toml
This commit is contained in:
董宏凯 committed 2026-09-07 14:46:36 +08:00
commit cf9fe58905
109 files changed
+18863 -982

No files matched your search

@@ -109,8 +109,8 @@ def _filter_available_skills_for_excluded_tools(
available_skills = []
return [name for name in available_skills if name.strip().lower() not in blocked]
def _log_es_query_routing_registration(
def _log_es_query_routing_registration(
*,
system_prompt: str,
app_config: AppConfig,
@@ -0,0 +1,5 @@
"""Assistant knowledge package helpers."""
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
__all__ = ["export_package_from_skill_ir"]
@@ -0,0 +1,79 @@
"""Disk-backed, replayable package records. Memory is bounded by one JSONL record.
The staging index is local SQLite, never a remote provider's database. It also
allows restoration to fetch only the vectors belonging to the current page.
"""
from __future__ import annotations
import json
import sqlite3
import tempfile
from collections.abc import Iterator
from contextlib import closing
from pathlib import Path
from typing import Any
class Records:
def __init__(self, path: Path, kind: str, slug: str | None = None):
self.path, self.kind, self.slug = path, kind, slug
def _where(self):
return ("kind=?", (self.kind,)) if self.slug is None else ("kind=? AND slug=?", (self.kind, self.slug))
def __iter__(self) -> Iterator[dict[str, Any]]:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
for (payload,) in db.execute(f"SELECT payload FROM records WHERE {where} ORDER BY seq", args):
yield json.loads(payload)
def __len__(self) -> int:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
return db.execute(f"SELECT count(*) FROM records WHERE {where}", args).fetchone()[0]
class PackageArchive(dict):
def __init__(self, path: Path):
super().__init__()
temporary = tempfile.NamedTemporaryFile(prefix="wiki-package-", suffix=".sqlite", dir=path.parent, delete=False)
self.path = Path(temporary.name)
temporary.close()
try:
with closing(sqlite3.connect(self.path)) as db, path.open(encoding="utf-8-sig") as handle:
db.execute("CREATE TABLE records(seq INTEGER PRIMARY KEY, kind TEXT, slug TEXT, payload TEXT)")
for line_no, line in enumerate(handle, 1):
if not line.strip():
continue
record = json.loads(line)
kind, payload = record.get("record_type"), record.get("payload")
if not isinstance(payload, dict):
raise ValueError(f"导出包第 {line_no} 行格式错误")
if kind == "source":
self["source"] = {**self.get("source", {}), **payload}
elif kind == "graph":
for key, subkind in (("nodes", "graph_node"), ("edges", "graph_edge")):
db.executemany("INSERT INTO records(kind, payload) VALUES (?, ?)", [(subkind, json.dumps(v, ensure_ascii=False)) for v in payload.get(key) or []])
elif kind in {"wiki_page", "document", "chunk", "vector", "entity", "relation"}:
slug = str(payload.get("wiki_slug") or payload.get("page_slug") or payload.get("slug") or "").strip("/")
db.execute("INSERT INTO records(kind, slug, payload) VALUES (?, ?, ?)", (kind, slug, json.dumps(payload, ensure_ascii=False)))
else:
raise ValueError(f"不支持的导出记录类型:{kind}")
db.execute("CREATE INDEX by_kind_slug ON records(kind, slug)")
db.commit()
source = self.get("source", {})
if source.get("package_version") != 1 or (source.get("provider") or source.get("type")) not in {"weknora", "assistant"}:
raise ValueError("仅接受 WeKnora / 助手知识库生成的兼容 Wiki 向量包(v1)")
for key, kind in (("wiki_pages", "wiki_page"), ("documents", "document"), ("chunks", "chunk"), ("vectors", "vector"), ("entities", "entity"), ("relations", "relation")):
self[key] = Records(self.path, kind)
self["graph"] = {"nodes": Records(self.path, "graph_node"), "edges": Records(self.path, "graph_edge")}
except BaseException:
self.close()
raise
def vectors_for(self, slug: str) -> Records:
return Records(self.path, "vector", slug)
def close(self):
self.path.unlink(missing_ok=True)
@@ -0,0 +1,22 @@
"""Normalize heterogeneous sources into the assistant import-package shape."""
from __future__ import annotations
from typing import Any
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
skill = ir.get("skill") or {}
return {
"source": {
"type": "skill",
"skill_name": skill.get("name"),
"source_digest": skill.get("source_digest"),
},
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
"documents": ir.get("files") or [],
"chunks": [],
"entities": ir.get("entities") or [],
"relations": ir.get("relations") or [],
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
}
@@ -0,0 +1,109 @@
"""File-backed assistant knowledge import/export packages.
The assistant knowledge base is fed by WeKnora-exported data. Keep that
boundary explicit: a WeKnora export is first materialized as a durable JSONL
package, then imported asynchronously from that package. JSONL keeps large
exports append-friendly and avoids treating the HTTP request body as the job
state.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
PACKAGE_VERSION = 1
PACKAGE_RECORD_TYPES = {
"source",
"wiki_page",
"document",
"chunk",
"vector",
"entity",
"relation",
"graph",
}
def package_path(root: Path, job_id: str) -> Path:
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
return root / f"{safe}.jsonl"
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
handle.write("\n")
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
path.parent.mkdir(parents=True, exist_ok=True)
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
tmp_path = path.with_suffix(path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
write_jsonl_record(
handle,
"source",
{
"package_version": PACKAGE_VERSION,
},
)
counts["source"] += 1
for record_type, payload in records:
if record_type not in PACKAGE_RECORD_TYPES:
continue
write_jsonl_record(handle, record_type, payload)
counts[record_type] = counts.get(record_type, 0) + 1
tmp_path.replace(path)
return counts
def read_package(path: Path) -> dict[str, Any]:
package: dict[str, Any] = {
"source": {},
"wiki_pages": [],
"documents": [],
"chunks": [],
"vectors": [],
"entities": [],
"relations": [],
"graph": {"nodes": [], "edges": []},
}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
record = json.loads(line)
if not isinstance(record, dict):
continue
record_type = str(record.get("record_type") or "")
payload = record.get("payload")
if not isinstance(payload, dict):
continue
if record_type == "source":
package["source"] = {**package.get("source", {}), **payload}
elif record_type == "wiki_page":
package["wiki_pages"].append(payload)
elif record_type == "document":
package["documents"].append(payload)
elif record_type == "chunk":
package["chunks"].append(payload)
elif record_type == "vector":
package["vectors"].append(payload)
elif record_type == "entity":
package["entities"].append(payload)
elif record_type == "relation":
package["relations"].append(payload)
elif record_type == "graph":
nodes = payload.get("nodes")
edges = payload.get("edges")
if isinstance(nodes, list):
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
if isinstance(edges, list):
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
for key, value in payload.items():
if key not in {"nodes", "edges"}:
package["graph"][key] = value
return package
@@ -11,10 +11,15 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
Gauss mode: application ORM data uses the official openGauss SQLAlchemy
dialect over asyncpg. LangGraph's PostgreSQL checkpointer is intentionally not
reused for GaussDB; configure the legacy ``checkpointer:`` section separately
(normally sqlite).
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
@@ -42,24 +47,40 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "gauss", "mysql"] = Field(
default="memory",
description=(
"Storage backend for application data. 'memory' for development "
"(no persistence across restarts), 'sqlite' for single-node "
"deployment, 'postgres' for production multi-node deployment, "
"'gauss' for openGauss/GaussDB business tables, and 'mysql' for "
"MySQL-backed business tables."
),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
),
)
gauss_url: str = Field(
default="",
description=(
"openGauss/GaussDB connection URL for the application ORM. "
"Use $GAUSS_DATABASE_URL in config.yaml to reference .env. "
"Example: opengauss+asyncpg://user:pass@host:26000/deerflow. "
"Plain opengauss:// or PostgreSQL-style URLs are normalized to the "
"official asynchronous openGauss SQLAlchemy dialect."
),
)
mysql_url: str = Field(
default="",
description=(
@@ -75,26 +96,26 @@ class DatabaseConfig(BaseModel):
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres/mysql only).",
description="Connection pool size for the app ORM engine (postgres/gauss/mysql only).",
)
max_overflow: int = Field(
default=10,
description=(
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
"Extra connections allowed beyond pool_size under load (postgres/gauss/mysql only). "
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
),
)
pool_timeout: int = Field(
default=30,
description=(
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
"Seconds a caller waits for a free pooled connection before erroring (postgres/gauss/mysql only). "
"SQLAlchemy default is 30."
),
)
pool_recycle: int = Field(
default=1800,
description=(
"Recycle pooled connections after this many seconds (postgres/mysql only). "
"Recycle pooled connections after this many seconds (postgres/gauss/mysql only). "
"Prevents stale-connection errors when a firewall, pgbouncer, or MySQL "
"wait_timeout silently drops idle connections. pool_pre_ping catches most "
"cases, but pool_recycle is the belt-and-suspenders safety net. "
@@ -133,12 +154,38 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "gauss":
url = self.gauss_url.strip()
if not url:
raise ValueError("database.gauss_url is required for backend='gauss'")
# GaussDB speaks the PostgreSQL wire protocol, but its server
# version string and DDL capabilities are not identical to
# PostgreSQL. Always select the official openGauss dialect instead
# of silently sending the URL through SQLAlchemy's generic PG
# dialect (the latter commonly connects and then fails while
# SQLAlchemy initializes/creates the schema).
for prefix in (
"opengauss://",
"gaussdb://",
"postgres://",
"postgresql://",
"postgresql+asyncpg://",
):
if url.startswith(prefix):
return "opengauss+asyncpg://" + url[len(prefix) :]
if url.startswith("opengauss+asyncpg://"):
return url
raise ValueError(
"database.gauss_url must use opengauss://, "
"opengauss+asyncpg://, or a PostgreSQL-style URL"
)
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
@@ -44,6 +44,7 @@ class WeKnoraConnectionConfig(BaseModel):
repr=False,
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@@ -52,12 +53,33 @@ class WeKnoraConnectionConfig(BaseModel):
return normalize_service_url(value)
class WeKnoraDirectGraphConfig(BaseModel):
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
direct_write_enabled: bool = False
neo4j_http_url: str = ""
username: str = ""
password: str = Field(default="", repr=False)
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
timeout_seconds: float = Field(default=30, gt=0, le=300)
model_config = ConfigDict(extra="ignore")
@field_validator("neo4j_http_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
model: str = "bge-m3"
model: str = "bge-embedding-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
local_model_path: str = ""
local_cache_dir: str = ""
local_process_isolation: bool = True
local_threads: int = Field(default=2, ge=1, le=32)
dimensions: int | None = Field(
default=None,
ge=1,
@@ -73,8 +95,8 @@ class LocalWikiEmbeddingConfig(BaseModel):
@classmethod
def _validate_provider(cls, value: str) -> str:
normalized = value.strip().lower()
if normalized not in {"openai", "openai_compatible"}:
raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
if normalized not in {"openai", "openai_compatible", "local"}:
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
return normalized
@field_validator("base_url")
@@ -112,8 +134,8 @@ class LocalWikiIndexConfig(BaseModel):
default="http://127.0.0.1:5174",
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
)
auto_sync: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
auto_sync: bool = True
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32)
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
@@ -180,7 +202,7 @@ class LocalWikiIndexConfig(BaseModel):
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
if self.enabled and not self.embedding.enabled:
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
return self
@@ -160,6 +160,18 @@ class PromptPrefixSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class SkillKnowledgeAutoSyncSettings(BaseModel):
"""Watcher/scheduler policy for bindings that explicitly opt in."""
watch_enabled: bool = True
schedule_enabled: bool = True
schedule_cron: str = Field(default="0 */6 * * *", min_length=5, max_length=128)
debounce_seconds: int = Field(default=120, ge=10, le=86_400)
write_remote_enabled: bool = True
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
model_config = ConfigDict(extra="ignore")
class AppearanceRouteParam(BaseModel):
"""Runtime query-param descriptor for top-bar shortcut buttons.
@@ -493,6 +505,17 @@ class ModelConcurrencySettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class LocalWikiEncoderSettings(BaseModel):
"""Runtime override for the administrator-managed offline Wiki encoder."""
model_path: str = Field(
default="",
max_length=4096,
description="Absolute or project-relative directory containing the offline BGE-M3 ONNX model.",
)
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -508,7 +531,13 @@ class SystemSettings(BaseModel):
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
local_wiki_encoder: LocalWikiEncoderSettings = Field(default_factory=LocalWikiEncoderSettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
knowledge_base_product_version: Literal["v1", "v2"] = Field(
default="v1",
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
)
model_config = ConfigDict(extra="ignore")
@@ -494,13 +494,28 @@ class WeKnoraClient:
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
params={"mode": "overview", "limit": max(1, min(limit, 500))},
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid wiki graph")
return dict(value)
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki generation status")
return value
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("Invalid Wiki folder response")
return value
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
async def list_wiki_pages(
self,
knowledge_base_id: str,
@@ -1,7 +1,37 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from __future__ import annotations
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from deerflow.integrations.weknora.local_index.retrieval import (
WikiRetrievalService,
is_fully_vectorized,
)
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = [
"WikiIndexError",
"WikiRetrievalService",
"WikiVectorSearchService",
"WikiSyncService",
"is_fully_vectorized",
]
def __getattr__(name: str) -> Any:
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
from deerflow.integrations.weknora.local_index import retrieval
return getattr(retrieval, name)
if name in {"WikiIndexError", "WikiVectorSearchService"}:
from deerflow.integrations.weknora.local_index import search
return getattr(search, name)
if name == "WikiSyncService":
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
return WikiSyncService
raise AttributeError(name)
@@ -105,4 +105,12 @@ def chunk_wiki_page(
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
# A short article with a short summary must not disappear merely because
# both sections fall below min_chars. Keep one combined, useful vector.
if not result and (content or summary):
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
normalized = re.sub(r"\s+", " ", fallback).strip()
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
prefix = prefix_lines if title_prefix_enabled else []
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
return result
@@ -4,7 +4,13 @@ from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path
from time import monotonic
import httpx
@@ -43,6 +49,14 @@ class StrictWikiEmbeddingClient:
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
self._local_model = None
self._local_process: subprocess.Popen[str] | None = None
self._local_lock = asyncio.Lock()
self._local_state = "idle"
self._local_error: str | None = None
self._local_started_at: str | None = None
self._local_ready_at: str | None = None
self._local_warmup_task: asyncio.Task[None] | None = None
@property
def dimensions(self) -> int | None:
@@ -54,16 +68,57 @@ class StrictWikiEmbeddingClient:
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
await self.reset_local_encoder()
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
"""Stop only the embedded worker and optionally switch its model path.
This is intentionally independent from the Gateway lifecycle: an
administrator can replace the offline model without restarting or
stopping the rest of DeerFlow.
"""
warmup = self._local_warmup_task
if warmup is not None and not warmup.done():
warmup.cancel()
await asyncio.gather(warmup, return_exceptions=True)
self._local_warmup_task = None
process = self._local_process
self._local_process = None
if process is not None:
await asyncio.to_thread(self._stop_local_process, process)
self._local_model = None
self._local_state = "idle"
self._local_error = None
self._local_started_at = None
self._local_ready_at = None
if model_path is not None:
self.config = self.config.model_copy(update={"local_model_path": model_path})
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if self.config.provider == "local":
if not texts:
return []
async with self._local_lock:
self._local_state = "starting"
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
try:
values = await asyncio.to_thread(self._embed_local, texts)
except Exception as exc:
self._local_state = "failed"
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
raise
self._local_state = "ready"
self._local_error = None
self._local_ready_at = datetime.now(UTC).isoformat()
return values
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
@@ -76,6 +131,214 @@ class StrictWikiEmbeddingClient:
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
def _embed_local(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled:
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
model_path = self._validated_local_model_path()
if self.config.local_process_isolation:
return self._embed_local_process(texts, model_path)
return self._embed_local_in_process(texts, model_path)
def _validated_local_model_path(self) -> Path:
model_path = self._local_model_path()
if model_path is None:
raise WikiEmbeddingError(
"离线本地编码器目录未配置;请设置 local_model_path 或 DEERFLOW_BGE_M3_MODEL_PATH",
category="configuration",
retryable=False,
)
if not model_path.is_dir() or not (
(model_path / "onnx" / "model.onnx").is_file()
or (model_path / "model.onnx").is_file()
):
raise WikiEmbeddingError(
f"离线本地编码器目录无效或缺少 ONNX 模型:{model_path}",
category="configuration",
retryable=False,
)
return model_path
def _embed_local_in_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
if self._local_model is None:
try:
from fastembed import TextEmbedding
except ImportError as exc:
raise WikiEmbeddingError("标准后端依赖缺少 fastembed,请检查离线 wheelhouse 和安装结果", retryable=False) from exc
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
options = {
"model_name": name,
"threads": self.config.local_threads,
"cache_dir": str(cache_dir),
"providers": ["CPUExecutionProvider"],
"specific_model_path": str(model_path),
"local_files_only": True,
}
self._local_model = TextEmbedding(**options)
values = []
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
if len(values) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
return values
def _embed_local_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
process = self._local_process
if process is None or process.poll() is not None:
process = self._start_local_process(model_path)
self._local_process = process
if process.stdin is None or process.stdout is None:
raise WikiEmbeddingError("本地编码器子进程通信通道不可用", retryable=False)
request_id = hashlib.sha256(f"{monotonic()}:{len(texts)}".encode()).hexdigest()[:16]
try:
process.stdin.write(
json.dumps({"id": request_id, "texts": texts}, ensure_ascii=False) + "\n"
)
process.stdin.flush()
response = self._read_local_process_message(process, request_id=request_id)
except (BrokenPipeError, OSError, ValueError, json.JSONDecodeError) as exc:
self._stop_local_process(process)
self._local_process = None
raise WikiEmbeddingError(
f"本地编码器子进程异常退出:{str(exc)[:500]}",
category="local_process",
retryable=True,
) from exc
if response.get("status") != "ok":
error = str(response.get("error") or "本地编码失败")[:1000]
if process.poll() is not None:
self._local_process = None
raise WikiEmbeddingError(error, category="local_process", retryable=False)
raw_vectors = response.get("vectors")
if not isinstance(raw_vectors, list) or len(raw_vectors) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
values: list[list[float]] = []
for raw in raw_vectors:
normalized = normalize_vector(raw, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
return values
def _start_local_process(self, model_path: Path) -> subprocess.Popen[str]:
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
cache_dir = (
Path(self.config.local_cache_dir).expanduser()
if self.config.local_cache_dir
else Path.home() / ".cache" / "deerflow" / "embeddings"
)
command = [
sys.executable,
"-m",
"deerflow.integrations.weknora.local_index.embedding_worker",
"--model-name",
name,
"--model-path",
str(model_path),
"--cache-dir",
str(cache_dir),
"--threads",
str(self.config.local_threads),
"--batch-size",
str(min(self.config.batch_size, 4)),
]
environment = os.environ.copy()
environment.update(
{
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"HF_DATASETS_OFFLINE": "1",
"PYTHONIOENCODING": "utf-8",
"PYTHONUTF8": "1",
}
)
creation_flags = 0
if os.name == "nt":
creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) | int(
getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0)
)
try:
process = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
encoding="utf-8",
bufsize=1,
env=environment,
creationflags=creation_flags,
)
response = self._read_local_process_message(process, ready=True)
except (OSError, ValueError, json.JSONDecodeError) as exc:
if "process" in locals():
self._stop_local_process(process)
raise WikiEmbeddingError(
f"无法启动本地编码器子进程:{str(exc)[:500]}",
category="local_process",
retryable=False,
) from exc
if response.get("status") != "ready":
error = str(response.get("error") or "本地编码器初始化失败")[:1000]
self._stop_local_process(process)
raise WikiEmbeddingError(error, category="local_process", retryable=False)
return process
@staticmethod
def _read_local_process_message(
process: subprocess.Popen[str],
*,
request_id: str | None = None,
ready: bool = False,
) -> dict[str, object]:
if process.stdout is None:
raise ValueError("本地编码器没有输出通道")
for _ in range(20):
line = process.stdout.readline()
if not line:
raise ValueError(f"本地编码器已退出(exit={process.poll()})")
payload = json.loads(line)
if not isinstance(payload, dict):
continue
if ready and payload.get("status") in {"ready", "error"}:
return payload
if request_id is not None and payload.get("id") == request_id:
return payload
raise ValueError("本地编码器返回了无法识别的响应")
@staticmethod
def _stop_local_process(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
try:
if process.stdin is not None:
process.stdin.write('{"command":"shutdown"}\n')
process.stdin.flush()
process.wait(timeout=2)
except Exception:
process.terminate()
try:
process.wait(timeout=2)
except Exception:
process.kill()
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
@@ -138,3 +401,80 @@ class StrictWikiEmbeddingClient:
async def embed_query(self, text: str) -> list[float]:
result = await self.embed_texts([text])
return result[0]
def local_encoder_status(self) -> dict[str, object]:
running = self._local_warmup_task is not None and not self._local_warmup_task.done()
process_ready = self._local_process is not None and self._local_process.poll() is None
process_failed = self._local_process is not None and not process_ready
state = (
"starting"
if running
else "ready"
if (self._local_model is not None or process_ready)
else "failed"
if process_failed
else self._local_state
)
last_error = self._local_error
if process_failed and not last_error:
last_error = f"本地编码器子进程已退出(exit={self._local_process.poll()})"
model_path = self._local_model_path()
model_files_ready = bool(
model_path
and model_path.is_dir()
and ((model_path / "onnx" / "model.onnx").is_file() or (model_path / "model.onnx").is_file())
)
return {
"provider": self.config.provider,
"model": self.config.model,
"dimensions": self.dimensions,
"state": state,
"running": running,
"last_error": last_error,
"started_at": self._local_started_at,
"ready_at": self._local_ready_at,
"offline_only": self.config.provider == "local",
"model_path_configured": model_path is not None,
"model_path_exists": bool(model_path and model_path.is_dir()),
"model_files_ready": model_files_ready,
"process_isolation": self.config.local_process_isolation,
"model_path": str(model_path) if model_path is not None else "",
}
def _local_model_path(self) -> Path | None:
raw = self.config.local_model_path.strip() or os.environ.get(
"DEERFLOW_BGE_M3_MODEL_PATH", ""
).strip()
if not raw:
return None
path = Path(raw).expanduser()
return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve()
def start_local_encoder(self) -> dict[str, object]:
"""Warm the embedded CPU encoder without coupling it to app startup."""
if self.config.provider != "local":
raise ValueError("当前 embedding provider 不是 local")
if not self.config.enabled:
raise ValueError("本地编码器未启用")
process_ready = self._local_process is not None and self._local_process.poll() is None
if self._local_model is not None or process_ready:
self._local_state = "ready"
return self.local_encoder_status()
if self._local_warmup_task is None or self._local_warmup_task.done():
self._local_state = "starting"
self._local_error = None
self._local_started_at = datetime.now(UTC).isoformat()
self._local_warmup_task = asyncio.create_task(
self._warm_local_encoder(),
name="llmwiki-local-encoder-warmup",
)
return self.local_encoder_status()
async def _warm_local_encoder(self) -> None:
try:
await self.embed_query("本地编码器启动检查")
except asyncio.CancelledError:
self._local_state = "idle"
raise
except Exception:
logger.exception("Local Wiki encoder warmup failed model=%s", self.config.model)
@@ -0,0 +1,98 @@
"""Offline FastEmbed worker isolated from the DeerFlow gateway process.
The protocol is UTF-8 JSON Lines over stdin/stdout. Model weights must already
exist on disk; every offline flag is enforced before FastEmbed is imported.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
def _write(payload: dict[str, object]) -> None:
sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def _build_model(args: argparse.Namespace):
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
os.environ["HF_DATASETS_OFFLINE"] = "1"
from fastembed import TextEmbedding
if args.model_name == "BAAI/bge-m3" and not any(
item["model"] == args.model_name for item in TextEmbedding.list_supported_models()
):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=args.model_name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
return TextEmbedding(
model_name=args.model_name,
threads=args.threads,
cache_dir=args.cache_dir,
providers=["CPUExecutionProvider"],
specific_model_path=args.model_path,
local_files_only=True,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--model-name", required=True)
parser.add_argument("--model-path", required=True)
parser.add_argument("--cache-dir", required=True)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--batch-size", type=int, default=2)
args = parser.parse_args()
model_path = Path(args.model_path)
if not model_path.is_dir():
_write({"status": "error", "error": f"离线模型目录不存在:{model_path}"})
return 2
try:
model = _build_model(args)
except Exception as exc: # noqa: BLE001
_write({"status": "error", "error": f"离线模型加载失败:{type(exc).__name__}: {exc}"})
return 2
_write({"status": "ready"})
for line in sys.stdin:
request: dict[str, object] = {}
try:
request = json.loads(line)
if request.get("command") == "shutdown":
return 0
request_id = str(request.get("id") or "")
texts = request.get("texts")
if not request_id or not isinstance(texts, list) or not all(isinstance(value, str) for value in texts):
raise ValueError("编码请求格式无效")
vectors = [
value.tolist()
for value in model.embed(texts, batch_size=max(1, min(args.batch_size, 4)))
]
_write({"id": request_id, "status": "ok", "vectors": vectors})
except Exception as exc: # noqa: BLE001
_write(
{
"id": str(request.get("id") or "") if isinstance(request, dict) else "",
"status": "error",
"error": f"本地编码失败:{type(exc).__name__}: {exc}",
}
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -36,8 +36,9 @@ class WikiSyncService:
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
@@ -111,6 +112,7 @@ class WikiSyncService:
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
@@ -160,7 +162,13 @@ class WikiSyncService:
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
# Notify even when the remote library is now empty. The global
# assistant base must then remove stale contributions from this
# source instead of retaining deleted Wiki pages forever.
if not failed and notify and self.on_completed:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,
@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,175 @@
"""ORM model for DeerFlow-native assistant knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
# ``global`` is the full-library aggregation base. ``custom`` bases are
# user-created assistant knowledge bases fed from WeKnora export packages.
# Nullable keeps startup column reconciliation safe for existing databases.
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
status: Mapped[str] = mapped_column(String(64), default="active")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeVectorRow(Base):
__tablename__ = "assistant_knowledge_vectors"
__table_args__ = (
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
Index("ix_assistant_knowledge_vector_page", "page_id"),
Index("ix_assistant_knowledge_vector_source", "source_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -176,7 +176,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -230,10 +230,10 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
added column are created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -355,13 +355,13 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
pool_recycle: Recycle connections after N seconds (postgres/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
@@ -379,11 +379,21 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend == "postgres":
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -423,7 +433,7 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
@@ -468,7 +478,7 @@ async def init_engine(
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
@@ -511,7 +521,7 @@ async def init_engine(
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s", backend)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -3,6 +3,7 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
@@ -35,6 +36,23 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
"""Iterate vector rows for streaming export.
Implementations with a database backend should override this method so
very large vector packages do not need to be materialized in memory.
"""
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
for row in snapshot.get("rows") or []:
if isinstance(row, dict):
yield row
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@@ -24,7 +24,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return dict(row) if row else None
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
@@ -2,6 +2,7 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
@@ -40,10 +41,11 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
return row
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
result = (await session.execute(stmt)).one_or_none()
return {**_dict(result[0]), "vector_count": result[1]} if result else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
@@ -181,6 +183,41 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
batch = max(1, min(int(batch_size), 5000))
offset = 0
while True:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
.limit(batch)
.offset(offset)
)
async with self._sf() as session:
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
if not rows:
break
for row in rows:
yield row
if len(rows) < batch:
break
offset += len(rows)
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -0,0 +1,73 @@
"""Add skill distillation and assistant knowledge tables.
Revision ID: 20260901_01
Revises: 20260825_01
Create Date: 2026-09-01
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
revision = "20260901_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
TABLES = [
SkillKnowledgeBindingRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncJobRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeArtifactRow,
SkillKnowledgeReviewRow,
SkillKnowledgeEntityRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeRelationContributionRow,
AssistantKnowledgeBaseRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeRelationContributionRow,
]
def upgrade() -> None:
bind = op.get_bind()
for model in TABLES:
model.__table__.create(bind, checkfirst=True)
def downgrade() -> None:
bind = op.get_bind()
for model in reversed(TABLES):
model.__table__.drop(bind, checkfirst=True)
@@ -0,0 +1,25 @@
"""Add assistant knowledge vector table.
Revision ID: 20260902_01
Revises: 20260901_01
Create Date: 2026-09-02
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import AssistantKnowledgeVectorRow
revision = "20260902_01"
down_revision = "20260901_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
AssistantKnowledgeVectorRow.__table__.create(op.get_bind(), checkfirst=True)
def downgrade() -> None:
AssistantKnowledgeVectorRow.__table__.drop(op.get_bind(), checkfirst=True)
@@ -0,0 +1,21 @@
"""Preserve source Wiki metadata during package round trips."""
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260903_01"
down_revision = ("20260902_01", "20260831_01")
branch_labels = None
depends_on = None
def upgrade():
columns = {col["name"] for col in sa.inspect(op.get_bind()).get_columns("assistant_knowledge_wiki_pages")}
if "metadata_json" not in columns:
op.add_column("assistant_knowledge_wiki_pages", sa.Column("metadata_json", PortableJSON(), nullable=True))
def downgrade():
op.drop_column("assistant_knowledge_wiki_pages", "metadata_json")
@@ -35,6 +35,18 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeVectorRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -100,6 +112,18 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.sensitive_words.model import SensitiveWordRow
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_buttons.model import TaskButtonRow
@@ -133,6 +157,15 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
"AssistantKnowledgeEntityRow",
"AssistantKnowledgeEntityContributionRow",
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
@@ -211,6 +244,16 @@ __all__ = [
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
"SkillKnowledgeArtifactRow",
"SkillKnowledgeBindingRow",
"SkillKnowledgeEntityContributionRow",
"SkillKnowledgeEntityRow",
"SkillKnowledgeRelationContributionRow",
"SkillKnowledgeRelationRow",
"SkillKnowledgeReviewRow",
"SkillKnowledgeSnapshotRow",
"SkillKnowledgeSyncItemRow",
"SkillKnowledgeSyncJobRow",
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
@@ -0,0 +1,15 @@
from deerflow.persistence.skill_knowledge.model import ( # noqa: F401
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]
@@ -0,0 +1,203 @@
"""ORM rows for skill-to-knowledge distillation and review."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class SkillKnowledgeBindingRow(Base):
__tablename__ = "skill_knowledge_bindings"
__table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
target_id: Mapped[str] = mapped_column(String(128), index=True)
target_name: Mapped[str] = mapped_column(String(255), default="")
remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeSnapshotRow(Base):
__tablename__ = "skill_knowledge_snapshots"
__table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
source_digest: Mapped[str] = mapped_column(String(80), index=True)
extractor_version: Mapped[str] = mapped_column(String(64))
prompt_hash: Mapped[str] = mapped_column(String(80))
model_name: Mapped[str] = mapped_column(String(128))
manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="ready")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeSyncJobRow(Base):
__tablename__ = "skill_knowledge_sync_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
trigger: Mapped[str] = mapped_column(String(32), default="manual")
target_count: Mapped[int] = mapped_column(Integer, default=0)
targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
total_count: Mapped[int] = mapped_column(Integer, default=0)
success_count: Mapped[int] = mapped_column(Integer, default=0)
partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
failed_count: Mapped[int] = mapped_column(Integer, default=0)
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeSyncItemRow(Base):
__tablename__ = "skill_knowledge_sync_items"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
target_type: Mapped[str] = mapped_column(String(32))
target_mode: Mapped[str] = mapped_column(String(32))
target_id: Mapped[str] = mapped_column(String(128))
skill_name: Mapped[str] = mapped_column(String(255), index=True)
snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="scanning")
progress: Mapped[int] = mapped_column(Integer, default=0)
attempts: Mapped[int] = mapped_column(Integer, default=0)
active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeArtifactRow(Base):
__tablename__ = "skill_knowledge_artifacts"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
kind: Mapped[str] = mapped_column(String(48), index=True)
slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeReviewRow(Base):
__tablename__ = "skill_knowledge_reviews"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
item_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
object_type: Mapped[str] = mapped_column(String(32), index=True)
object_key: Mapped[str] = mapped_column(String(128))
original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeEntityRow(Base):
__tablename__ = "skill_knowledge_entities"
__table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512))
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeEntityContributionRow(Base):
__tablename__ = "skill_knowledge_entity_contributions"
__table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_entity_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class SkillKnowledgeRelationRow(Base):
__tablename__ = "skill_knowledge_relations"
__table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeRelationContributionRow(Base):
__tablename__ = "skill_knowledge_relation_contributions"
__table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_relation_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,795 @@
"""SQL repository for skill knowledge jobs, snapshots and contributions."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
list_json_fields = {
"aliases_json",
"evidence_json",
"manifest_json",
"refs_json",
"targets_json",
}
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
return data
class SkillKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
entities = (
await session.execute(
select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == target_type,
SkillKnowledgeEntityRow.target_id == target_id,
)
)
).scalars()
for entity in entities:
contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (
await session.execute(
select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == target_type,
SkillKnowledgeRelationRow.target_id == target_id,
)
)
).scalars()
for relation in relations:
contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def create_job(
self,
*,
skill_names: list[str],
targets: list[dict[str, Any]],
created_by: str | None,
force: bool = False,
trigger: str = "manual",
review_mode: str = "auto_high_confidence",
) -> dict[str, Any]:
names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
normalized_targets: list[dict[str, Any]] = []
seen_targets: set[tuple[str, str]] = set()
for target in targets:
target_type = str(target.get("target_type") or "").strip()
target_id = str(target.get("target_id") or "").strip()
if not target_type or not target_id or (target_type, target_id) in seen_targets:
continue
seen_targets.add((target_type, target_id))
normalized_targets.append(
{
"target_type": target_type,
"target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
"target_id": target_id,
"target_name": str(target.get("target_name") or target_id),
"remote_business_key": target.get("remote_business_key"),
"readonly": bool(target.get("readonly", False)),
}
)
job_id = str(uuid4())
now = _now()
async with self._sf() as session:
job = SkillKnowledgeSyncJobRow(
id=job_id,
trigger=trigger,
target_count=len(normalized_targets),
targets_json=normalized_targets,
status="queued",
total_count=0,
review_mode=review_mode,
created_by=created_by,
created_at=now,
)
session.add(job)
for skill_name in names:
for target in normalized_targets:
stmt = select(SkillKnowledgeBindingRow).where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.target_type == target["target_type"],
SkillKnowledgeBindingRow.target_id == target["target_id"],
)
binding = (await session.execute(stmt)).scalar_one_or_none()
if binding is None:
binding = SkillKnowledgeBindingRow(
id=str(uuid4()),
skill_name=skill_name,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
target_name=target["target_name"],
remote_business_key=target.get("remote_business_key"),
status="queued",
created_by=created_by,
created_at=now,
updated_at=now,
)
session.add(binding)
else:
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
continue
binding.target_mode = target["target_mode"]
binding.target_name = target["target_name"]
binding.remote_business_key = target.get("remote_business_key")
binding.status = "queued"
binding.active = True
# The current manual submitter owns the relationship UI
# from this point onward. Target write authorization is
# enforced by the router before this method is called.
if trigger == "manual":
binding.created_by = created_by
binding.updated_at = now
item = SkillKnowledgeSyncItemRow(
id=str(uuid4()),
job_id=job_id,
binding_id=binding.id,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
skill_name=skill_name,
status="queued",
phase="scanning",
progress=0,
active_dedupe_key=binding.id,
created_at=now,
updated_at=now,
)
session.add(item)
binding.last_job_id = job_id
binding.last_item_id = item.id
job.total_count += 1
if job.total_count == 0:
job.status = "completed"
job.completed_at = now
await session.flush()
result = _dict(job)
result["force"] = force
await session.commit()
return result
async def list_bindings(
self,
*,
skill_names: list[str] | None = None,
target_id: str | None = None,
include_detached: bool = False,
created_by: str | None = None,
) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeBindingRow)
if skill_names:
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
if target_id:
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
if not include_detached:
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
if created_by is not None:
stmt = stmt.where(SkillKnowledgeBindingRow.created_by == created_by)
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
return _dict(row) if row else None
async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
if active_only:
stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def has_active_item(self, binding_id: str) -> bool:
async with self._sf() as session:
value = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
return value is not None
async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
if row is None or not row.active:
return None
row.auto_sync_enabled = enabled
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return None
items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
result = _dict(job)
result["items"] = [_dict(item) for item in items]
return result
async def list_jobs(self, *, limit: int = 50, created_by: str | None = None) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
if created_by is not None:
stmt = stmt.where(SkillKnowledgeSyncJobRow.created_by == created_by)
async with self._sf() as session:
jobs = list((await session.execute(stmt)).scalars())
if not jobs:
return []
job_ids = [row.id for row in jobs]
item_rows = list(
(
await session.execute(
select(SkillKnowledgeSyncItemRow)
.where(SkillKnowledgeSyncItemRow.job_id.in_(job_ids))
.order_by(SkillKnowledgeSyncItemRow.created_at)
)
).scalars()
)
items_by_job: dict[str, list[dict[str, Any]]] = {job_id: [] for job_id in job_ids}
for item in item_rows:
items_by_job[item.job_id].append(_dict(item))
result = []
for job in jobs:
data = _dict(job)
data["items"] = items_by_job[job.id]
result.append(data)
return result
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
now = _now()
stmt = (
select(SkillKnowledgeSyncItemRow)
.where(
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
)
)
.order_by(SkillKnowledgeSyncItemRow.created_at)
.limit(limit)
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def claim_item(
self,
item_id: str,
*,
owner: str,
lease_seconds: int = 900,
) -> dict[str, Any] | None:
"""Atomically claim a queued item, or reclaim one whose worker lease expired."""
now = _now()
stmt = (
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
),
)
.values(
status="running",
started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
attempts=SkillKnowledgeSyncItemRow.attempts + 1,
lease_owner=owner,
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
async with self._sf() as session:
result = await session.execute(stmt)
if result.rowcount != 1:
await session.rollback()
return None
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
SkillKnowledgeSyncItemRow.status == "running",
SkillKnowledgeSyncItemRow.lease_owner == owner,
)
.values(
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
await session.commit()
return result.rowcount == 1
async def update_item(
self,
item_id: str,
*,
status: str | None = None,
phase: str | None = None,
progress: int | None = None,
snapshot_id: str | None = None,
error_code: str | None = None,
error_message: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
if status is not None:
row.status = status
if status == "running" and row.started_at is None:
row.started_at = _now()
row.attempts += 1
if status in {"completed", "partial_failed", "failed", "canceled"}:
row.completed_at = _now()
row.active_dedupe_key = None
elif status == "queued":
row.completed_at = None
if status != "running":
row.lease_owner = None
row.lease_until = None
if phase is not None:
row.phase = phase
if progress is not None:
row.progress = max(0, min(100, progress))
if snapshot_id is not None:
row.snapshot_id = snapshot_id
row.last_error_code = error_code
row.last_error_message = error_message
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def refresh_job(self, job_id: str) -> None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return
rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
counts = {str(status): int(count) for status, count in rows}
job.success_count = counts.get("completed", 0)
job.partial_failed_count = counts.get("partial_failed", 0)
job.failed_count = counts.get("failed", 0)
active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
if counts.get("running", 0) or counts.get("awaiting_review", 0):
job.status = "running"
job.started_at = job.started_at or _now()
elif active == 0 and sum(counts.values()) >= job.total_count:
if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
job.status = "failed"
elif job.failed_count or job.partial_failed_count:
job.status = "partial_failed"
else:
job.status = "completed"
job.completed_at = _now()
await session.commit()
async def cancel_job(self, job_id: str) -> bool:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return False
job.cancel_requested = True
await session.execute(
SkillKnowledgeSyncItemRow.__table__.update()
.where(
SkillKnowledgeSyncItemRow.job_id == job_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
)
.values(
status="canceled",
phase="completed",
completed_at=_now(),
progress=100,
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
)
)
job.status = "canceled"
job.completed_at = _now()
await session.commit()
return True
async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
SkillKnowledgeSyncItemRow.id != row.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
return None
row.status = "queued"
row.phase = "scanning"
row.progress = 0
row.completed_at = None
row.active_dedupe_key = row.binding_id
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
stmt = select(SkillKnowledgeSnapshotRow).where(
SkillKnowledgeSnapshotRow.skill_name == skill_name,
SkillKnowledgeSnapshotRow.source_digest == source_digest,
SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
SkillKnowledgeSnapshotRow.model_name == model_name,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = SkillKnowledgeSnapshotRow(
id=str(uuid4()),
skill_name=skill_name,
source_digest=source_digest,
extractor_version=extractor_version,
prompt_hash=prompt_hash,
model_name=model_name,
manifest_json=manifest,
ir_json=ir,
counts_json=counts,
status="ready",
created_at=_now(),
)
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
return _dict(row) if row else None
async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
rows = [
SkillKnowledgeReviewRow(
id=str(uuid4()),
job_id=job_id,
item_id=item_id,
binding_id=binding_id,
snapshot_id=snapshot_id,
object_type=kind,
object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
original_json=obj,
status="pending",
created_at=_now(),
)
for kind, obj in objects
]
async with self._sf() as session:
session.add_all(rows)
await session.commit()
return len(rows)
async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeReviewRow)
if job_id:
stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
if item_id:
stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
if status:
stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
final_status = "approved" if action == "approve" else "rejected"
affected_items: set[str] = set()
async with self._sf() as session:
rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
for row in rows:
row.status = final_status
row.edited_json = (edits or {}).get(row.id)
row.reason = reason
row.reviewed_by = actor
row.reviewed_at = _now()
affected_items.add(row.item_id)
await session.commit()
return sorted(affected_items)
async def pending_review_count(self, item_id: str) -> int:
stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
snapshot = await self.get_snapshot(snapshot_id)
if snapshot is None:
return None
ir = dict(snapshot.get("ir") or {})
reviews = await self.list_reviews(item_id=item_id, status=None)
if not reviews:
return ir
by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
for key in ("entities", "relations"):
kind = key[:-1]
accepted = []
for obj in ir.get(key, []):
review = by_key.get((kind, str(obj.get("local_id"))))
if review and review.get("status") == "rejected":
continue
accepted.append((review or {}).get("edited") or obj)
ir[key] = accepted
return ir
async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
now = _now()
async with self._sf() as session:
item = await session.get(SkillKnowledgeSyncItemRow, item_id)
if item is None:
raise KeyError(item_id)
binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
if binding is None:
raise KeyError(item.binding_id)
if not binding.active:
raise RuntimeError("Binding was detached before staged activation")
await session.execute(
SkillKnowledgeArtifactRow.__table__.update()
.where(
SkillKnowledgeArtifactRow.binding_id == binding.id,
SkillKnowledgeArtifactRow.status == "active",
)
.values(status="retired", updated_at=now)
)
for artifact in artifacts:
session.add(
SkillKnowledgeArtifactRow(
id=str(uuid4()),
binding_id=binding.id,
snapshot_id=snapshot_id,
kind=str(artifact.get("kind") or "wiki_page"),
slug=artifact.get("slug"),
content_text=artifact.get("content"),
remote_id=artifact.get("remote_id"),
business_key=artifact.get("business_key"),
remote_refs_json=artifact.get("remote_refs") or [],
status="active",
metadata_json=artifact.get("metadata") or {},
created_at=now,
updated_at=now,
)
)
# Retire contributions from the old active version only after all staged artifacts are ready.
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
entity_ids: dict[str, str] = {}
for entity in ir.get("entities", []):
normalized = normalize_name(str(entity.get("name") or ""))
if not normalized:
continue
entity_type = str(entity.get("type") or "concept")
stmt = select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == binding.target_type,
SkillKnowledgeEntityRow.target_id == binding.target_id,
SkillKnowledgeEntityRow.entity_type == entity_type,
SkillKnowledgeEntityRow.normalized_name == normalized,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeEntityRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
entity_type=entity_type,
normalized_name=normalized,
display_name=str(entity.get("name") or normalized),
aliases_json=entity.get("aliases") or [],
attributes_json=entity.get("attributes") or {},
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
entity_ids[str(entity.get("local_id"))] = canonical.id
session.add(
SkillKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_entity_id=str(entity.get("local_id")),
evidence_json=entity.get("evidence") or [],
attributes_json=entity.get("attributes") or {},
confidence=float(entity.get("confidence") or 0),
)
)
for relation in ir.get("relations", []):
source_id = entity_ids.get(str(relation.get("source_local_id")))
target_id = entity_ids.get(str(relation.get("target_local_id")))
confidence = float(relation.get("confidence") or 0)
if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
continue
predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
stmt = select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == binding.target_type,
SkillKnowledgeRelationRow.target_id == binding.target_id,
SkillKnowledgeRelationRow.source_entity_id == source_id,
SkillKnowledgeRelationRow.predicate == predicate,
SkillKnowledgeRelationRow.target_entity_id == target_id,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeRelationRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
source_entity_id=source_id,
predicate=predicate,
target_entity_id=target_id,
description=str(relation.get("description") or ""),
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
session.add(
SkillKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_relation_id=str(relation.get("local_id")),
evidence_json=relation.get("evidence") or [],
confidence=confidence,
)
)
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
binding.current_snapshot_id = snapshot_id
binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
binding.synced_digest = binding.source_digest
binding.status = status
binding.last_synced_at = now
binding.last_error_code = None
binding.last_error_message = None
binding.updated_at = now
item.snapshot_id = snapshot_id
item.status = "completed" if status in {"synced", "pending_remote_write"} else status
item.phase = "completed"
item.progress = 100
item.completed_at = now
item.active_dedupe_key = None
item.lease_owner = None
item.lease_until = None
item.updated_at = now
await session.flush()
result = _dict(binding)
await session.commit()
await self.refresh_job(item.job_id)
return result
async def detach_binding(
self,
binding_id: str,
*,
delete_contributions: bool,
cleanup_failed: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
binding = await session.get(SkillKnowledgeBindingRow, binding_id)
if binding is None:
return None
binding.active = False
binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
binding.updated_at = _now()
if delete_contributions:
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
await session.flush()
data = _dict(binding)
await session.commit()
return data
async def mark_skill_stale(self, skill_name: str) -> int:
async with self._sf() as session:
result = await session.execute(
SkillKnowledgeBindingRow.__table__.update()
.where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.active.is_(True),
SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
)
.values(status="stale", updated_at=_now())
)
await session.commit()
return int(result.rowcount or 0)
def make_skill_knowledge_store(session_factory):
return SkillKnowledgeRepository(session_factory) if session_factory is not None else None
@@ -3,7 +3,7 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
@@ -14,6 +14,12 @@ from sqlalchemy.types import TypeDecorator
# fixed offset is exact and avoids depending on the `tzdata` package (which the
# stdlib `zoneinfo` needs on Windows / minimal containers).
BEIJING_TZ = timezone(timedelta(hours=8))
_POSTGRES_FAMILY_DIALECTS = frozenset({"postgresql", "opengauss"})
def _is_postgres_family(dialect) -> bool:
"""Return whether *dialect* preserves timezone-aware timestamps natively."""
return dialect.name in _POSTGRES_FAMILY_DIALECTS
class PortableJSON(TypeDecorator):
@@ -87,17 +93,17 @@ class BeijingDateTime(TypeDecorator):
def process_bind_param(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Naive values follow the codebase convention of being UTC.
value = value.replace(tzinfo=timezone.utc)
value = value.replace(tzinfo=UTC)
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
def process_result_value(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Stored as naive Beijing wall-clock; re-attach the zone.
@@ -0,0 +1,10 @@
"""Skill knowledge distillation primitives.
The package is deliberately independent from ``app.*``. Gateway composition
injects persistence and remote adapters at runtime.
"""
from deerflow.skill_knowledge.extractor import build_skill_ir
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
@@ -0,0 +1,152 @@
"""IR construction and validation.
V1 intentionally starts with deterministic, evidence-backed extraction. The
schema is compatible with a future model-backed extractor, while never letting
unvalidated model output flow directly into persistence or graph writers.
"""
from __future__ import annotations
import hashlib
import re
from typing import Any
from deerflow.skill_knowledge.scanner import ScanResult
EXTRACTOR_VERSION = "skill-knowledge-v1"
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
PREDICATE_PATTERNS = {
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
}
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
line_start = text.count("\n", 0, start) + 1
line_end = text.count("\n", 0, end) + 1
quote = " ".join(text[start:end].split())[:240]
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
entities: list[dict[str, Any]] = [
{
"local_id": "ent_skill",
"type": "skill",
"name": display_name or scan.skill_name,
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
"description": description,
"attributes": {},
"evidence": [],
"confidence": 1.0,
}
]
relations: list[dict[str, Any]] = []
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
for file in scan.files:
if not file.text:
continue
for predicate, pattern in PREDICATE_PATTERNS.items():
for match in pattern.finditer(file.text):
name = match.group(1).strip(" `\"'“”()[]")
if not name or len(name) > 80:
continue
key = ("concept", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "concept",
"name": name,
"aliases": [],
"description": "从技能业务说明中抽取的知识对象",
"attributes": {},
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": predicate,
"target_local_id": entity_id,
"description": " ".join(match.group(0).split())[:240],
"relation_source": "explicit_text",
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
for ref in file.references:
name = ref["target_path"]
key = ("file", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "file",
"name": name,
"aliases": [],
"description": "技能文档引用的资料文件",
"attributes": {},
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": "references",
"target_local_id": entity_id,
"description": f"{file.path} 引用 {name}",
"relation_source": "deterministic_structure",
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
title = display_name or scan.skill_name
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
content = "\n".join(
[
f"# {title}",
"",
description or "由技能知识文件安全归纳生成。",
"",
"## 知识资料",
"",
*(summaries or ["- 暂无可归纳知识文件"]),
"",
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
]
)
return {
"extractor_version": EXTRACTOR_VERSION,
"prompt_hash": PROMPT_HASH,
"model_name": "deterministic-v1",
"skill": {
"name": scan.skill_name,
"display_name": title,
"description": description,
"source_root": scan.source_root,
"source_digest": scan.source_digest,
},
"files": scan.manifest,
"entities": entities,
"relations": relations,
"wiki": {
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
"pages": [],
},
"safety": {"blocked_files": scan.blocked_files},
}
@@ -0,0 +1,115 @@
"""Guarded graph-write plans.
No caller can submit Cypher. The writer accepts only normalized node/edge
records and executes fixed parameterized templates when explicitly enabled.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from typing import Any, Protocol
class GraphTransaction(Protocol):
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
@dataclass(frozen=True, slots=True)
class GraphWritePlan:
scope: str
nodes: tuple[dict[str, Any], ...]
edges: tuple[dict[str, Any], ...]
NODE_QUERY = """
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
SET n.display_name = $display_name, n.entity_type = $entity_type,
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
n.confidence = $confidence, n.updated_at = datetime()
""".strip()
EDGE_QUERY = """
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
SET r.predicate = $predicate, r.binding_id = $binding_id,
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
r.evidence = $evidence, r.updated_at = datetime()
""".strip()
def _key(*parts: str) -> str:
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
def plan_from_ir(
ir: dict[str, Any],
*,
target_type: str,
target_id: str,
binding_id: str,
snapshot_id: str,
job_id: str,
) -> GraphWritePlan:
"""Build a stable write plan; callers cannot supply executable Cypher."""
scope = f"{target_type}:{target_id}"
local_keys: dict[str, str] = {}
nodes: list[dict[str, Any]] = []
for entity in ir.get("entities") or []:
entity_type = str(entity.get("type") or "concept")
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
if not normalized_name:
continue
entity_key = _key(entity_type, normalized_name)
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
nodes.append(
{
"entity_key": entity_key,
"display_name": str(entity.get("name") or normalized_name),
"entity_type": entity_type,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": float(entity.get("confidence") or 1.0),
}
)
edges: list[dict[str, Any]] = []
for relation in ir.get("relations") or []:
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
evidence = list(relation.get("evidence") or [])
confidence = float(relation.get("confidence") or 0)
predicate = str(relation.get("predicate") or "related_to")
if not source_key or not target_key or confidence < 0.75 or not evidence:
continue
edges.append(
{
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
"source_key": source_key,
"target_key": target_key,
"predicate": predicate,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": confidence,
"evidence": evidence,
}
)
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
"""Return Neo4j transactional HTTP statements using only fixed templates."""
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
return statements
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
if not enabled:
return {"nodes": 0, "edges": 0}
for node in plan.nodes:
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
for edge in plan.edges:
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
@@ -0,0 +1,31 @@
"""Portable entity/relation normalization used by both knowledge products."""
from __future__ import annotations
import re
import unicodedata
def normalize_name(value: str) -> str:
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
text = text.strip("`'\"“”‘’《》〈〉[](){}")
text = re.sub(r"\s+", " ", text)
return text
def normalize_predicate(value: str) -> str:
normalized = normalize_name(value).replace(" ", "_")
aliases = {
"依赖": "depends_on",
"需要": "depends_on",
"requires": "depends_on",
"调用": "uses",
"使用": "uses",
"uses": "uses",
"生成": "produces",
"输出": "produces",
"读取": "consumes",
"输入": "consumes",
"引用": "references",
}
return aliases.get(normalized, normalized or "related_to")
@@ -0,0 +1,351 @@
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
from __future__ import annotations
import csv
import hashlib
import json
import re
import zipfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
CODE_SUFFIXES = {
".py",
".pyc",
".pyo",
".ts",
".tsx",
".js",
".jsx",
".mjs",
".cjs",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".psm1",
".java",
".kt",
".kts",
".go",
".rs",
".c",
".cc",
".cpp",
".h",
".hpp",
".cs",
".php",
".rb",
".pl",
".lua",
".swift",
".scala",
".sql",
".vue",
".svelte",
}
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
EXCLUDED_FILENAMES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"bun.lockb",
"poetry.lock",
"uv.lock",
"pipfile.lock",
"cargo.lock",
"composer.lock",
}
SENSITIVE_NAME_RE = re.compile(
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
re.IGNORECASE,
)
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
MAX_TEXT_BYTES = 2 * 1024 * 1024
MAX_ARCHIVE_FILES = 1_000
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
@dataclass(slots=True)
class ScannedFile:
path: str
kind: str
sha256: str
size: int
text: str = ""
title: str = ""
references: list[dict[str, Any]] = field(default_factory=list)
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
def manifest(self) -> dict[str, Any]:
return {
"path": self.path,
"kind": self.kind,
"sha256": self.sha256,
"size": self.size,
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
"title": self.title,
"references": self.references,
"safety": self.safety,
}
@dataclass(slots=True)
class ScanResult:
skill_name: str
source_root: str
source_digest: str
files: list[ScannedFile]
blocked_files: list[dict[str, str]]
skipped_code_count: int
skipped_other_count: int
@property
def manifest(self) -> list[dict[str, Any]]:
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
@property
def counts(self) -> dict[str, int]:
return {
"knowledge_files": len(self.files),
"blocked_files": len(self.blocked_files),
"skipped_code_files": self.skipped_code_count,
"skipped_other_files": self.skipped_other_count,
}
def _safe_relative(path: Path, root: Path) -> str:
resolved = path.resolve()
resolved_root = root.resolve()
try:
relative = resolved.relative_to(resolved_root)
except ValueError as exc:
raise ValueError("Skill file escapes its source root") from exc
return relative.as_posix()
def _strip_markdown_code(text: str) -> str:
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
def _read_text(path: Path, suffix: str) -> str:
raw = path.read_bytes()
if len(raw) > MAX_TEXT_BYTES:
raise ValueError("text file exceeds 2 MiB parsing limit")
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
return _strip_markdown_code(text)
if suffix in {".csv", ".tsv"}:
delimiter = "\t" if suffix == ".tsv" else ","
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
if suffix == ".json":
try:
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
except json.JSONDecodeError:
return text
return text
def _title(text: str, fallback: str) -> str:
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
return match.group(1).strip() if match else fallback
def _kind(suffix: str) -> str:
return {
".md": "markdown",
".mdx": "markdown",
".txt": "text",
".csv": "table",
".tsv": "table",
".xlsx": "spreadsheet",
".xls": "spreadsheet",
".pdf": "pdf",
".docx": "office",
".pptx": "office",
".png": "image",
".jpg": "image",
".jpeg": "image",
".webp": "image",
".gif": "image",
".bmp": "image",
".zip": "archive",
}.get(suffix, "structured_data")
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
with zipfile.ZipFile(path) as archive:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_FILES:
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
total_size = sum(member.file_size for member in members)
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
for member in members:
member_path = Path(member.filename.replace("\\", "/"))
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
unix_mode = member.external_attr >> 16
if member.is_dir():
continue
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
continue
suffix = member_path.suffix.lower()
lower_name = member_path.name.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
skipped_other += 1
continue
raw = archive.read(member)
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
continue
files.append(
ScannedFile(
path=virtual_path,
kind=_kind(suffix),
sha256=hashlib.sha256(raw).hexdigest(),
size=len(raw),
text=text,
title=_title(text, member_path.stem),
)
)
return files, blocked, skipped_code, skipped_other
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
"""Scan one already-resolved skill directory without following symlinks."""
root = skill_root.resolve()
if not root.is_dir():
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
if path.is_symlink() or not path.is_file():
continue
relative = _safe_relative(path, root)
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
continue
lower_name = path.name.lower()
suffix = path.suffix.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
skipped_other += 1
continue
# Never upload an archive wholesale: it may contain excluded code or a
# blocked secret. Only independently validated knowledge members enter
# the manifest and digest.
if suffix == ".zip":
try:
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
files.extend(nested)
blocked.extend(nested_blocked)
skipped_code += nested_code
skipped_other += nested_other
except (OSError, zipfile.BadZipFile):
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
continue
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
text = ""
warnings: list[str] = []
if suffix in TEXT_SUFFIXES:
try:
text = _read_text(path, suffix)
except ValueError as exc:
warnings.append(str(exc))
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
continue
references = []
if suffix in {".md", ".mdx"}:
for target in MARKDOWN_LINK_RE.findall(text):
normalized = target.replace("\\", "/").lstrip("./")
if normalized:
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
files.append(
ScannedFile(
path=relative,
kind=_kind(suffix),
sha256=digest,
size=len(raw),
text=text,
title=_title(text, path.stem),
references=references,
safety={"status": "allowed", "warnings": warnings},
)
)
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return ScanResult(
skill_name=skill_name,
source_root=str(root),
source_digest=source_digest,
files=files,
blocked_files=blocked,
skipped_code_count=skipped_code,
skipped_other_count=skipped_other,
)
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
allowed: list[ScannedFile] = []
root = Path(scan.source_root)
for item in scan.files:
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
text = await extract_file_to_markdown(root / item.path)
if text:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
continue
item.text = text
item.title = _title(text, item.title)
else:
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
allowed.append(item)
scan.files = allowed
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return scan
@@ -256,6 +256,24 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
return None
async def extract_file_to_markdown(file_path: Path) -> str | None:
"""Extract a supported document to Markdown without creating a sidecar file.
Knowledge-ingest scanners use this variant because running a read-only
distillation task must never add generated ``.md`` files to a live skill.
Conversion always runs off the event loop; Office/PDF parsers can be CPU
intensive even for deceptively small inputs.
"""
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
try:
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
except Exception as exc:
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
return None
# Regex for bold-only lines that look like section headings.
# Targets SEC filing structural headings that pymupdf4llm renders as **bold**
# rather than # Markdown headings (because they use same font size as body text,
@@ -53,6 +53,10 @@ postgres = [
"psycopg-pool>=3.3.0",
]
mysql = ["asyncmy>=0.2.10"]
gauss = [
"asyncpg>=0.29",
"opengauss-sqlalchemy>=2.4,<3",
]
pymupdf = ["pymupdf4llm>=0.0.17"]
[build-system]