Merge remote-tracking branch 'origin/master' into xuhd
# Conflicts: # frontend-web/.env.production # frontend-web/.env.sit # frontend-web/package.json # offline-backend-20260512/backend/config.yaml
This commit is contained in:
commit
1668c1da87
177 files changed
+21030
-2738
No files matched your search
@@ -299,6 +299,16 @@ def _build_middlewares(
|
||||
if not is_bootstrap and not is_writing_setup:
|
||||
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
|
||||
|
||||
# KnowledgeRagMiddleware — inject knowledge-base recall (phase 2). Gated
|
||||
# per-conversation by the ``knowledge_rag_enabled`` runtime flag, defaulting
|
||||
# to the global ``knowledge.rag_enabled`` config. No-op when knowledge is
|
||||
# disabled. Skipped in bootstrap/writing-setup like MemoryMiddleware.
|
||||
knowledge_cfg = getattr(resolved_app_config, "knowledge", None)
|
||||
if not is_bootstrap and not is_writing_setup and knowledge_cfg is not None and knowledge_cfg.enabled:
|
||||
from deerflow.agents.middlewares.knowledge_rag_middleware import KnowledgeRagMiddleware
|
||||
|
||||
middlewares.append(KnowledgeRagMiddleware(limit=knowledge_cfg.rag_limit, default_enabled=knowledge_cfg.rag_enabled))
|
||||
|
||||
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
|
||||
# ingests raw image data). In delegated mode the view_image tool already
|
||||
# returns a text description and stores nothing in `viewed_images`, so the
|
||||
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
"""KnowledgeRagMiddleware — inject knowledge-base recall before the model call.
|
||||
|
||||
Phase 2 of the product knowledge base. When enabled (per-conversation runtime
|
||||
flag ``knowledge_rag_enabled``, falling back to the global
|
||||
``knowledge.rag_enabled`` config), the latest user question is used to retrieve
|
||||
the most relevant knowledge notes, which are injected as a transient
|
||||
``<knowledge-context>`` system message — not persisted to state, so it never
|
||||
leaks to the frontend transcript.
|
||||
|
||||
Retrieval is hybrid (vector + keyword) when an embedding endpoint is
|
||||
configured, else keyword/wiki-query layered search. Failures are swallowed: RAG
|
||||
must never break a conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TAG = "knowledge-context"
|
||||
|
||||
|
||||
def _coerce_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text", "")))
|
||||
return " ".join(parts)
|
||||
return str(content or "")
|
||||
|
||||
|
||||
def _latest_human_text(messages: list) -> str:
|
||||
for msg in reversed(messages or []):
|
||||
if isinstance(msg, HumanMessage):
|
||||
return _coerce_text(msg.content).strip()
|
||||
return ""
|
||||
|
||||
|
||||
class KnowledgeRagMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Inject knowledge-base recall as transient system context."""
|
||||
|
||||
state_schema = AgentState
|
||||
|
||||
def __init__(self, *, limit: int = 5, default_enabled: bool = False) -> None:
|
||||
super().__init__()
|
||||
self._limit = limit
|
||||
self._default_enabled = default_enabled
|
||||
self._service = None # lazy
|
||||
|
||||
def _get_service(self):
|
||||
if self._service is not None:
|
||||
return self._service
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.knowledge import make_knowledge_service
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
config = get_app_config()
|
||||
self._service = make_knowledge_service(get_session_factory(), config.knowledge)
|
||||
except Exception:
|
||||
logger.exception("KnowledgeRagMiddleware: failed to build knowledge service")
|
||||
self._service = None
|
||||
return self._service
|
||||
|
||||
def _enabled_for(self, request: Any) -> bool:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is not None and getattr(runtime, "context", None):
|
||||
flag = runtime.context.get("knowledge_rag_enabled")
|
||||
if flag is not None:
|
||||
return bool(flag)
|
||||
return self._default_enabled
|
||||
|
||||
async def _build_context_message(self, request: Any) -> HumanMessage | None:
|
||||
if not self._enabled_for(request):
|
||||
return None
|
||||
service = self._get_service()
|
||||
if service is None:
|
||||
return None
|
||||
query = _latest_human_text(getattr(request, "messages", None) or [])
|
||||
if not query:
|
||||
return None
|
||||
try:
|
||||
result = await service.build_rag_context(query, limit=self._limit)
|
||||
except Exception:
|
||||
logger.exception("KnowledgeRagMiddleware: recall failed (non-fatal)")
|
||||
return None
|
||||
text = result.get("text") if isinstance(result, dict) else ""
|
||||
if not text:
|
||||
return None
|
||||
fenced = (
|
||||
f"<{_TAG}>\n"
|
||||
"[系统提示:以下是知识库召回的背景资料,不是用户的新输入。回答时优先参考,"
|
||||
"并在使用到时标注来源 note_id。]\n\n"
|
||||
f"{text}\n"
|
||||
f"</{_TAG}>"
|
||||
)
|
||||
return HumanMessage(content=fenced)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(self, request: Any, handler: Any) -> Any:
|
||||
msg = await self._build_context_message(request)
|
||||
if msg is not None:
|
||||
request = request.override(messages=[*request.messages, msg])
|
||||
return await handler(request)
|
||||
|
||||
@override
|
||||
def wrap_model_call(self, request: Any, handler: Any) -> Any:
|
||||
# Sync path: best-effort. The gateway runtime is async, so this is rarely
|
||||
# hit; run the async recall in a private loop when there is none.
|
||||
if not self._enabled_for(request):
|
||||
return handler(request)
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
# Cannot block a running loop from sync code; skip injection.
|
||||
return handler(request)
|
||||
msg = loop.run_until_complete(self._build_context_message(request))
|
||||
except Exception:
|
||||
msg = None
|
||||
if msg is not None:
|
||||
request = request.override(messages=[*request.messages, msg])
|
||||
return handler(request)
|
||||
|
||||
# ---- after_agent: enqueue background auto-sedimentation -------------
|
||||
def _resolve_thread_id(self, runtime: Runtime | None) -> str | None:
|
||||
if runtime is not None and getattr(runtime, "context", None):
|
||||
tid = runtime.context.get("thread_id")
|
||||
if tid:
|
||||
return str(tid)
|
||||
try:
|
||||
from langgraph.config import get_config
|
||||
|
||||
return get_config().get("configurable", {}).get("thread_id")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@override
|
||||
def after_agent(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
"""When auto-ingest is on, hand the finished turn to the background queue."""
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
knowledge_cfg = getattr(get_app_config(), "knowledge", None)
|
||||
if knowledge_cfg is None or not knowledge_cfg.enabled or not knowledge_cfg.auto_ingest_enabled:
|
||||
return None
|
||||
|
||||
from deerflow.knowledge.auto_ingest import AutoIngestItem, get_auto_ingest_queue
|
||||
|
||||
queue = get_auto_ingest_queue()
|
||||
if queue is None:
|
||||
return None
|
||||
|
||||
messages = state.get("messages") if isinstance(state, dict) else getattr(state, "messages", None)
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
from deerflow.runtime.references import build_reference_batches
|
||||
from deerflow.runtime.serialization import serialize_lc_object
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
serialized = [serialize_lc_object(m) for m in messages]
|
||||
thread_id = self._resolve_thread_id(runtime)
|
||||
if not thread_id:
|
||||
return None
|
||||
title = state.get("title") if isinstance(state, dict) else getattr(state, "title", None)
|
||||
queue.enqueue(
|
||||
AutoIngestItem(
|
||||
thread_id=thread_id,
|
||||
user_id=get_effective_user_id(),
|
||||
messages=serialized,
|
||||
thread_title=title if isinstance(title, str) else None,
|
||||
reference_batches=build_reference_batches(serialized),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("KnowledgeRagMiddleware.after_agent enqueue failed (non-fatal)")
|
||||
return None
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Browser-backed page context helpers."""
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Small file-backed task queue for browser page fetch requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LOCK = threading.RLock()
|
||||
_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$")
|
||||
_TERMINAL_STATUSES = {"done", "error", "expired"}
|
||||
|
||||
|
||||
def tasks_dir() -> Path:
|
||||
"""Return the browser-context task directory, creating it if needed."""
|
||||
|
||||
path = runtime_home() / "browser-context" / "tasks"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None = None) -> str:
|
||||
return (value or _now()).isoformat()
|
||||
|
||||
|
||||
def _parse_iso(value: Any) -> datetime | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _task_path(task_id: str) -> Path | None:
|
||||
if not _TASK_ID_RE.fullmatch(task_id or ""):
|
||||
return None
|
||||
return tasks_dir() / f"{task_id}.json"
|
||||
|
||||
|
||||
def _read_task(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except Exception:
|
||||
logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True)
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _write_task(path: Path, data: dict[str, Any]) -> None:
|
||||
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
|
||||
with tmp.open("w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, separators=(",", ":"))
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _public_task(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": data.get("id"),
|
||||
"kind": data.get("kind") or "fetch",
|
||||
"url": data.get("url") or "",
|
||||
"instruction": data.get("instruction") or "",
|
||||
"session_id": data.get("session_id") or "",
|
||||
"action": data.get("action") or None,
|
||||
"created_at": data.get("created_at") or "",
|
||||
"updated_at": data.get("updated_at") or "",
|
||||
"max_content_chars": int(data.get("max_content_chars") or 200_000),
|
||||
}
|
||||
|
||||
|
||||
def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]:
|
||||
if data.get("status") in _TERMINAL_STATUSES:
|
||||
return data
|
||||
expires_at = _parse_iso(data.get("expires_at"))
|
||||
if expires_at is not None and now >= expires_at:
|
||||
data = dict(data)
|
||||
data["status"] = "expired"
|
||||
data["error"] = "Browser task expired before the extension returned a result."
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return data
|
||||
|
||||
|
||||
def prune_browser_tasks(max_age_seconds: int = 86_400) -> int:
|
||||
"""Delete old terminal tasks and return the number removed."""
|
||||
|
||||
cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds)))
|
||||
removed = 0
|
||||
with _LOCK:
|
||||
for path in tasks_dir().glob("*.json"):
|
||||
data = _read_task(path)
|
||||
if not data or data.get("status") not in _TERMINAL_STATUSES:
|
||||
continue
|
||||
updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at"))
|
||||
if updated_at is None or updated_at < cutoff:
|
||||
try:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
logger.debug("Failed to delete old browser task: %s", path, exc_info=True)
|
||||
return removed
|
||||
|
||||
|
||||
def create_browser_task(
|
||||
*,
|
||||
user_id: str,
|
||||
url: str,
|
||||
instruction: str = "",
|
||||
timeout_seconds: int = 60,
|
||||
max_content_chars: int = 200_000,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
kind: str = "fetch",
|
||||
session_id: str = "",
|
||||
action: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a browser task for the current user's extension to pick up.
|
||||
|
||||
``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or
|
||||
``"act"`` for an interactive observe/act step driven by ``action`` within a
|
||||
long-lived ``session_id`` (a tab kept open by the extension).
|
||||
"""
|
||||
|
||||
now = _now()
|
||||
task_id = uuid4().hex
|
||||
task = {
|
||||
"id": task_id,
|
||||
"user_id": str(user_id or "default"),
|
||||
"status": "pending",
|
||||
"kind": kind if kind in {"fetch", "act"} else "fetch",
|
||||
"url": url,
|
||||
"instruction": instruction or "",
|
||||
"session_id": str(session_id or ""),
|
||||
"action": action or None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))),
|
||||
"max_content_chars": max(1_000, int(max_content_chars)),
|
||||
"worker_id": None,
|
||||
"claimed_at": None,
|
||||
"result": None,
|
||||
"error": None,
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
path = tasks_dir() / f"{task_id}.json"
|
||||
with _LOCK:
|
||||
_write_task(path, task)
|
||||
return task
|
||||
|
||||
|
||||
def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
"""Read a task if it belongs to the requested user."""
|
||||
|
||||
path = _task_path(task_id)
|
||||
if path is None:
|
||||
return None
|
||||
with _LOCK:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
|
||||
return None
|
||||
return _expire_if_needed(path, data, _now())
|
||||
|
||||
|
||||
def claim_next_browser_task(
|
||||
*,
|
||||
user_id: str,
|
||||
worker_id: str,
|
||||
lease_seconds: int = 120,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Claim the oldest pending task for a user."""
|
||||
|
||||
now = _now()
|
||||
user_id = str(user_id or "default")
|
||||
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
|
||||
lease_seconds = max(10, int(lease_seconds))
|
||||
|
||||
with _LOCK:
|
||||
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||
for path in paths:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != user_id:
|
||||
continue
|
||||
|
||||
data = _expire_if_needed(path, data, now)
|
||||
status = data.get("status")
|
||||
if status == "running":
|
||||
updated_at = _parse_iso(data.get("updated_at")) or now
|
||||
if (now - updated_at).total_seconds() <= lease_seconds:
|
||||
continue
|
||||
status = "pending"
|
||||
data = dict(data)
|
||||
data["status"] = status
|
||||
|
||||
if status != "pending":
|
||||
continue
|
||||
|
||||
data = dict(data)
|
||||
data["status"] = "running"
|
||||
data["worker_id"] = worker_id
|
||||
data["claimed_at"] = _iso(now)
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return _public_task(data)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def complete_browser_task(
|
||||
*,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
title: str = "",
|
||||
url: str = "",
|
||||
content: str = "",
|
||||
content_type: str = "markdown",
|
||||
error: str = "",
|
||||
metadata: dict[str, Any] | None = None,
|
||||
observation: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Mark a browser task complete or failed.
|
||||
|
||||
For ``act`` tasks the extension returns a structured ``observation`` (final
|
||||
URL/title, interactive element list, page text) instead of a captured
|
||||
markdown document; it is stored alongside the result verbatim.
|
||||
"""
|
||||
|
||||
path = _task_path(task_id)
|
||||
if path is None:
|
||||
return None
|
||||
|
||||
now = _now()
|
||||
with _LOCK:
|
||||
data = _read_task(path)
|
||||
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
|
||||
return None
|
||||
|
||||
data = dict(data)
|
||||
if error:
|
||||
data["status"] = "error"
|
||||
data["error"] = str(error).strip()[:4000]
|
||||
data["result"] = None
|
||||
else:
|
||||
raw_content = str(content or "").strip()
|
||||
max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000))
|
||||
truncated = len(raw_content) > max_content_chars
|
||||
data["status"] = "done"
|
||||
data["error"] = None
|
||||
data["result"] = {
|
||||
"title": str(title or "").strip()[:500],
|
||||
"url": str(url or data.get("url") or "").strip()[:4096],
|
||||
"content": raw_content[:max_content_chars] if truncated else raw_content,
|
||||
"content_type": content_type if content_type in {"markdown", "text"} else "markdown",
|
||||
"truncated": truncated,
|
||||
"observation": observation if isinstance(observation, dict) else None,
|
||||
"captured_at": _iso(now),
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
data["updated_at"] = _iso(now)
|
||||
_write_task(path, data)
|
||||
return data
|
||||
|
||||
|
||||
async def wait_for_browser_task_result(
|
||||
*,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
timeout_seconds: int,
|
||||
poll_interval_seconds: float = 1.0,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Wait for a browser task to reach a terminal state."""
|
||||
|
||||
deadline = time.monotonic() + max(1, int(timeout_seconds))
|
||||
interval = max(0.25, float(poll_interval_seconds))
|
||||
while True:
|
||||
task = get_browser_task(task_id=task_id, user_id=user_id)
|
||||
if task is None:
|
||||
return None
|
||||
if task.get("status") in _TERMINAL_STATUSES:
|
||||
return task
|
||||
if time.monotonic() >= deadline:
|
||||
return task
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
@@ -10,11 +10,13 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.gateway_config import GatewayConfig
|
||||
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.log_system_config import LogSystemConfig
|
||||
from deerflow.config.login_config import AuthLoginConfig
|
||||
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
|
||||
@@ -133,6 +135,8 @@ class AppConfig(BaseModel):
|
||||
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
|
||||
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
|
||||
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
|
||||
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
|
||||
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
|
||||
|
||||
@classmethod
|
||||
def resolve_config_path(cls, config_path: str | None = None) -> Path:
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Configuration for browser page context capture."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class BrowserContextConfig(BaseModel):
|
||||
"""Controls the optional Chrome-extension page capture integration."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable /api/browser-context endpoints for Chrome extension page capture.",
|
||||
)
|
||||
max_content_chars: int = Field(
|
||||
default=200_000,
|
||||
ge=1_000,
|
||||
description="Maximum captured page text stored per request. Longer content is truncated.",
|
||||
)
|
||||
max_title_chars: int = Field(
|
||||
default=300,
|
||||
ge=20,
|
||||
description="Maximum title length accepted from the browser extension.",
|
||||
)
|
||||
max_url_chars: int = Field(
|
||||
default=2_048,
|
||||
ge=100,
|
||||
description="Maximum source URL length accepted from the browser extension.",
|
||||
)
|
||||
filename_prefix: str = Field(
|
||||
default="webpage",
|
||||
description="Prefix used for Markdown files created from captured pages.",
|
||||
)
|
||||
route_base: str = Field(
|
||||
default="/page/workspace",
|
||||
description="Frontend route base used when returning chat paths to the extension.",
|
||||
)
|
||||
task_result_timeout_seconds: int = Field(
|
||||
default=60,
|
||||
ge=5,
|
||||
description="Maximum time the browser_fetch_page tool waits for the extension result.",
|
||||
)
|
||||
task_poll_interval_seconds: float = Field(
|
||||
default=1.0,
|
||||
ge=0.25,
|
||||
description="How often the backend tool checks for a returned browser task result.",
|
||||
)
|
||||
task_lease_seconds: int = Field(
|
||||
default=120,
|
||||
ge=10,
|
||||
description="How long a claimed browser task is reserved before another extension may retry it.",
|
||||
)
|
||||
tool_result_max_chars: int = Field(
|
||||
default=60_000,
|
||||
ge=1_000,
|
||||
description="Maximum page characters returned from browser_fetch_page into the model context.",
|
||||
)
|
||||
actions_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).",
|
||||
)
|
||||
action_result_timeout_seconds: int = Field(
|
||||
default=45,
|
||||
ge=5,
|
||||
description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.",
|
||||
)
|
||||
max_observed_elements: int = Field(
|
||||
default=150,
|
||||
ge=10,
|
||||
description="Maximum number of interactive elements returned in a browser_act observation.",
|
||||
)
|
||||
observation_text_max_chars: int = Field(
|
||||
default=12_000,
|
||||
ge=500,
|
||||
description="Maximum page text characters returned alongside a browser_act observation.",
|
||||
)
|
||||
+110
-110
@@ -3,7 +3,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
@@ -53,115 +53,115 @@ class McpServerConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayTableColumn(BaseModel):
|
||||
"""A configured table column for structured skill result display."""
|
||||
|
||||
label: str = Field(description="Column label shown in the frontend table")
|
||||
field: str = Field(description="Field name or dot path read from each result item")
|
||||
width: str | None = Field(default=None, description="Optional CSS width for this column")
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayResultSetConfig(BaseModel):
|
||||
"""Display adapter for one result array inside a skill response."""
|
||||
|
||||
key: str = Field(description="Stable key for this result set")
|
||||
label: str | None = Field(default=None, description="Human-readable result set label")
|
||||
enabled: bool = Field(default=True, description="Whether this result set is enabled")
|
||||
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
|
||||
default="citation",
|
||||
description="How the frontend should render this result set",
|
||||
)
|
||||
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
|
||||
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
|
||||
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
|
||||
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
|
||||
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
|
||||
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
|
||||
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
|
||||
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
|
||||
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
|
||||
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
|
||||
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
|
||||
link_strategy: Literal["auto", "url", "id"] = Field(
|
||||
default="auto",
|
||||
description="Whether links use a URL field, an ID template, or automatic fallback",
|
||||
alias="linkStrategy",
|
||||
)
|
||||
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
|
||||
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
|
||||
default="singleBracket",
|
||||
description="Citation token style expected in model output",
|
||||
alias="citationToken",
|
||||
)
|
||||
max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
|
||||
custom_instruction: str = Field(
|
||||
default="",
|
||||
description="User-authored instruction for custom rendering schema generation",
|
||||
alias="customInstruction",
|
||||
)
|
||||
table_columns: list[SkillDisplayTableColumn] = Field(
|
||||
default_factory=list,
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayConfig(BaseModel):
|
||||
"""Frontend display adapter for structured skill results."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
|
||||
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
|
||||
default="none",
|
||||
description="How the frontend should render this skill's structured results",
|
||||
)
|
||||
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
|
||||
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
|
||||
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
|
||||
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
|
||||
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
|
||||
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
|
||||
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
|
||||
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
|
||||
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
|
||||
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
|
||||
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
|
||||
link_strategy: Literal["auto", "url", "id"] = Field(
|
||||
default="auto",
|
||||
description="Whether links use a URL field, an ID template, or automatic fallback",
|
||||
alias="linkStrategy",
|
||||
)
|
||||
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
|
||||
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
|
||||
default="doubleBracket",
|
||||
description="Citation token style expected in model output",
|
||||
alias="citationToken",
|
||||
)
|
||||
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
|
||||
custom_instruction: str = Field(
|
||||
default="",
|
||||
description="User-authored instruction for custom rendering schema generation",
|
||||
alias="customInstruction",
|
||||
)
|
||||
table_columns: list[SkillDisplayTableColumn] = Field(
|
||||
default_factory=list,
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
result_sets: list[SkillDisplayResultSetConfig] = Field(
|
||||
default_factory=list,
|
||||
description="Optional per-result-array display adapters",
|
||||
alias="resultSets",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillStateConfig(BaseModel):
|
||||
"""Configuration for a single skill's state."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether this skill is enabled")
|
||||
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
|
||||
class SkillDisplayTableColumn(BaseModel):
|
||||
"""A configured table column for structured skill result display."""
|
||||
|
||||
label: str = Field(description="Column label shown in the frontend table")
|
||||
field: str = Field(description="Field name or dot path read from each result item")
|
||||
width: str | None = Field(default=None, description="Optional CSS width for this column")
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayResultSetConfig(BaseModel):
|
||||
"""Display adapter for one result array inside a skill response."""
|
||||
|
||||
key: str = Field(description="Stable key for this result set")
|
||||
label: str | None = Field(default=None, description="Human-readable result set label")
|
||||
enabled: bool = Field(default=True, description="Whether this result set is enabled")
|
||||
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
|
||||
default="citation",
|
||||
description="How the frontend should render this result set",
|
||||
)
|
||||
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
|
||||
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
|
||||
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
|
||||
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
|
||||
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
|
||||
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
|
||||
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
|
||||
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
|
||||
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
|
||||
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
|
||||
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
|
||||
link_strategy: Literal["auto", "url", "id"] = Field(
|
||||
default="auto",
|
||||
description="Whether links use a URL field, an ID template, or automatic fallback",
|
||||
alias="linkStrategy",
|
||||
)
|
||||
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
|
||||
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
|
||||
default="singleBracket",
|
||||
description="Citation token style expected in model output",
|
||||
alias="citationToken",
|
||||
)
|
||||
max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
|
||||
custom_instruction: str = Field(
|
||||
default="",
|
||||
description="User-authored instruction for custom rendering schema generation",
|
||||
alias="customInstruction",
|
||||
)
|
||||
table_columns: list[SkillDisplayTableColumn] = Field(
|
||||
default_factory=list,
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillDisplayConfig(BaseModel):
|
||||
"""Frontend display adapter for structured skill results."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
|
||||
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
|
||||
default="none",
|
||||
description="How the frontend should render this skill's structured results",
|
||||
)
|
||||
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
|
||||
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
|
||||
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
|
||||
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
|
||||
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
|
||||
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
|
||||
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
|
||||
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
|
||||
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
|
||||
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
|
||||
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
|
||||
link_strategy: Literal["auto", "url", "id"] = Field(
|
||||
default="auto",
|
||||
description="Whether links use a URL field, an ID template, or automatic fallback",
|
||||
alias="linkStrategy",
|
||||
)
|
||||
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
|
||||
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
|
||||
default="doubleBracket",
|
||||
description="Citation token style expected in model output",
|
||||
alias="citationToken",
|
||||
)
|
||||
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
|
||||
custom_instruction: str = Field(
|
||||
default="",
|
||||
description="User-authored instruction for custom rendering schema generation",
|
||||
alias="customInstruction",
|
||||
)
|
||||
table_columns: list[SkillDisplayTableColumn] = Field(
|
||||
default_factory=list,
|
||||
description="Columns used when mode is table",
|
||||
alias="tableColumns",
|
||||
)
|
||||
result_sets: list[SkillDisplayResultSetConfig] = Field(
|
||||
default_factory=list,
|
||||
description="Optional per-result-array display adapters",
|
||||
alias="resultSets",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillStateConfig(BaseModel):
|
||||
"""Configuration for a single skill's state."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Whether this skill is enabled")
|
||||
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
|
||||
|
||||
|
||||
class ExtensionsConfig(BaseModel):
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Configuration for the product knowledge base (obsidian-wiki compatible)."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class KnowledgeExportConfig(BaseModel):
|
||||
"""Knowledge graph export switches (``wiki-export`` compatible output)."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Master switch for graph export.")
|
||||
graph_html_enabled: bool = Field(default=True, description="Generate wiki-export/graph.html.")
|
||||
graph_json_enabled: bool = Field(default=True, description="Generate wiki-export/graph.json.")
|
||||
|
||||
|
||||
class KnowledgeQueryConfig(BaseModel):
|
||||
"""Layered query strategy flags (``wiki-query`` style)."""
|
||||
|
||||
use_hot_md: bool = Field(default=True, description="Consult hot.md when searching.")
|
||||
use_index_md: bool = Field(default=True, description="Consult index.md when searching.")
|
||||
use_frontmatter_scan: bool = Field(default=True, description="Scan frontmatter when searching.")
|
||||
|
||||
|
||||
class KnowledgeEmbeddingConfig(BaseModel):
|
||||
"""Vector embedding config (phase 2).
|
||||
|
||||
Vector RAG is opt-in: set ``enabled: true`` and point ``base_url`` at an
|
||||
OpenAI-compatible ``/embeddings`` endpoint. When disabled or unreachable,
|
||||
search transparently falls back to keyword scoring, so the feature degrades
|
||||
gracefully on offline deployments without an embedding model.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(default=False, description="Enable vector embeddings.")
|
||||
provider: str = Field(default="openai", description="Embedding provider (openai-compatible).")
|
||||
model: str = Field(default="", description="Embedding model name, e.g. text-embedding-3-small / bge-m3.")
|
||||
base_url: str = Field(default="", description="OpenAI-compatible base URL (…/v1). Empty disables vector mode.")
|
||||
api_key: str = Field(default="", description="API key for the embedding endpoint ($ENV resolved like other config).")
|
||||
dimensions: int | None = Field(default=None, description="Optional embedding dimensions hint sent to the endpoint.")
|
||||
batch_size: int = Field(default=32, description="Texts per embedding request.")
|
||||
timeout_seconds: int = Field(default=30, description="HTTP timeout per embedding request.")
|
||||
|
||||
|
||||
class KnowledgeConfig(BaseModel):
|
||||
"""Product knowledge base configuration.
|
||||
|
||||
The first release ships a global, shared, obsidian-wiki compatible vault.
|
||||
Knowledge metadata is mirrored into the database for fast listing/search,
|
||||
while ``content_md`` is written to a Markdown vault that stays compatible
|
||||
with ``ar9av/obsidian-wiki`` (index.md / log.md / hot.md / .manifest.json,
|
||||
frontmatter, wikilinks, wiki-export graph artifacts).
|
||||
"""
|
||||
|
||||
enabled: bool = Field(default=True, description="Enable the knowledge base feature.")
|
||||
provider: str = Field(default="obsidian_wiki", description="Knowledge organization provider.")
|
||||
vault_dir: str = Field(
|
||||
default="data/knowledge/obsidian-vault",
|
||||
description="Vault directory (relative to DEER_FLOW_HOME, or absolute).",
|
||||
)
|
||||
project_name: str = Field(default="zncm", description="Project namespace for projects/<name>/ pages.")
|
||||
auto_ingest_enabled: bool = Field(default=False, description="Auto-sediment conversations in the background (phase 3).")
|
||||
auto_ingest_min_chars: int = Field(default=300, description="Minimum characters before auto-ingest considers a turn.")
|
||||
llm_extract_enabled: bool = Field(default=True, description="Distill notes with an LLM (wiki-capture style) instead of the rule-based fallback.")
|
||||
extract_model_name: str | None = Field(default=None, description="Model (from models[]) used to distill knowledge. Falls back to models[0] when unset.")
|
||||
extract_max_input_chars: int = Field(default=12000, description="Max characters of conversation transcript fed to the extraction LLM.")
|
||||
max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.")
|
||||
max_source_items: int = Field(default=20, description="Maximum source items recorded per note.")
|
||||
related_notes_enabled: bool = Field(default=True, description="On capture, find existing notes on the same topic and link them under ## Related (real cross-references instead of placeholders).")
|
||||
related_notes_limit: int = Field(default=3, description="Maximum real related notes linked into each captured note.")
|
||||
multi_note_enabled: bool = Field(default=True, description="Allow the LLM extractor to split one conversation into multiple cross-linked knowledge notes.")
|
||||
rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).")
|
||||
rag_limit: int = Field(default=5, description="Maximum knowledge snippets injected per query.")
|
||||
sensitive_redaction_enabled: bool = Field(default=True, description="Redact secrets/PII (tokens, keys, phone, id-card, email) before persisting notes (phase 3).")
|
||||
dedup_similarity_threshold: float = Field(default=0.82, description="Title/summary similarity above which two notes are flagged as duplicates (phase 3).")
|
||||
entity_extraction_enabled: bool = Field(default=True, description="Extract entities/relations during LLM distillation and feed them into the graph (phase 4).")
|
||||
export: KnowledgeExportConfig = Field(default_factory=KnowledgeExportConfig, description="Graph export configuration.")
|
||||
query: KnowledgeQueryConfig = Field(default_factory=KnowledgeQueryConfig, description="Layered query configuration.")
|
||||
embedding: KnowledgeEmbeddingConfig = Field(default_factory=KnowledgeEmbeddingConfig, description="Embedding configuration.")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Product knowledge base module (obsidian-wiki compatible).
|
||||
|
||||
Self-contained: knowledge ingestion, storage (DB mirror + Markdown vault),
|
||||
search and graph export all live here. The Gateway only depends on
|
||||
:class:`KnowledgeService` via :func:`make_knowledge_service`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
from deerflow.knowledge.service import EmptyThreadError, KnowledgeService
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
|
||||
__all__ = [
|
||||
"EmptyThreadError",
|
||||
"KnowledgeRepository",
|
||||
"KnowledgeService",
|
||||
"ObsidianWikiAdapter",
|
||||
"make_knowledge_service",
|
||||
]
|
||||
|
||||
|
||||
def make_knowledge_service(
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
config: KnowledgeConfig,
|
||||
) -> KnowledgeService | None:
|
||||
"""Build the knowledge service.
|
||||
|
||||
Returns ``None`` when persistence is memory-only (no session factory) or
|
||||
the feature is disabled — the router then responds 503.
|
||||
"""
|
||||
if session_factory is None or not config.enabled:
|
||||
return None
|
||||
return KnowledgeService(KnowledgeRepository(session_factory), config)
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Background auto-sediment queue (phase 3).
|
||||
|
||||
A single asyncio worker drains a bounded queue of finished conversations and
|
||||
sediments the worthwhile ones into the knowledge base — entirely off the chat
|
||||
request path, so it never adds latency to a reply. Enqueue happens from
|
||||
``KnowledgeRagMiddleware.after_agent`` when ``knowledge.auto_ingest_enabled``.
|
||||
|
||||
Value judgment is intentionally cheap (min assistant chars + dedup via
|
||||
``skip_existing``); the LLM still distills the kept conversations via the normal
|
||||
``capture_thread`` path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_QUEUE: AutoIngestQueue | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _QueueUser:
|
||||
"""Minimal CurrentUser stand-in (only ``.id`` is needed for context)."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AutoIngestItem:
|
||||
thread_id: str
|
||||
user_id: str
|
||||
messages: list[dict[str, Any]]
|
||||
thread_title: str | None = None
|
||||
reference_batches: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
class AutoIngestQueue:
|
||||
"""Bounded background queue + single worker for auto-sedimentation."""
|
||||
|
||||
def __init__(self, service_provider: Callable[[], Any], *, min_chars: int = 300, maxsize: int = 256) -> None:
|
||||
self._provider = service_provider
|
||||
self._min_chars = min_chars
|
||||
self._queue: asyncio.Queue[AutoIngestItem] = asyncio.Queue(maxsize=maxsize)
|
||||
self._task: asyncio.Task | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._worker(), name="knowledge-auto-ingest")
|
||||
logger.info("Knowledge auto-ingest worker started")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await self._task
|
||||
self._task = None
|
||||
|
||||
def enqueue(self, item: AutoIngestItem) -> None:
|
||||
try:
|
||||
self._queue.put_nowait(item)
|
||||
except asyncio.QueueFull:
|
||||
logger.warning("Knowledge auto-ingest queue full; dropping thread %s", item.thread_id)
|
||||
|
||||
def _worth_ingesting(self, item: AutoIngestItem) -> bool:
|
||||
chars = 0
|
||||
for m in item.messages:
|
||||
if str(m.get("type") or m.get("role") or "").lower() == "ai":
|
||||
content = m.get("content")
|
||||
chars += len(content) if isinstance(content, str) else len(str(content or ""))
|
||||
return chars >= self._min_chars
|
||||
|
||||
async def _process(self, item: AutoIngestItem) -> None:
|
||||
service = self._provider()
|
||||
if service is None:
|
||||
return
|
||||
if not self._worth_ingesting(item):
|
||||
logger.debug("Auto-ingest skipped (below min chars) thread=%s", item.thread_id)
|
||||
return
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
token = set_current_user(_QueueUser(id=item.user_id))
|
||||
try:
|
||||
result = await service.capture_thread(
|
||||
thread_id=item.thread_id,
|
||||
messages=item.messages,
|
||||
thread_title=item.thread_title,
|
||||
status="draft", # auto-sedimented notes land as drafts for review
|
||||
include_sources=True,
|
||||
reference_batches=item.reference_batches,
|
||||
created_by=item.user_id,
|
||||
skip_existing=True,
|
||||
)
|
||||
if result.get("created"):
|
||||
logger.info("Auto-ingested thread %s -> note %s", item.thread_id, result["note"]["id"])
|
||||
except Exception:
|
||||
logger.exception("Auto-ingest failed for thread %s", item.thread_id)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
async def _worker(self) -> None:
|
||||
while True:
|
||||
item = await self._queue.get()
|
||||
try:
|
||||
await self._process(item)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Auto-ingest worker error")
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
|
||||
def set_auto_ingest_queue(queue: AutoIngestQueue | None) -> None:
|
||||
global _QUEUE
|
||||
_QUEUE = queue
|
||||
|
||||
|
||||
def get_auto_ingest_queue() -> AutoIngestQueue | None:
|
||||
return _QUEUE
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Markdown chunking for embedding/index (phase 2).
|
||||
|
||||
Splits a note body into overlapping chunks along Markdown structure (headings →
|
||||
paragraphs) so each embedded chunk is semantically coherent and bounded in
|
||||
size. Frontmatter is assumed already stripped by the caller.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
||||
|
||||
|
||||
def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
|
||||
"""Return a list of text chunks for ``content``.
|
||||
|
||||
Sections are cut at Markdown headings; long sections are further split into
|
||||
~``max_chars`` windows with ``overlap`` characters of carry-over.
|
||||
"""
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Split into heading-delimited sections, keeping the heading with its body.
|
||||
sections: list[str] = []
|
||||
last = 0
|
||||
for m in _HEADING_RE.finditer(text):
|
||||
if m.start() > last:
|
||||
sections.append(text[last : m.start()].strip())
|
||||
last = m.start()
|
||||
sections.append(text[last:].strip())
|
||||
sections = [s for s in sections if s]
|
||||
|
||||
chunks: list[str] = []
|
||||
for section in sections:
|
||||
if len(section) <= max_chars:
|
||||
chunks.append(section)
|
||||
continue
|
||||
# Pack paragraphs into windows, then hard-split anything still too long.
|
||||
buf = ""
|
||||
for para in _split_paragraphs(section):
|
||||
if len(buf) + len(para) + 2 <= max_chars:
|
||||
buf = f"{buf}\n\n{para}".strip()
|
||||
else:
|
||||
if buf:
|
||||
chunks.append(buf)
|
||||
if len(para) <= max_chars:
|
||||
buf = para
|
||||
else:
|
||||
for i in range(0, len(para), max_chars - overlap):
|
||||
chunks.append(para[i : i + max_chars])
|
||||
buf = ""
|
||||
if buf:
|
||||
chunks.append(buf)
|
||||
return [c for c in chunks if c.strip()]
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Duplicate detection for knowledge notes (phase 3).
|
||||
|
||||
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
|
||||
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
|
||||
works on offline deployments. Returns *suggestions*; merging is an explicit,
|
||||
user-driven action.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.manifest import content_hash
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
||||
|
||||
|
||||
def _tokens(*texts: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for t in texts:
|
||||
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
|
||||
return out
|
||||
|
||||
|
||||
def _jaccard(a: set[str], b: set[str]) -> float:
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
inter = len(a & b)
|
||||
union = len(a | b)
|
||||
return inter / union if union else 0.0
|
||||
|
||||
|
||||
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
|
||||
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
|
||||
|
||||
``primary`` is the most recently updated note in each cluster.
|
||||
"""
|
||||
notes = notes[:max_notes]
|
||||
feats: list[tuple[dict[str, Any], set[str], str]] = []
|
||||
for n in notes:
|
||||
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
|
||||
feats.append((n, toks, content_hash(n.get("content_md") or "")))
|
||||
|
||||
used: set[str] = set()
|
||||
groups: list[dict[str, Any]] = []
|
||||
for i in range(len(feats)):
|
||||
ni, ti, hi = feats[i]
|
||||
if ni["id"] in used:
|
||||
continue
|
||||
dups: list[dict[str, Any]] = []
|
||||
for j in range(i + 1, len(feats)):
|
||||
nj, tj, hj = feats[j]
|
||||
if nj["id"] in used:
|
||||
continue
|
||||
if hi and hi == hj:
|
||||
score, reason = 1.0, "identical-content"
|
||||
else:
|
||||
score = _jaccard(ti, tj)
|
||||
reason = "similar-title-summary"
|
||||
if score >= threshold:
|
||||
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
|
||||
used.add(nj["id"])
|
||||
if dups:
|
||||
used.add(ni["id"])
|
||||
# Primary = newest by updated_at.
|
||||
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
|
||||
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
|
||||
others = [
|
||||
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
|
||||
for c in cluster
|
||||
if c["id"] != primary["id"]
|
||||
]
|
||||
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
|
||||
return groups
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Embedding generation for vector RAG (phase 2).
|
||||
|
||||
Talks to an OpenAI-compatible ``/embeddings`` endpoint configured under
|
||||
``knowledge.embedding``. Everything here is **best-effort**: if embeddings are
|
||||
disabled, unconfigured, or the endpoint errors, callers get an empty result and
|
||||
the search layer falls back to keyword scoring — so vector RAG never breaks a
|
||||
deployment that has no embedding model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.knowledge_config import KnowledgeEmbeddingConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cosine_similarity(a: list[float], b: list[float]) -> float:
|
||||
"""Cosine similarity of two equal-length vectors (0 when degenerate)."""
|
||||
if not a or not b or len(a) != len(b):
|
||||
return 0.0
|
||||
dot = sum(x * y for x, y in zip(a, b, strict=False))
|
||||
na = math.sqrt(sum(x * x for x in a))
|
||||
nb = math.sqrt(sum(y * y for y in b))
|
||||
if na == 0 or nb == 0:
|
||||
return 0.0
|
||||
return dot / (na * nb)
|
||||
|
||||
|
||||
class EmbeddingClient:
|
||||
"""Thin async client over an OpenAI-compatible embeddings endpoint."""
|
||||
|
||||
def __init__(self, config: KnowledgeEmbeddingConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return bool(self.config.enabled and self.config.base_url and self.config.model)
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Return one vector per input text; ``[]`` on any failure/disabled."""
|
||||
if not self.is_enabled() or not texts:
|
||||
return []
|
||||
try:
|
||||
import httpx
|
||||
except Exception: # pragma: no cover - httpx is a core dep
|
||||
logger.warning("httpx unavailable; knowledge embeddings disabled")
|
||||
return []
|
||||
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.config.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
||||
|
||||
vectors: list[list[float]] = []
|
||||
batch = max(1, self.config.batch_size)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
|
||||
for start in range(0, len(texts), batch):
|
||||
chunk = texts[start : start + batch]
|
||||
payload: dict = {"model": self.config.model, "input": chunk}
|
||||
if self.config.dimensions:
|
||||
payload["dimensions"] = self.config.dimensions
|
||||
resp = await client.post(url, headers=headers, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
items = sorted(data.get("data", []), key=lambda d: d.get("index", 0))
|
||||
vectors.extend([item.get("embedding", []) for item in items])
|
||||
except Exception:
|
||||
logger.exception("Knowledge embedding request failed; falling back to keyword search")
|
||||
return []
|
||||
return vectors
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
vecs = await self.embed_texts([text])
|
||||
return vecs[0] if vecs else []
|
||||
@@ -0,0 +1,227 @@
|
||||
"""obsidian-wiki ``wiki-export`` compatible graph generation.
|
||||
|
||||
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
|
||||
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
|
||||
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
|
||||
``graph.html`` (no external CDN, safe to iframe offline).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.markdown_vault import extract_wikilinks
|
||||
|
||||
|
||||
def _vault_key(vault_path: str | None) -> str | None:
|
||||
if not vault_path:
|
||||
return None
|
||||
key = vault_path.replace("\\", "/")
|
||||
if key.endswith(".md"):
|
||||
key = key[:-3]
|
||||
return key
|
||||
|
||||
|
||||
def build_graph(
|
||||
notes: list[dict[str, Any]],
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
|
||||
|
||||
Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
|
||||
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
|
||||
note→entity / entity→entity edges are added on top of the wikilink and
|
||||
shared-tag edges.
|
||||
"""
|
||||
nodes = []
|
||||
by_key: dict[str, str] = {}
|
||||
note_ids: set[str] = set()
|
||||
for n in notes:
|
||||
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
|
||||
note_ids.add(n["id"])
|
||||
key = _vault_key(n.get("vault_path"))
|
||||
if key:
|
||||
by_key[key] = n["id"]
|
||||
|
||||
# Only surface entities that are still mentioned by a LIVE note. Entities whose
|
||||
# only referencing notes were deleted/archived would otherwise linger forever
|
||||
# as orphan nodes ("graph doesn't follow my deletions").
|
||||
mentioned: set[str] = set()
|
||||
for rel in relations or []:
|
||||
if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
|
||||
mentioned.add(rel["to_entity_id"])
|
||||
|
||||
entity_ids: set[str] = set()
|
||||
for ent in entities or []:
|
||||
if ent["id"] not in mentioned:
|
||||
continue
|
||||
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
|
||||
entity_ids.add(ent["id"])
|
||||
|
||||
edges: list[dict[str, Any]] = []
|
||||
edge_seen: set[tuple[str, str, str]] = set()
|
||||
|
||||
def add_edge(a: str, b: str, label: str, weight: float) -> None:
|
||||
if a == b:
|
||||
return
|
||||
key = (min(a, b), max(a, b), label)
|
||||
if key in edge_seen:
|
||||
return
|
||||
edge_seen.add(key)
|
||||
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
|
||||
|
||||
# Wikilink edges.
|
||||
for n in notes:
|
||||
for target in extract_wikilinks(n.get("content_md") or ""):
|
||||
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
|
||||
if tid:
|
||||
add_edge(n["id"], tid, "links", 2)
|
||||
|
||||
# Shared-tag edges.
|
||||
tag_to_notes: dict[str, list[str]] = {}
|
||||
for n in notes:
|
||||
for tag in n.get("tags") or []:
|
||||
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
|
||||
for tag, ids in tag_to_notes.items():
|
||||
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
|
||||
continue
|
||||
for i in range(len(ids)):
|
||||
for j in range(i + 1, len(ids)):
|
||||
add_edge(ids[i], ids[j], "related", 1)
|
||||
|
||||
# Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
|
||||
valid = note_ids | entity_ids
|
||||
for rel in relations or []:
|
||||
src = rel.get("from_note_id") or rel.get("from_entity_id")
|
||||
tgt = rel.get("to_note_id") or rel.get("to_entity_id")
|
||||
if src in valid and tgt in valid:
|
||||
add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
def to_graphml(graph: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
|
||||
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
|
||||
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
|
||||
' <graph edgedefault="undirected">',
|
||||
]
|
||||
for n in graph["nodes"]:
|
||||
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
|
||||
for e in graph["edges"]:
|
||||
lines.append(
|
||||
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
|
||||
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
|
||||
)
|
||||
lines += [" </graph>", "</graphml>"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_cypher(graph: dict[str, Any]) -> str:
|
||||
lines = []
|
||||
for n in graph["nodes"]:
|
||||
label = str(n["label"]).replace("'", "\\'")
|
||||
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
|
||||
for e in graph["edges"]:
|
||||
rel = str(e["label"]).upper()
|
||||
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_html(graph: dict[str, Any]) -> str:
|
||||
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
|
||||
data_json = json.dumps(graph, ensure_ascii=False)
|
||||
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
|
||||
|
||||
|
||||
def export_all(
|
||||
vault_root: Path | str,
|
||||
notes: list[dict[str, Any]],
|
||||
*,
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
write_html: bool = True,
|
||||
write_json: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
|
||||
out_dir = Path(vault_root).resolve() / "wiki-export"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
graph = build_graph(notes, entities, relations)
|
||||
written: list[str] = []
|
||||
if write_json:
|
||||
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
written.append("graph.json")
|
||||
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
|
||||
written.append("graph.graphml")
|
||||
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
|
||||
written.append("cypher.txt")
|
||||
if write_html:
|
||||
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
|
||||
written.append("graph.html")
|
||||
return {
|
||||
"files": written,
|
||||
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
|
||||
"graph": graph,
|
||||
}
|
||||
|
||||
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Knowledge Graph</title>
|
||||
<style>
|
||||
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
|
||||
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
|
||||
canvas{display:block}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="hud"></div>
|
||||
<div id="tip"></div>
|
||||
<canvas id="c"></canvas>
|
||||
<script>
|
||||
const GRAPH = __GRAPH_DATA__;
|
||||
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
|
||||
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
|
||||
let W=0,H=0;
|
||||
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
|
||||
addEventListener('resize',resize);resize();
|
||||
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
|
||||
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
|
||||
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
|
||||
let cam={x:0,y:0,k:1},drag=null,pan=null;
|
||||
function step(){
|
||||
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
|
||||
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
|
||||
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
|
||||
}
|
||||
function draw(){
|
||||
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
|
||||
ctx.strokeStyle='rgba(120,150,255,.25)';
|
||||
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
|
||||
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
|
||||
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
|
||||
}
|
||||
function loop(){step();draw();requestAnimationFrame(loop);}loop();
|
||||
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
|
||||
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
|
||||
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
|
||||
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
|
||||
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
|
||||
addEventListener('mouseup',()=>{drag=null;pan=null;});
|
||||
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Extract knowledge notes from threads / searches / tool results.
|
||||
|
||||
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
|
||||
keep high-value content (user questions, final assistant answers, cited
|
||||
sources), drop noise (empty/greeting/intermediate reasoning), and write a
|
||||
*declarative* note rather than a chat transcript. An LLM extractor can later
|
||||
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
|
||||
|
||||
# Greeting / low-value lines filtered from extracted turns.
|
||||
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _one_line(text: str, limit: int = 300) -> str:
|
||||
"""Collapse whitespace and clip ``text`` to a single line."""
|
||||
collapsed = _WS_RE.sub(" ", text or "").strip()
|
||||
return collapsed[:limit]
|
||||
|
||||
|
||||
def message_text(msg: dict[str, Any]) -> str:
|
||||
"""Return the plain-text content of a serialized LangChain message."""
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
|
||||
parts.append(block["text"])
|
||||
return "\n".join(p for p in parts if p).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _msg_type(msg: dict[str, Any]) -> str:
|
||||
return str(msg.get("type") or msg.get("role") or "").lower()
|
||||
|
||||
|
||||
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
|
||||
"""Split messages into ``{user, ai}`` turns keyed by human messages.
|
||||
|
||||
For each turn we keep the last substantial AI message (with text body).
|
||||
"""
|
||||
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
|
||||
turns: list[dict[str, str]] = []
|
||||
current: dict[str, str] | None = None
|
||||
for msg in msgs:
|
||||
mtype = _msg_type(msg)
|
||||
text = message_text(msg)
|
||||
if mtype == "human":
|
||||
if current and (current.get("user") or current.get("ai")):
|
||||
turns.append(current)
|
||||
current = {"user": text, "ai": ""}
|
||||
elif mtype == "ai" and text:
|
||||
if current is None:
|
||||
current = {"user": "", "ai": text}
|
||||
else:
|
||||
current["ai"] = text # last substantial AI message wins
|
||||
if current and (current.get("user") or current.get("ai")):
|
||||
turns.append(current)
|
||||
# Drop turns that are pure greetings with no answer.
|
||||
cleaned = []
|
||||
for t in turns:
|
||||
user = (t.get("user") or "").strip()
|
||||
ai = (t.get("ai") or "").strip()
|
||||
if not ai and _GREETING_RE.match(user):
|
||||
continue
|
||||
if not user and not ai:
|
||||
continue
|
||||
cleaned.append({"user": user, "ai": ai})
|
||||
return cleaned
|
||||
|
||||
|
||||
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
|
||||
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
|
||||
return thread_title.strip()[:200]
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
first = t["user"].splitlines()[0].strip()
|
||||
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
|
||||
return "知识笔记"
|
||||
|
||||
|
||||
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
|
||||
"""Simple rule-based summary: the first answer, clipped."""
|
||||
for t in turns:
|
||||
if t.get("ai"):
|
||||
text = re.sub(r"\s+", " ", t["ai"]).strip()
|
||||
return text[:limit] + ("…" if len(text) > limit else "")
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
text = re.sub(r"\s+", " ", t["user"]).strip()
|
||||
return text[:limit] + ("…" if len(text) > limit else "")
|
||||
return ""
|
||||
|
||||
|
||||
def related_links(project_name: str | None) -> list[str]: # noqa: ARG001
|
||||
"""Structural wikilinks every note used to carry.
|
||||
|
||||
Product decision: notes no longer auto-link the vault meta/hub pages
|
||||
(``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every
|
||||
note's ``## Related``. Only real, content-specific related notes (injected by
|
||||
the service layer, which has DB access) are linked now. Kept as a function
|
||||
(returns empty) so existing callers and the wiki-capture contract stay intact.
|
||||
"""
|
||||
return []
|
||||
|
||||
|
||||
# Backwards-compatible internal alias.
|
||||
_related_links = related_links
|
||||
|
||||
|
||||
def _related_section_md(related_lines: list[str], source_lines: list[str]) -> str:
|
||||
"""Render the ``## Related`` section, or ``""`` when there is nothing to show."""
|
||||
if not related_lines and not source_lines:
|
||||
return ""
|
||||
inner = "\n".join(related_lines)
|
||||
if source_lines:
|
||||
sources_block = "### Sources\n\n" + "\n".join(source_lines)
|
||||
inner = (inner + "\n\n" + sources_block) if inner else sources_block
|
||||
return f"## Related\n\n{inner}\n"
|
||||
|
||||
|
||||
def _build_body(
|
||||
*,
|
||||
title: str,
|
||||
turns: list[dict[str, str]],
|
||||
mode: str,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
) -> str:
|
||||
"""Render the wiki-capture 5-section declarative body."""
|
||||
full = mode == "full"
|
||||
answer_clip = 4000 if full else 800
|
||||
|
||||
context_lines = []
|
||||
for t in turns:
|
||||
if t.get("user"):
|
||||
q = re.sub(r"\s+", " ", t["user"]).strip()
|
||||
context_lines.append(f"- {q[:300]}")
|
||||
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
|
||||
|
||||
finding_lines = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
if t.get("ai"):
|
||||
ans = t["ai"].strip()
|
||||
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
|
||||
finding_lines.append(f"### 结论 {i}\n\n{ans}")
|
||||
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
|
||||
|
||||
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
|
||||
|
||||
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
|
||||
|
||||
related_lines = [f"- {link}" for link in _related_links(project_name)]
|
||||
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
|
||||
related_section = _related_section_md(related_lines, source_lines)
|
||||
|
||||
return (
|
||||
f"# {title}\n\n"
|
||||
f"## Context\n\n{context}\n\n"
|
||||
f"## Finding / Decision\n\n{finding}\n\n"
|
||||
f"## Reasoning\n\n{reasoning}\n\n"
|
||||
f"## Implications\n\n{implications}\n\n"
|
||||
f"{related_section}"
|
||||
)
|
||||
|
||||
|
||||
def extract_thread(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
) -> NoteDraft | None:
|
||||
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
|
||||
turns = split_turns(messages, max_messages=max_messages)
|
||||
if not turns:
|
||||
return None
|
||||
final_title = (title or "").strip() or _auto_title(thread_title, turns)
|
||||
summary = _summary(turns)
|
||||
sources = sources or []
|
||||
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
|
||||
base_tags = list(tags or [])
|
||||
for default in (project_name or "zncm", "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
category="synthesis",
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
tags=base_tags,
|
||||
confidence=0.6,
|
||||
related=_related_links(project_name),
|
||||
sources=sources,
|
||||
source_key=f"thread:{thread_id}",
|
||||
)
|
||||
|
||||
|
||||
def extract_search(
|
||||
query: str,
|
||||
sources: list[SourceDraft],
|
||||
*,
|
||||
thread_id: str | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
) -> NoteDraft:
|
||||
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
|
||||
final_title = (title or "").strip() or f"检索:{query[:40]}"
|
||||
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
|
||||
for s in sources:
|
||||
label = s.title or s.url or "来源"
|
||||
line = f"- **{label}**"
|
||||
if s.url:
|
||||
line += f" — {s.url}"
|
||||
if s.snippet:
|
||||
line += "\n - " + _one_line(s.snippet)
|
||||
body_lines.append(line)
|
||||
body_lines += [
|
||||
"\n## Reasoning\n",
|
||||
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
|
||||
"## Implications\n",
|
||||
"- 引用前请核对原始来源。\n",
|
||||
"## Related\n",
|
||||
*[f"- {link}" for link in _related_links(project_name)],
|
||||
]
|
||||
base_tags = list(tags or [])
|
||||
for default in (project_name or "zncm", "knowledge-base", "reference"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
|
||||
content_md="\n".join(body_lines),
|
||||
category="references",
|
||||
source_type="search",
|
||||
source_id=f"search:{query[:120]}",
|
||||
tags=base_tags,
|
||||
confidence=0.4,
|
||||
related=_related_links(project_name),
|
||||
sources=sources,
|
||||
source_key=None,
|
||||
)
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
"""LLM-based knowledge extraction (wiki-capture style).
|
||||
|
||||
Turns a conversation into a *declarative* knowledge note instead of dumping the
|
||||
raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note,
|
||||
keep only high-value findings/decisions, drop greetings / intermediate
|
||||
reasoning / verbatim transcripts, mark inferred content, and emit structured
|
||||
JSON that we render into the Context / Finding / Reasoning / Implications /
|
||||
Related body.
|
||||
|
||||
Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread`
|
||||
when the LLM is unavailable or returns something unusable, so capture never
|
||||
hard-fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge import extractor
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
|
||||
from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"}
|
||||
|
||||
|
||||
class LLMRejectedIngest(Exception):
|
||||
"""The LLM judged the conversation as having no knowledge value.
|
||||
|
||||
Raised (instead of returning ``[]``) when the model explicitly sets
|
||||
``should_ingest: false`` with no notes — so callers can *skip* ingestion
|
||||
entirely rather than falling back to the rule-based verbatim dump.
|
||||
"""
|
||||
|
||||
|
||||
def _is_falsey(value: Any) -> bool:
|
||||
if value is False:
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in ("false", "0", "no")
|
||||
return False
|
||||
|
||||
_SYSTEM_PROMPT = """\
|
||||
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\
|
||||
**陈述式知识(declarative knowledge)**,而不是聊天流水账。
|
||||
|
||||
要求:
|
||||
- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。
|
||||
- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。
|
||||
- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。
|
||||
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
|
||||
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
|
||||
references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
|
||||
- 用中文输出。
|
||||
|
||||
一次对话可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
|
||||
- 默认只产出 **1 页**;只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
|
||||
- 每一页聚焦单一主题,标题与 summary 必须彼此区分,不要把同一结论拆到多页。
|
||||
- 不值得沉淀时返回空的 notes 数组。
|
||||
|
||||
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
|
||||
{
|
||||
"should_ingest": true,
|
||||
"reason": "为什么值得/不值得沉淀",
|
||||
"notes": [
|
||||
{
|
||||
"title": "简洁的知识标题(不超过 40 字)",
|
||||
"summary": "1-2 句话概括这页知识",
|
||||
"category": "synthesis",
|
||||
"context": "问题背景或触发场景,1-3 句",
|
||||
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
|
||||
"reasoning": "为什么成立、取舍、不确定性",
|
||||
"implications": ["后续建议/风险/下一步", "..."],
|
||||
"tags": ["标签1", "标签2"],
|
||||
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
|
||||
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
if t.get("user"):
|
||||
parts.append(f"[用户 {i}] {t['user']}")
|
||||
if t.get("ai"):
|
||||
parts.append(f"[助手 {i}] {t['ai']}")
|
||||
if sources:
|
||||
parts.append("\n[来源]")
|
||||
for s in sources[:20]:
|
||||
label = s.title or s.url or s.tool_name or "来源"
|
||||
parts.append(f"- {label}" + (f" ({s.url})" if s.url else ""))
|
||||
text = "\n\n".join(parts)
|
||||
if len(text) > max_chars:
|
||||
text = text[:max_chars] + "\n…(已截断)"
|
||||
return text
|
||||
|
||||
|
||||
def _parse_json(raw: str) -> dict[str, Any] | None:
|
||||
if not raw:
|
||||
return None
|
||||
text = raw.strip()
|
||||
# Strip ```json fences if present.
|
||||
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
|
||||
if fence:
|
||||
text = fence.group(1).strip()
|
||||
# Otherwise grab the outermost { ... }.
|
||||
if not text.startswith("{"):
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
text = text[start : end + 1]
|
||||
try:
|
||||
data = json.loads(text)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based")
|
||||
return None
|
||||
|
||||
|
||||
def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str:
|
||||
def _bullets(items: Any) -> str:
|
||||
if isinstance(items, str):
|
||||
items = [items]
|
||||
if not isinstance(items, list):
|
||||
return ""
|
||||
return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip())
|
||||
|
||||
context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]"
|
||||
findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_"
|
||||
reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)"
|
||||
implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。"
|
||||
|
||||
related_lines = [f"- {link}" for link in related]
|
||||
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
|
||||
related_section = extractor._related_section_md(related_lines, source_lines)
|
||||
|
||||
return (
|
||||
f"# {title}\n\n"
|
||||
f"## Context\n\n{context}\n\n"
|
||||
f"## Finding / Decision\n\n{findings}\n\n"
|
||||
f"## Reasoning\n\n{reasoning}\n\n"
|
||||
f"## Implications\n\n{implications}\n\n"
|
||||
f"{related_section}"
|
||||
)
|
||||
|
||||
|
||||
def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""Normalize the LLM JSON into a list of per-note dicts.
|
||||
|
||||
Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy
|
||||
single-note shape (``{"title": ..., "findings": ...}``) for backward
|
||||
compatibility. Notes without any title/findings are dropped.
|
||||
"""
|
||||
raw_notes = data.get("notes")
|
||||
if isinstance(raw_notes, list):
|
||||
candidates = [n for n in raw_notes if isinstance(n, dict)]
|
||||
elif data.get("title") or data.get("findings") or data.get("context"):
|
||||
candidates = [data] # legacy single-object output
|
||||
else:
|
||||
candidates = []
|
||||
return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")]
|
||||
|
||||
|
||||
def _build_note_draft(
|
||||
note_data: dict[str, Any],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None,
|
||||
turns: list[dict[str, str]],
|
||||
title_override: str | None,
|
||||
tags: list[str] | None,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
source_key_suffix: str = "",
|
||||
) -> NoteDraft:
|
||||
"""Render one note dict into a :class:`NoteDraft`."""
|
||||
category = str(note_data.get("category") or "synthesis").strip().lower()
|
||||
if category not in _VALID_CATEGORIES:
|
||||
category = "synthesis"
|
||||
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
|
||||
summary = str(note_data.get("summary") or "").strip()
|
||||
related = extractor.related_links(project_name)
|
||||
body = _render_body(note_data, final_title, related, sources)
|
||||
|
||||
base_tags = list(tags or [])
|
||||
llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else []
|
||||
for tag in [*llm_tags, project_name or "zncm", "knowledge-base"]:
|
||||
tag = str(tag).strip()
|
||||
if tag and tag not in base_tags:
|
||||
base_tags.append(tag)
|
||||
|
||||
try:
|
||||
confidence = float(note_data.get("confidence") or 0.7)
|
||||
except (TypeError, ValueError):
|
||||
confidence = 0.7
|
||||
|
||||
entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else []
|
||||
relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else []
|
||||
|
||||
return NoteDraft(
|
||||
title=final_title,
|
||||
summary=summary or final_title,
|
||||
content_md=body,
|
||||
category=category,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
tags=base_tags,
|
||||
confidence=max(0.0, min(confidence, 1.0)),
|
||||
related=related,
|
||||
sources=sources,
|
||||
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
|
||||
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
|
||||
source_key=f"thread:{thread_id}{source_key_suffix}",
|
||||
)
|
||||
|
||||
|
||||
async def extract_thread_llm_multi(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
allow_multi: bool = True,
|
||||
max_notes: int = 4,
|
||||
) -> list[NoteDraft]:
|
||||
"""Distill a thread into one *or several* cross-linkable knowledge notes.
|
||||
|
||||
The LLM may split a conversation that covers multiple independent topics into
|
||||
several declarative notes. Returns ``[]`` when there is nothing to sediment
|
||||
or the LLM path fails — callers should fall back to the rule-based extractor.
|
||||
When ``allow_multi`` is false only the first note is kept.
|
||||
"""
|
||||
turns = extractor.split_turns(messages, max_messages=max_messages)
|
||||
if not turns:
|
||||
return []
|
||||
sources = sources or []
|
||||
|
||||
try:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
transcript = _build_transcript(turns, sources, max_chars=max_input_chars)
|
||||
prompt = [
|
||||
SystemMessage(content=_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
|
||||
HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"),
|
||||
]
|
||||
response = await model.ainvoke(prompt)
|
||||
raw = response.content if isinstance(response.content, str) else str(response.content)
|
||||
data = _parse_json(raw)
|
||||
except Exception:
|
||||
logger.exception("[knowledge] thread %s — LLM extraction call failed; using rule-based fallback", thread_id)
|
||||
return []
|
||||
|
||||
if not data:
|
||||
logger.warning("[knowledge] thread %s — LLM returned no parsable JSON; using rule-based fallback", thread_id)
|
||||
return []
|
||||
|
||||
note_dicts = _notes_from_data(data)
|
||||
if not note_dicts:
|
||||
reason = str(data.get("reason") or "")[:200]
|
||||
if _is_falsey(data.get("should_ingest")):
|
||||
# The model deliberately judged this conversation worthless (greeting,
|
||||
# meta chatter, no findings). Skip ingestion entirely — do NOT fall
|
||||
# back to the rule-based verbatim dump.
|
||||
logger.warning("[knowledge] thread %s SKIPPED — model judged not worth ingesting: %s", thread_id, reason)
|
||||
raise LLMRejectedIngest(reason or "no knowledge value")
|
||||
logger.warning(
|
||||
"[knowledge] thread %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
|
||||
thread_id,
|
||||
data.get("should_ingest"),
|
||||
reason,
|
||||
)
|
||||
return []
|
||||
if not allow_multi:
|
||||
note_dicts = note_dicts[:1]
|
||||
else:
|
||||
note_dicts = note_dicts[: max(1, max_notes)]
|
||||
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
# The user-supplied title override only applies to a single-note result;
|
||||
# multi-note splits keep each note's own LLM title.
|
||||
override = title if (len(note_dicts) == 1) else None
|
||||
suffix = "" if len(note_dicts) == 1 else f"#{i}"
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
turns=turns,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
source_key_suffix=suffix,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
async def extract_thread_llm(
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
max_messages: int = 80,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
) -> NoteDraft | None:
|
||||
"""Distill a thread into a single :class:`NoteDraft` (backward-compatible).
|
||||
|
||||
Thin wrapper over :func:`extract_thread_llm_multi` that returns the first
|
||||
note, or ``None`` when nothing was produced.
|
||||
"""
|
||||
try:
|
||||
drafts = await extract_thread_llm_multi(
|
||||
messages,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
mode=mode,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
max_messages=max_messages,
|
||||
model_name=model_name,
|
||||
max_input_chars=max_input_chars,
|
||||
allow_multi=False,
|
||||
)
|
||||
except LLMRejectedIngest:
|
||||
return None
|
||||
return drafts[0] if drafts else None
|
||||
@@ -0,0 +1,74 @@
|
||||
""".manifest.json read/write for the vault.
|
||||
|
||||
The manifest records, per page, the source key, content hash, vault path and
|
||||
update time — so batch ingest can skip already-sedimented threads (delta/dedup)
|
||||
exactly like obsidian-wiki's native ``.manifest.json``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def content_hash(text: str) -> str:
|
||||
"""Stable sha256 hex digest of ``text`` (used for dedup)."""
|
||||
return hashlib.sha256((text or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class Manifest:
|
||||
"""JSON manifest stored at ``<vault>/.manifest.json``."""
|
||||
|
||||
def __init__(self, root: Path | str) -> None:
|
||||
self.path = Path(root).resolve() / ".manifest.json"
|
||||
|
||||
def load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
|
||||
try:
|
||||
data = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
|
||||
data.setdefault("pages", {})
|
||||
data.setdefault("sources", {})
|
||||
return data
|
||||
|
||||
def save(self, data: dict[str, Any]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def record_page(
|
||||
self,
|
||||
*,
|
||||
note_id: str,
|
||||
vault_path: str,
|
||||
source_key: str | None,
|
||||
hash_value: str,
|
||||
updated: str,
|
||||
) -> None:
|
||||
data = self.load()
|
||||
data["pages"][note_id] = {
|
||||
"vault_path": vault_path,
|
||||
"source": source_key,
|
||||
"hash": hash_value,
|
||||
"updated": updated,
|
||||
}
|
||||
if source_key:
|
||||
data["sources"][source_key] = {"note_id": note_id, "hash": hash_value, "updated": updated}
|
||||
data["updated"] = updated
|
||||
self.save(data)
|
||||
|
||||
def remove_page(self, note_id: str) -> None:
|
||||
data = self.load()
|
||||
page = data["pages"].pop(note_id, None)
|
||||
if page and page.get("source"):
|
||||
data["sources"].pop(page["source"], None)
|
||||
self.save(data)
|
||||
|
||||
def has_source(self, source_key: str) -> bool:
|
||||
return source_key in self.load().get("sources", {})
|
||||
|
||||
def get_source(self, source_key: str) -> dict[str, Any] | None:
|
||||
return self.load().get("sources", {}).get(source_key)
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
"""obsidian-wiki compatible Markdown vault writer/reader.
|
||||
|
||||
Responsibilities (dev doc §10 ``markdown_vault.py``):
|
||||
|
||||
- Generate obsidian-wiki compatible frontmatter.
|
||||
- Produce safe, path-traversal-proof file names/paths.
|
||||
- Write/update/read Markdown note files.
|
||||
- Parse frontmatter + extract ``[[wikilinks]]``.
|
||||
|
||||
The vault layout follows §8 of the dev doc and stays compatible with
|
||||
``ar9av/obsidian-wiki`` (concepts/ references/ synthesis/ journal/ projects/…,
|
||||
plus index.md / log.md / hot.md / .manifest.json / _meta/taxonomy.md /
|
||||
wiki-export/).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
|
||||
_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
|
||||
_UNSAFE_CHARS_RE = re.compile(r"[^\w一-鿿-]+")
|
||||
|
||||
# Top-level vault directories that must exist (obsidian-wiki native layout).
|
||||
_VAULT_DIRS = ("concepts", "entities", "skills", "references", "synthesis", "journal", "projects", "_meta", "wiki-export")
|
||||
|
||||
|
||||
def slugify(title: str, *, max_len: int = 60) -> str:
|
||||
"""Return a filesystem-safe slug (ASCII + CJK), never empty."""
|
||||
text = unicodedata.normalize("NFKC", (title or "").strip()).lower()
|
||||
text = text.replace(" ", "-")
|
||||
text = _UNSAFE_CHARS_RE.sub("-", text).strip("-")
|
||||
text = re.sub(r"-{2,}", "-", text)
|
||||
if len(text) > max_len:
|
||||
text = text[:max_len].rstrip("-")
|
||||
return text or "note"
|
||||
|
||||
|
||||
def extract_wikilinks(text: str) -> list[str]:
|
||||
"""Return the targets of every ``[[wikilink]]`` in ``text`` (de-duplicated)."""
|
||||
seen: list[str] = []
|
||||
for match in _WIKILINK_RE.finditer(text or ""):
|
||||
target = match.group(1).strip()
|
||||
if target and target not in seen:
|
||||
seen.append(target)
|
||||
return seen
|
||||
|
||||
|
||||
def normalize_wikilink_target(target: str) -> str:
|
||||
"""Canonicalize a ``[[wikilink]]`` target for resolution/comparison.
|
||||
|
||||
Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a
|
||||
trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to
|
||||
the same canonical ``index`` key used by the index/graph/resolver layers.
|
||||
"""
|
||||
t = (target or "").strip()
|
||||
if not t:
|
||||
return ""
|
||||
t = t.split("|", 1)[0].strip()
|
||||
t = t.replace("\\", "/").lstrip("/")
|
||||
if t.lower().endswith(".md"):
|
||||
t = t[:-3]
|
||||
return t
|
||||
|
||||
|
||||
class VaultPathError(ValueError):
|
||||
"""Raised when a requested vault path escapes the vault root."""
|
||||
|
||||
|
||||
class MarkdownVault:
|
||||
"""Reads/writes Markdown files inside a single vault root directory."""
|
||||
|
||||
def __init__(self, root: Path | str) -> None:
|
||||
self.root = Path(root).resolve()
|
||||
|
||||
# -- path safety ------------------------------------------------------
|
||||
def resolve(self, rel_path: str) -> Path:
|
||||
"""Resolve ``rel_path`` under the vault root, blocking traversal."""
|
||||
rel = str(rel_path).replace("\\", "/").lstrip("/")
|
||||
candidate = (self.root / rel).resolve()
|
||||
try:
|
||||
candidate.relative_to(self.root)
|
||||
except ValueError as exc:
|
||||
raise VaultPathError(f"Path escapes vault root: {rel_path!r}") from exc
|
||||
return candidate
|
||||
|
||||
def note_relpath(self, *, category: str, project_name: str | None, slug: str, note_id: str) -> str:
|
||||
"""Build a vault-relative path for a note.
|
||||
|
||||
Project knowledge goes under ``projects/<project>/<category>/`` per
|
||||
wiki-capture; a short id suffix guarantees uniqueness.
|
||||
"""
|
||||
from deerflow.knowledge.skill_templates import category_to_dir
|
||||
|
||||
cat_dir = category_to_dir(category)
|
||||
safe_slug = slugify(slug)
|
||||
suffix = note_id.split("_")[-1][:8]
|
||||
filename = f"{safe_slug}-{suffix}.md"
|
||||
if project_name:
|
||||
return f"projects/{slugify(project_name)}/{cat_dir}/{filename}"
|
||||
return f"{cat_dir}/{filename}"
|
||||
|
||||
# -- scaffold ---------------------------------------------------------
|
||||
def ensure_scaffold(self, *, project_name: str | None = None) -> None:
|
||||
"""Create the vault directory tree + base files if missing."""
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
for d in _VAULT_DIRS:
|
||||
(self.root / d).mkdir(parents=True, exist_ok=True)
|
||||
if project_name:
|
||||
base = self.root / "projects" / slugify(project_name)
|
||||
for d in ("concepts", "references", "synthesis", "journal"):
|
||||
(base / d).mkdir(parents=True, exist_ok=True)
|
||||
# Base vault files.
|
||||
index = self.root / "index.md"
|
||||
if not index.exists():
|
||||
index.write_text("# Knowledge Index\n\n_页面索引。每次写入新页面后自动更新。_\n\n", encoding="utf-8")
|
||||
log = self.root / "log.md"
|
||||
if not log.exists():
|
||||
log.write_text("# Activity Log\n\n_CAPTURE / INGEST / QUERY / EXPORT 记录。_\n\n", encoding="utf-8")
|
||||
hot = self.root / "hot.md"
|
||||
if not hot.exists():
|
||||
hot.write_text("# Hot\n\n_近期活动摘要。_\n\n", encoding="utf-8")
|
||||
taxonomy = self.root / "_meta" / "taxonomy.md"
|
||||
if not taxonomy.exists():
|
||||
taxonomy.write_text(
|
||||
"# Taxonomy\n\n- synthesis — 多步骤分析、方案、结论\n- concepts — 概念、框架、模型\n"
|
||||
"- references — 外部来源、搜索结果、文章资料\n- decision — 架构或设计决策\n- journal — 完整会话摘要\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# -- frontmatter ------------------------------------------------------
|
||||
@staticmethod
|
||||
def build_frontmatter(
|
||||
*,
|
||||
note_id: str,
|
||||
title: str,
|
||||
category: str,
|
||||
tags: list[str],
|
||||
sources: list[str],
|
||||
summary: str,
|
||||
created: str,
|
||||
updated: str,
|
||||
base_confidence: float,
|
||||
lifecycle: str,
|
||||
provenance: dict[str, float] | None = None,
|
||||
relationships: list[dict[str, str]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build an obsidian-wiki / wiki-capture compatible frontmatter dict."""
|
||||
created_date = (created or "")[:10]
|
||||
return {
|
||||
"id": note_id,
|
||||
"title": title,
|
||||
"category": category,
|
||||
"tags": list(tags or []),
|
||||
"sources": list(sources or []),
|
||||
"created": created,
|
||||
"updated": updated,
|
||||
"summary": summary or "",
|
||||
"provenance": provenance or {"extracted": 0.7, "inferred": 0.2, "ambiguous": 0.1},
|
||||
"base_confidence": round(float(base_confidence or 0.0), 2),
|
||||
"lifecycle": lifecycle,
|
||||
"lifecycle_changed": created_date,
|
||||
"relationships": relationships or [],
|
||||
}
|
||||
|
||||
# -- IO ---------------------------------------------------------------
|
||||
def write_note(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> Path:
|
||||
"""Write ``frontmatter`` + ``body`` to ``rel_path`` (creates dirs)."""
|
||||
path = self.resolve(rel_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fm = yaml.safe_dump(frontmatter, allow_unicode=True, sort_keys=False, default_flow_style=False)
|
||||
path.write_text(f"---\n{fm}---\n\n{body.strip()}\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
def read_note(self, rel_path: str) -> tuple[dict[str, Any], str] | None:
|
||||
"""Return ``(frontmatter, body)`` for a note, or ``None`` if missing."""
|
||||
path = self.resolve(rel_path)
|
||||
if not path.exists():
|
||||
return None
|
||||
return self.parse(path.read_text(encoding="utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def parse(text: str) -> tuple[dict[str, Any], str]:
|
||||
"""Split raw Markdown into ``(frontmatter dict, body)``."""
|
||||
match = _FRONTMATTER_RE.match(text or "")
|
||||
if not match:
|
||||
return {}, text or ""
|
||||
try:
|
||||
fm = yaml.safe_load(match.group(1)) or {}
|
||||
except yaml.YAMLError:
|
||||
fm = {}
|
||||
if not isinstance(fm, dict):
|
||||
fm = {}
|
||||
return fm, match.group(2) or ""
|
||||
|
||||
def delete_note(self, rel_path: str) -> None:
|
||||
"""Remove a note file if present (used on hard delete; archive keeps it)."""
|
||||
path = self.resolve(rel_path)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
|
||||
|
||||
def now_iso_with_tz(dt: datetime) -> str:
|
||||
"""ISO-8601 string with timezone offset (frontmatter ``created``/``updated``)."""
|
||||
if dt.tzinfo is None:
|
||||
return dt.isoformat()
|
||||
return dt.isoformat(timespec="seconds")
|
||||
@@ -0,0 +1,141 @@
|
||||
"""ORM models for the product knowledge base.
|
||||
|
||||
Tables mirror the obsidian-wiki vault into the database so the product can
|
||||
list, filter, search and audit notes quickly without re-parsing Markdown. The
|
||||
vault Markdown remains the source-of-truth for ``content_md`` / frontmatter /
|
||||
graph; these rows are an index + audit layer.
|
||||
|
||||
The knowledge base is **global / shared** — there is no ``user_id`` isolation.
|
||||
Writes record ``created_by`` / ``updated_by`` for auditing only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class KnowledgeNoteRow(Base):
|
||||
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
|
||||
|
||||
__tablename__ = "knowledge_notes"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=0.0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
|
||||
class KnowledgeSourceRow(Base):
|
||||
"""Knowledge source table (mirrors ``knowledge_sources``)."""
|
||||
|
||||
__tablename__ = "knowledge_sources"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(64))
|
||||
thread_id: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(128))
|
||||
tool_name: Mapped[str | None] = mapped_column(String(128))
|
||||
title: Mapped[str | None] = mapped_column(String(512))
|
||||
url: Mapped[str | None] = mapped_column(Text)
|
||||
snippet: Mapped[str | None] = mapped_column(Text)
|
||||
raw_json: Mapped[str | None] = mapped_column(PortableLongText())
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeTagRow(Base):
|
||||
"""Tag table (mirrors ``knowledge_tags``)."""
|
||||
|
||||
__tablename__ = "knowledge_tags"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeNoteTagRow(Base):
|
||||
"""Note ↔ tag join table (mirrors ``knowledge_note_tags``)."""
|
||||
|
||||
__tablename__ = "knowledge_note_tags"
|
||||
__table_args__ = (UniqueConstraint("note_id", "tag_id", name="uq_knowledge_note_tags"),)
|
||||
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), primary_key=True)
|
||||
tag_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_tags.id", ondelete="CASCADE"), primary_key=True)
|
||||
|
||||
|
||||
class KnowledgeEmbeddingRow(Base):
|
||||
"""Per-chunk embedding index (phase 2, mirrors ``knowledge_embeddings``).
|
||||
|
||||
The vector is stored as a JSON float array; first-release similarity is
|
||||
computed in Python (no external vector DB). ``vector_id`` is reserved for a
|
||||
future external store mapping.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_embeddings"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
|
||||
chunk_index: Mapped[int] = mapped_column(Integer, default=0)
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
vector: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
vector_id: Mapped[str | None] = mapped_column(String(256))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeNoteVersionRow(Base):
|
||||
"""Immutable snapshot of a note before each update (phase 3 versioning)."""
|
||||
|
||||
__tablename__ = "knowledge_note_versions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
status: Mapped[str] = mapped_column(String(32))
|
||||
tags_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
edited_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
|
||||
class KnowledgeEntityRow(Base):
|
||||
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
|
||||
|
||||
__tablename__ = "knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), index=True)
|
||||
entity_type: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeRelationRow(Base):
|
||||
"""Relation between notes/entities (phase 4, mirrors ``knowledge_relations``)."""
|
||||
|
||||
__tablename__ = "knowledge_relations"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
from_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
to_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
from_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
to_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
relation_type: Mapped[str] = mapped_column(String(128))
|
||||
weight: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
"""ObsidianWikiAdapter — the obsidian-wiki vault facade.
|
||||
|
||||
Owns the Markdown vault: writes notes with wiki-capture frontmatter/body, keeps
|
||||
``index.md`` / ``log.md`` / ``hot.md`` / ``.manifest.json`` in sync after every
|
||||
write, and produces wiki-export graph artifacts. Output stays compatible with
|
||||
``ar9av/obsidian-wiki`` without ever invoking its CLI.
|
||||
|
||||
This class is storage-only (no DB, no LLM). The :class:`KnowledgeService`
|
||||
combines it with the repository + extractor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge import exporter
|
||||
from deerflow.knowledge.manifest import Manifest, content_hash
|
||||
from deerflow.knowledge.markdown_vault import MarkdownVault
|
||||
from deerflow.knowledge.schemas import NoteDraft
|
||||
|
||||
_INDEX_PAGES_HEADER = "## Pages"
|
||||
_HOT_HEADER = "## Recent"
|
||||
_HOT_MAX = 20
|
||||
|
||||
|
||||
class ObsidianWikiAdapter:
|
||||
"""Read/write an obsidian-wiki compatible vault."""
|
||||
|
||||
def __init__(self, vault_root: Path | str, *, project_name: str = "zncm", export_html: bool = True, export_json: bool = True) -> None:
|
||||
self.vault = MarkdownVault(vault_root)
|
||||
self.root = self.vault.root
|
||||
self.manifest = Manifest(self.root)
|
||||
self.project_name = project_name
|
||||
self.export_html = export_html
|
||||
self.export_json = export_json
|
||||
|
||||
def ensure_ready(self) -> None:
|
||||
self.vault.ensure_scaffold(project_name=self.project_name)
|
||||
|
||||
# -- writes -----------------------------------------------------------
|
||||
def write_note(
|
||||
self,
|
||||
note_id: str,
|
||||
draft: NoteDraft,
|
||||
*,
|
||||
created: str,
|
||||
updated: str,
|
||||
status: str,
|
||||
) -> str:
|
||||
"""Write the note file, sync vault metadata, return the vault path."""
|
||||
self.ensure_ready()
|
||||
rel = self.vault.note_relpath(category=draft.category, project_name=self.project_name, slug=draft.title, note_id=note_id)
|
||||
sources_fm = self._frontmatter_sources(draft)
|
||||
relationships = [{"target": link, "type": "related"} for link in (draft.related or [])[:8]]
|
||||
frontmatter = self.vault.build_frontmatter(
|
||||
note_id=note_id,
|
||||
title=draft.title,
|
||||
category=draft.category,
|
||||
tags=draft.tags,
|
||||
sources=sources_fm,
|
||||
summary=draft.summary,
|
||||
created=created,
|
||||
updated=updated,
|
||||
base_confidence=draft.confidence,
|
||||
lifecycle=status,
|
||||
relationships=relationships,
|
||||
)
|
||||
self.vault.write_note(rel, frontmatter, draft.content_md)
|
||||
self.manifest.record_page(
|
||||
note_id=note_id,
|
||||
vault_path=rel,
|
||||
source_key=draft.source_key,
|
||||
hash_value=content_hash(draft.content_md),
|
||||
updated=updated,
|
||||
)
|
||||
self._update_index(note_id, rel, draft.title, draft.summary)
|
||||
self._append_log("CAPTURE" if draft.source_type in ("thread", "search", "tool") else "INGEST", note_id, draft.title, updated)
|
||||
self._update_hot(draft.title, rel, updated)
|
||||
return rel
|
||||
|
||||
def update_note_file(
|
||||
self,
|
||||
note_id: str,
|
||||
vault_path: str | None,
|
||||
*,
|
||||
title: str,
|
||||
content_md: str,
|
||||
tags: list[str],
|
||||
status: str,
|
||||
summary: str,
|
||||
category: str,
|
||||
created: str,
|
||||
updated: str,
|
||||
) -> str:
|
||||
"""Rewrite a note file in place (frontmatter refreshed), sync metadata."""
|
||||
self.ensure_ready()
|
||||
rel = vault_path
|
||||
if not rel:
|
||||
rel = self.vault.note_relpath(category=category, project_name=self.project_name, slug=title, note_id=note_id)
|
||||
# Preserve original `sources`/`provenance`/`category` frontmatter when present.
|
||||
existing = self.vault.read_note(rel)
|
||||
sources_fm = existing[0].get("sources", []) if existing else []
|
||||
provenance = existing[0].get("provenance") if existing else None
|
||||
relationships = existing[0].get("relationships") if existing else []
|
||||
effective_category = (existing[0].get("category") if existing else None) or category
|
||||
frontmatter = self.vault.build_frontmatter(
|
||||
note_id=note_id,
|
||||
title=title,
|
||||
category=effective_category,
|
||||
tags=tags,
|
||||
sources=sources_fm,
|
||||
summary=summary,
|
||||
created=created,
|
||||
updated=updated,
|
||||
base_confidence=float((existing[0].get("base_confidence") if existing else 0.0) or 0.0),
|
||||
lifecycle=status,
|
||||
provenance=provenance,
|
||||
relationships=relationships,
|
||||
)
|
||||
self.vault.write_note(rel, frontmatter, content_md)
|
||||
self.manifest.record_page(
|
||||
note_id=note_id,
|
||||
vault_path=rel,
|
||||
source_key=self.manifest.load().get("pages", {}).get(note_id, {}).get("source"),
|
||||
hash_value=content_hash(content_md),
|
||||
updated=updated,
|
||||
)
|
||||
self._update_index(note_id, rel, title, summary)
|
||||
self._append_log("UPDATE", note_id, title, updated)
|
||||
self._update_hot(title, rel, updated)
|
||||
return rel
|
||||
|
||||
def archive_note(self, note_id: str, vault_path: str | None, *, updated: str) -> None:
|
||||
"""Mark a note archived in its frontmatter + log (soft delete)."""
|
||||
if vault_path:
|
||||
parsed = self.vault.read_note(vault_path)
|
||||
if parsed:
|
||||
fm, body = parsed
|
||||
fm["lifecycle"] = "archived"
|
||||
fm["updated"] = updated
|
||||
self.vault.write_note(vault_path, fm, body)
|
||||
self._remove_from_index(note_id)
|
||||
self._append_log("ARCHIVE", note_id, "", updated)
|
||||
|
||||
def export_graph(
|
||||
self,
|
||||
notes: list[dict[str, Any]],
|
||||
*,
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
updated: str,
|
||||
) -> dict[str, Any]:
|
||||
self.ensure_ready()
|
||||
result = exporter.export_all(
|
||||
self.root,
|
||||
notes,
|
||||
entities=entities,
|
||||
relations=relations,
|
||||
write_html=self.export_html,
|
||||
write_json=self.export_json,
|
||||
)
|
||||
self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
|
||||
return result
|
||||
|
||||
def export_file_path(self, filename: str) -> Path | None:
|
||||
allowed = {"graph.json", "graph.graphml", "cypher.txt", "graph.html"}
|
||||
if filename not in allowed:
|
||||
return None
|
||||
path = (self.root / "wiki-export" / filename).resolve()
|
||||
try:
|
||||
path.relative_to(self.root)
|
||||
except ValueError:
|
||||
return None
|
||||
return path if path.exists() else None
|
||||
|
||||
# -- frontmatter source serialization ---------------------------------
|
||||
@staticmethod
|
||||
def _frontmatter_sources(draft: NoteDraft) -> list[str]:
|
||||
out: list[str] = []
|
||||
if draft.source_type == "thread" and draft.source_id:
|
||||
out.append(f"thread:{draft.source_id}")
|
||||
for s in draft.sources[:20]:
|
||||
if s.url:
|
||||
out.append(f"url:{s.url}")
|
||||
elif s.thread_id:
|
||||
out.append(f"thread:{s.thread_id}")
|
||||
elif s.tool_name:
|
||||
out.append(f"tool:{s.tool_name}")
|
||||
# De-dup preserving order.
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for item in out:
|
||||
if item not in seen:
|
||||
seen.add(item)
|
||||
deduped.append(item)
|
||||
return deduped
|
||||
|
||||
# -- index.md ---------------------------------------------------------
|
||||
def _index_path(self) -> Path:
|
||||
return self.root / "index.md"
|
||||
|
||||
def _update_index(self, note_id: str, rel: str, title: str, summary: str | None) -> None:
|
||||
path = self._index_path()
|
||||
text = path.read_text(encoding="utf-8") if path.exists() else "# Knowledge Index\n\n"
|
||||
link = rel[:-3] if rel.endswith(".md") else rel
|
||||
summary_txt = (summary or "").strip().replace("\n", " ")
|
||||
if len(summary_txt) > 120:
|
||||
summary_txt = summary_txt[:120] + "…"
|
||||
line = f"- [[{link}|{title}]]" + (f" — {summary_txt}" if summary_txt else "") + f" <!-- id:{note_id} -->"
|
||||
lines = text.splitlines()
|
||||
# Drop any existing line for this note id.
|
||||
lines = [ln for ln in lines if f"<!-- id:{note_id} -->" not in ln]
|
||||
if _INDEX_PAGES_HEADER not in text:
|
||||
lines.append("")
|
||||
lines.append(_INDEX_PAGES_HEADER)
|
||||
lines.append("")
|
||||
# Insert after the Pages header.
|
||||
try:
|
||||
insert_at = lines.index(_INDEX_PAGES_HEADER) + 1
|
||||
except ValueError:
|
||||
insert_at = len(lines)
|
||||
lines.insert(insert_at, line)
|
||||
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
def _remove_from_index(self, note_id: str) -> None:
|
||||
path = self._index_path()
|
||||
if not path.exists():
|
||||
return
|
||||
lines = [ln for ln in path.read_text(encoding="utf-8").splitlines() if f"<!-- id:{note_id} -->" not in ln]
|
||||
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
# -- log.md -----------------------------------------------------------
|
||||
def _append_log(self, action: str, note_id: str, label: str, updated: str) -> None:
|
||||
path = self.root / "log.md"
|
||||
text = path.read_text(encoding="utf-8") if path.exists() else "# Activity Log\n\n"
|
||||
entry = f"- {updated} {action} {note_id}" + (f" — {label}" if label else "")
|
||||
path.write_text(text.rstrip() + "\n" + entry + "\n", encoding="utf-8")
|
||||
|
||||
# -- hot.md -----------------------------------------------------------
|
||||
def _update_hot(self, title: str, rel: str, updated: str) -> None:
|
||||
path = self.root / "hot.md"
|
||||
link = rel[:-3] if rel.endswith(".md") else rel
|
||||
new_line = f"- {updated} [[{link}|{title}]]"
|
||||
existing_lines: list[str] = []
|
||||
if path.exists():
|
||||
for ln in path.read_text(encoding="utf-8").splitlines():
|
||||
if ln.startswith("- ") and "[[" in ln:
|
||||
existing_lines.append(ln)
|
||||
existing_lines = [ln for ln in existing_lines if f"[[{link}" not in ln]
|
||||
existing_lines.insert(0, new_line)
|
||||
existing_lines = existing_lines[:_HOT_MAX]
|
||||
body = "# Hot\n\n_近期活动摘要。_\n\n" + _HOT_HEADER + "\n\n" + "\n".join(existing_lines) + "\n"
|
||||
path.write_text(body, encoding="utf-8")
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Database read/write for the knowledge base.
|
||||
|
||||
This layer never calls an LLM and never touches the Markdown vault — it only
|
||||
persists/queries the relational mirror (notes, sources, tags). The vault and
|
||||
extraction live in :mod:`deerflow.knowledge.markdown_vault` and
|
||||
:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.knowledge.models import (
|
||||
KnowledgeEmbeddingRow,
|
||||
KnowledgeEntityRow,
|
||||
KnowledgeNoteRow,
|
||||
KnowledgeNoteTagRow,
|
||||
KnowledgeNoteVersionRow,
|
||||
KnowledgeRelationRow,
|
||||
KnowledgeSourceRow,
|
||||
KnowledgeTagRow,
|
||||
)
|
||||
from deerflow.knowledge.schemas import SourceDraft
|
||||
|
||||
|
||||
def _new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
|
||||
def _iso(value: Any) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
class KnowledgeRepository:
|
||||
"""SQLAlchemy-backed repository for knowledge notes/sources/tags."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# -- serialization ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
d["updated_at"] = _iso(d.get("updated_at"))
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
|
||||
d = row.to_dict()
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
return d
|
||||
|
||||
# -- tags -------------------------------------------------------------
|
||||
async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
|
||||
cleaned = []
|
||||
seen: set[str] = set()
|
||||
for raw in names or []:
|
||||
name = (raw or "").strip()
|
||||
if not name or name.lower() in seen:
|
||||
continue
|
||||
seen.add(name.lower())
|
||||
cleaned.append(name)
|
||||
if not cleaned:
|
||||
return []
|
||||
existing = {
|
||||
r.name.lower(): r
|
||||
for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
|
||||
}
|
||||
result: list[KnowledgeTagRow] = []
|
||||
for name in cleaned:
|
||||
row = existing.get(name.lower())
|
||||
if row is None:
|
||||
row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
existing[name.lower()] = row
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
|
||||
stmt = (
|
||||
select(KnowledgeTagRow.name)
|
||||
.join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
||||
.where(KnowledgeNoteTagRow.note_id == note_id)
|
||||
.order_by(KnowledgeTagRow.name)
|
||||
)
|
||||
return list((await session.execute(stmt)).scalars())
|
||||
|
||||
async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
|
||||
if not note_ids:
|
||||
return {}
|
||||
stmt = (
|
||||
select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
|
||||
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
||||
.where(KnowledgeNoteTagRow.note_id.in_(note_ids))
|
||||
.order_by(KnowledgeTagRow.name)
|
||||
)
|
||||
out: dict[str, list[str]] = {nid: [] for nid in note_ids}
|
||||
for note_id, name in (await session.execute(stmt)).all():
|
||||
out.setdefault(note_id, []).append(name)
|
||||
return out
|
||||
|
||||
async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
|
||||
await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
|
||||
tags = await self._get_or_create_tags(session, names)
|
||||
for tag in tags:
|
||||
session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
|
||||
|
||||
# -- notes ------------------------------------------------------------
|
||||
async def create_note(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
summary: str | None,
|
||||
content_md: str,
|
||||
source_type: str,
|
||||
source_id: str | None,
|
||||
status: str,
|
||||
confidence: float = 0.0,
|
||||
vault_path: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
created_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
note_id = _new_id("kb")
|
||||
now = datetime.now(UTC)
|
||||
row = KnowledgeNoteRow(
|
||||
id=note_id,
|
||||
title=title[:512],
|
||||
summary=summary,
|
||||
content_md=content_md,
|
||||
vault_path=vault_path,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
status=status,
|
||||
confidence=confidence,
|
||||
created_by=created_by,
|
||||
updated_by=created_by,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
await self._replace_note_tags(session, note_id, tags or [])
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, note_id)
|
||||
return d
|
||||
|
||||
async def set_vault_path(self, note_id: str, vault_path: str) -> None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeNoteRow, note_id)
|
||||
if row is None:
|
||||
return
|
||||
row.vault_path = vault_path
|
||||
await session.commit()
|
||||
|
||||
async def update_note(
|
||||
self,
|
||||
note_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
summary: str | None = None,
|
||||
content_md: str | None = None,
|
||||
status: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
vault_path: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeNoteRow, note_id)
|
||||
if row is None:
|
||||
return None
|
||||
if title is not None:
|
||||
row.title = title[:512]
|
||||
if summary is not None:
|
||||
row.summary = summary
|
||||
if content_md is not None:
|
||||
row.content_md = content_md
|
||||
if status is not None:
|
||||
row.status = status
|
||||
if vault_path is not None:
|
||||
row.vault_path = vault_path
|
||||
if updated_by is not None:
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = datetime.now(UTC)
|
||||
if tags is not None:
|
||||
await self._replace_note_tags(session, note_id, tags)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, note_id)
|
||||
return d
|
||||
|
||||
async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
|
||||
return await self.update_note(note_id, status=status, updated_by=updated_by)
|
||||
|
||||
async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeNoteRow, note_id)
|
||||
if row is None:
|
||||
return None
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, note_id)
|
||||
if with_sources:
|
||||
stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
|
||||
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
|
||||
return d
|
||||
|
||||
async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None:
|
||||
"""Return the first (non-archived) note whose vault_path matches.
|
||||
|
||||
``vault_path`` is matched both with and without the ``.md`` suffix, since
|
||||
wikilink targets are stored without it while ``vault_path`` keeps it.
|
||||
"""
|
||||
stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path
|
||||
candidates = {vault_path, stripped, f"{stripped}.md"}
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeNoteRow)
|
||||
.where(KnowledgeNoteRow.vault_path.in_(list(candidates)))
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
return None
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, row.id)
|
||||
return d
|
||||
|
||||
async def find_by_title(self, title: str) -> dict[str, Any] | None:
|
||||
"""Return the first (non-archived) note with a case-insensitive title match."""
|
||||
name = (title or "").strip()
|
||||
if not name:
|
||||
return None
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeNoteRow)
|
||||
.where(func.lower(KnowledgeNoteRow.title) == name.lower())
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
.order_by(KnowledgeNoteRow.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
return None
|
||||
d = self._note_to_dict(row)
|
||||
d["tags"] = await self._tags_for(session, row.id)
|
||||
return d
|
||||
|
||||
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
|
||||
"""Return the first (non-archived) note for a (source_type, source_id)."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeNoteRow)
|
||||
.where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
.limit(1)
|
||||
)
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
return self._note_to_dict(row) if row else None
|
||||
|
||||
async def list_notes(
|
||||
self,
|
||||
*,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeNoteRow)
|
||||
count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
|
||||
|
||||
conditions = []
|
||||
if status:
|
||||
conditions.append(KnowledgeNoteRow.status == status)
|
||||
elif not include_archived:
|
||||
conditions.append(KnowledgeNoteRow.status != "archived")
|
||||
if source_type:
|
||||
conditions.append(KnowledgeNoteRow.source_type == source_type)
|
||||
if keyword:
|
||||
like = f"%{keyword}%"
|
||||
conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
|
||||
if tag:
|
||||
tag_subq = (
|
||||
select(KnowledgeNoteTagRow.note_id)
|
||||
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
|
||||
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
|
||||
)
|
||||
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
|
||||
|
||||
for cond in conditions:
|
||||
stmt = stmt.where(cond)
|
||||
count_stmt = count_stmt.where(cond)
|
||||
|
||||
total = (await session.execute(count_stmt)).scalar_one()
|
||||
stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
ids = [r.id for r in rows]
|
||||
tags_map = await self._tags_for_many(session, ids)
|
||||
items = []
|
||||
for r in rows:
|
||||
d = self._note_to_dict(r)
|
||||
d["tags"] = tags_map.get(r.id, [])
|
||||
items.append(d)
|
||||
return items, int(total)
|
||||
|
||||
async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
|
||||
"""Return every note (light fields + tags) for graph export."""
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeNoteRow)
|
||||
if not include_archived:
|
||||
stmt = stmt.where(KnowledgeNoteRow.status != "archived")
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
tags_map = await self._tags_for_many(session, [r.id for r in rows])
|
||||
out = []
|
||||
for r in rows:
|
||||
d = self._note_to_dict(r)
|
||||
d["tags"] = tags_map.get(r.id, [])
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
# -- embeddings (phase 2) --------------------------------------------
|
||||
async def replace_embeddings(self, note_id: str, chunks: list[tuple[int, str, list[float]]]) -> None:
|
||||
"""Replace all embedding rows for a note with ``(index, content, vector)``."""
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
|
||||
for idx, content, vector in chunks:
|
||||
session.add(
|
||||
KnowledgeEmbeddingRow(
|
||||
id=_new_id("kbemb"),
|
||||
note_id=note_id,
|
||||
chunk_index=idx,
|
||||
content=content,
|
||||
vector=list(vector),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def delete_embeddings(self, note_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
|
||||
await session.commit()
|
||||
|
||||
async def all_embeddings(self) -> list[dict[str, Any]]:
|
||||
"""Return every embedding chunk joined to its (non-archived) note."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeEmbeddingRow, KnowledgeNoteRow.title, KnowledgeNoteRow.status)
|
||||
.join(KnowledgeNoteRow, KnowledgeNoteRow.id == KnowledgeEmbeddingRow.note_id)
|
||||
.where(KnowledgeNoteRow.status != "archived")
|
||||
)
|
||||
out = []
|
||||
for emb, title, _status in (await session.execute(stmt)).all():
|
||||
out.append({"note_id": emb.note_id, "chunk_index": emb.chunk_index, "content": emb.content, "vector": emb.vector or [], "title": title})
|
||||
return out
|
||||
|
||||
async def has_embeddings(self) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(KnowledgeEmbeddingRow.id).limit(1))).first()
|
||||
return row is not None
|
||||
|
||||
# -- versions (phase 3) ----------------------------------------------
|
||||
async def add_version(self, note: dict[str, Any], *, edited_by: str | None) -> None:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
KnowledgeNoteVersionRow(
|
||||
id=_new_id("kbver"),
|
||||
note_id=note["id"],
|
||||
title=note.get("title") or "",
|
||||
summary=note.get("summary"),
|
||||
content_md=note.get("content_md") or "",
|
||||
status=note.get("status") or "approved",
|
||||
tags_json=note.get("tags") or [],
|
||||
edited_by=edited_by,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeNoteVersionRow).where(KnowledgeNoteVersionRow.note_id == note_id).order_by(KnowledgeNoteVersionRow.created_at.desc())
|
||||
out = []
|
||||
for r in (await session.execute(stmt)).scalars():
|
||||
d = r.to_dict()
|
||||
d["tags"] = d.pop("tags_json", [])
|
||||
d["created_at"] = _iso(d.get("created_at"))
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
# -- entities & relations (phase 4) ----------------------------------
|
||||
async def get_or_create_entity(self, name: str, entity_type: str | None) -> dict[str, Any]:
|
||||
name = (name or "").strip()
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeEntityRow).where(func.lower(KnowledgeEntityRow.name) == name.lower())
|
||||
row = (await session.execute(stmt)).scalars().first()
|
||||
if row is None:
|
||||
row = KnowledgeEntityRow(id=_new_id("kbent"), name=name, entity_type=entity_type, created_at=datetime.now(UTC))
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return {"id": row.id, "name": row.name, "entity_type": row.entity_type}
|
||||
|
||||
async def add_relation(
|
||||
self,
|
||||
*,
|
||||
from_note_id: str | None = None,
|
||||
to_note_id: str | None = None,
|
||||
from_entity_id: str | None = None,
|
||||
to_entity_id: str | None = None,
|
||||
relation_type: str,
|
||||
weight: float = 1.0,
|
||||
) -> None:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
KnowledgeRelationRow(
|
||||
id=_new_id("kbrel"),
|
||||
from_note_id=from_note_id,
|
||||
to_note_id=to_note_id,
|
||||
from_entity_id=from_entity_id,
|
||||
to_entity_id=to_entity_id,
|
||||
relation_type=relation_type,
|
||||
weight=weight,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def clear_note_relations(self, note_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(KnowledgeRelationRow).where(KnowledgeRelationRow.from_note_id == note_id))
|
||||
await session.commit()
|
||||
|
||||
async def list_note_entities(self, note_id: str) -> list[dict[str, Any]]:
|
||||
"""Return the entities mentioned by a note (for note-detail display)."""
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(KnowledgeEntityRow.name, KnowledgeEntityRow.entity_type)
|
||||
.join(KnowledgeRelationRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
|
||||
.where(KnowledgeRelationRow.from_note_id == note_id)
|
||||
.order_by(KnowledgeEntityRow.name)
|
||||
)
|
||||
seen: set[str] = set()
|
||||
out: list[dict[str, Any]] = []
|
||||
for name, etype in (await session.execute(stmt)).all():
|
||||
key = (name or "").lower()
|
||||
if not name or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({"name": name, "entity_type": etype})
|
||||
return out
|
||||
|
||||
async def prune_orphan_entities(self) -> int:
|
||||
"""Delete entity rows no longer referenced by any relation. Returns count."""
|
||||
async with self._sf() as session:
|
||||
referenced: set[str] = set()
|
||||
for col in (KnowledgeRelationRow.from_entity_id, KnowledgeRelationRow.to_entity_id):
|
||||
for (eid,) in (await session.execute(select(col).where(col.isnot(None)))).all():
|
||||
referenced.add(eid)
|
||||
all_ids = list((await session.execute(select(KnowledgeEntityRow.id))).scalars())
|
||||
orphans = [eid for eid in all_ids if eid not in referenced]
|
||||
if orphans:
|
||||
await session.execute(delete(KnowledgeEntityRow).where(KnowledgeEntityRow.id.in_(orphans)))
|
||||
await session.commit()
|
||||
return len(orphans)
|
||||
|
||||
async def list_entities(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(KnowledgeEntityRow))).scalars()
|
||||
return [{"id": r.id, "name": r.name, "entity_type": r.entity_type} for r in rows]
|
||||
|
||||
async def list_relations(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(KnowledgeRelationRow))).scalars()
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"from_note_id": r.from_note_id,
|
||||
"to_note_id": r.to_note_id,
|
||||
"from_entity_id": r.from_entity_id,
|
||||
"to_entity_id": r.to_entity_id,
|
||||
"relation_type": r.relation_type,
|
||||
"weight": r.weight,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
# -- sources ----------------------------------------------------------
|
||||
async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
|
||||
if not sources:
|
||||
return
|
||||
import json
|
||||
|
||||
async with self._sf() as session:
|
||||
for src in sources:
|
||||
session.add(
|
||||
KnowledgeSourceRow(
|
||||
id=_new_id("kbsrc"),
|
||||
note_id=note_id,
|
||||
source_type=src.source_type,
|
||||
thread_id=src.thread_id,
|
||||
message_id=src.message_id,
|
||||
tool_name=src.tool_name,
|
||||
title=(src.title or "")[:512] or None,
|
||||
url=src.url,
|
||||
snippet=src.snippet,
|
||||
raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
|
||||
created_by=created_by,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Internal domain models for the knowledge module.
|
||||
|
||||
These are plain dataclasses passed between the extractor, the vault writer and
|
||||
the repository. The Gateway router defines its own Pydantic request/response
|
||||
models on top of the dicts the service returns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# Allowed note source types (see product-knowledge-base-dev.md §7.1).
|
||||
SOURCE_TYPES = ("thread", "search", "tool", "file", "manual", "report")
|
||||
# Allowed lifecycle states.
|
||||
STATUSES = ("draft", "approved", "archived")
|
||||
# obsidian-wiki wiki-capture categories.
|
||||
CATEGORIES = ("synthesis", "concepts", "references", "decision", "journal")
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceDraft:
|
||||
"""A normalized knowledge source (search result, tool call, thread, …)."""
|
||||
|
||||
source_type: str
|
||||
thread_id: str | None = None
|
||||
message_id: str | None = None
|
||||
tool_name: str | None = None
|
||||
title: str | None = None
|
||||
url: str | None = None
|
||||
snippet: str | None = None
|
||||
raw: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NoteDraft:
|
||||
"""A knowledge note ready to be persisted + written to the vault.
|
||||
|
||||
Produced by the extractor (from a thread / search / tool result) or built
|
||||
directly for a manual note. ``content_md`` here is the *body* — the vault
|
||||
writer prepends obsidian-wiki frontmatter.
|
||||
"""
|
||||
|
||||
title: str
|
||||
summary: str
|
||||
content_md: str
|
||||
category: str = "synthesis"
|
||||
source_type: str = "manual"
|
||||
source_id: str | None = None
|
||||
status: str = "approved"
|
||||
confidence: float = 0.0
|
||||
tags: list[str] = field(default_factory=list)
|
||||
key_points: list[str] = field(default_factory=list)
|
||||
actions: list[str] = field(default_factory=list)
|
||||
related: list[str] = field(default_factory=list)
|
||||
sources: list[SourceDraft] = field(default_factory=list)
|
||||
# Phase 4: extracted graph signal. entities=[{name,type}],
|
||||
# relations=[{from,to,type}] (names refer to entities or this note's title).
|
||||
entities: list[dict] = field(default_factory=list)
|
||||
relations: list[dict] = field(default_factory=list)
|
||||
# Stable identity hint used for manifest dedup (e.g. ``thread:thread_id``).
|
||||
source_key: str | None = None
|
||||
|
||||
|
||||
def normalize_status(status: str | None) -> str:
|
||||
"""Coerce an arbitrary status string to a valid lifecycle state."""
|
||||
if status in STATUSES:
|
||||
return status
|
||||
return "approved"
|
||||
|
||||
|
||||
def normalize_source_type(source_type: str | None) -> str:
|
||||
"""Coerce an arbitrary source-type string to a valid value."""
|
||||
if source_type in SOURCE_TYPES:
|
||||
return source_type
|
||||
return "manual"
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Knowledge search: keyword / vector / hybrid (phase 1 + 2).
|
||||
|
||||
- ``keyword`` — title>tags>summary>body scoring over the DB mirror (always works).
|
||||
- ``vector`` — cosine over per-chunk embeddings (requires a configured
|
||||
embedding endpoint; otherwise transparently falls back to ``keyword``).
|
||||
- ``hybrid`` — min-max-normalized blend of keyword + vector.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.knowledge.embeddings import cosine_similarity
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.knowledge.embeddings import EmbeddingClient
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
|
||||
|
||||
|
||||
def _tokens(text: str) -> list[str]:
|
||||
return [t.lower() for t in _TOKEN_RE.findall(text or "")]
|
||||
|
||||
|
||||
def _snippet(content: str, terms: list[str], *, width: int = 160) -> str:
|
||||
body = re.sub(r"\s+", " ", content or "").strip()
|
||||
low = body.lower()
|
||||
for term in terms:
|
||||
idx = low.find(term)
|
||||
if idx >= 0:
|
||||
start = max(0, idx - width // 3)
|
||||
end = min(len(body), start + width)
|
||||
prefix = "…" if start > 0 else ""
|
||||
suffix = "…" if end < len(body) else ""
|
||||
return f"{prefix}{body[start:end]}{suffix}"
|
||||
return body[:width] + ("…" if len(body) > width else "")
|
||||
|
||||
|
||||
async def _keyword_scores(repo: KnowledgeRepository, query: str, *, candidate_cap: int) -> dict[str, tuple[float, dict[str, Any]]]:
|
||||
terms = _tokens(query)
|
||||
if not terms:
|
||||
return {}
|
||||
candidates, _total = await repo.list_notes(limit=candidate_cap, offset=0)
|
||||
scored: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
for note in candidates:
|
||||
title = (note.get("title") or "").lower()
|
||||
summary = (note.get("summary") or "").lower()
|
||||
content = (note.get("content_md") or "").lower()
|
||||
tags = " ".join(note.get("tags") or []).lower()
|
||||
score = 0.0
|
||||
for term in terms:
|
||||
if term in title:
|
||||
score += 3.0
|
||||
if term in tags:
|
||||
score += 2.0
|
||||
if term in summary:
|
||||
score += 2.0
|
||||
score += min(content.count(term), 5) * 0.5
|
||||
if score > 0:
|
||||
scored[note["id"]] = (score, note)
|
||||
return scored
|
||||
|
||||
|
||||
async def _vector_scores(
|
||||
repo: KnowledgeRepository,
|
||||
query: str,
|
||||
embedding_client: EmbeddingClient,
|
||||
) -> dict[str, tuple[float, dict[str, Any]]]:
|
||||
qvec = await embedding_client.embed_query(query)
|
||||
if not qvec:
|
||||
return {}
|
||||
embeddings = await repo.all_embeddings()
|
||||
best: dict[str, tuple[float, str, str]] = {} # note_id -> (score, title, snippet)
|
||||
for emb in embeddings:
|
||||
sim = cosine_similarity(qvec, emb.get("vector") or [])
|
||||
if sim <= 0:
|
||||
continue
|
||||
nid = emb["note_id"]
|
||||
if nid not in best or sim > best[nid][0]:
|
||||
best[nid] = (sim, emb.get("title") or "", emb.get("content") or "")
|
||||
return {nid: (score, {"id": nid, "title": title, "_chunk": chunk}) for nid, (score, title, chunk) in best.items()}
|
||||
|
||||
|
||||
def _normalize(scores: dict[str, float]) -> dict[str, float]:
|
||||
if not scores:
|
||||
return {}
|
||||
mx = max(scores.values())
|
||||
if mx <= 0:
|
||||
return {k: 0.0 for k in scores}
|
||||
return {k: v / mx for k, v in scores.items()}
|
||||
|
||||
|
||||
async def search_notes(
|
||||
repo: KnowledgeRepository,
|
||||
query: str,
|
||||
*,
|
||||
mode: str = "keyword",
|
||||
limit: int = 8,
|
||||
candidate_cap: int = 500,
|
||||
embedding_client: EmbeddingClient | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Score notes against ``query`` and return the top ``limit`` matches."""
|
||||
terms = _tokens(query)
|
||||
if not terms:
|
||||
return []
|
||||
|
||||
want_vector = mode in ("vector", "hybrid") and embedding_client is not None and embedding_client.is_enabled()
|
||||
keyword = await _keyword_scores(repo, query, candidate_cap=candidate_cap) if mode != "vector" or not want_vector else {}
|
||||
vector = await _vector_scores(repo, query, embedding_client) if want_vector else {}
|
||||
|
||||
# Pure vector (when available); else keyword; hybrid blends both.
|
||||
if mode == "vector" and want_vector:
|
||||
combined = {nid: (sc, meta) for nid, (sc, meta) in vector.items()}
|
||||
ranked = sorted(combined.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
return await _materialize(repo, ranked, terms)
|
||||
|
||||
if mode == "hybrid" and want_vector:
|
||||
kw_norm = _normalize({k: v[0] for k, v in keyword.items()})
|
||||
vec_norm = _normalize({k: v[0] for k, v in vector.items()})
|
||||
ids = set(kw_norm) | set(vec_norm)
|
||||
blended: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
for nid in ids:
|
||||
score = 0.5 * kw_norm.get(nid, 0.0) + 0.5 * vec_norm.get(nid, 0.0)
|
||||
meta = keyword.get(nid, (0, None))[1] or vector.get(nid, (0, {}))[1]
|
||||
blended[nid] = (score, meta)
|
||||
ranked = sorted(blended.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
return await _materialize(repo, ranked, terms)
|
||||
|
||||
# keyword (default, and fallback when vector unavailable)
|
||||
ranked = sorted(keyword.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
if not ranked and vector: # mode=vector requested but keyword empty: use vector
|
||||
ranked = sorted(vector.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
|
||||
return await _materialize(repo, ranked, terms)
|
||||
|
||||
|
||||
async def _materialize(repo: KnowledgeRepository, ranked: list[tuple[str, tuple[float, dict[str, Any]]]], terms: list[str]) -> list[dict[str, Any]]:
|
||||
max_score = ranked[0][1][0] if ranked else 1.0
|
||||
results: list[dict[str, Any]] = []
|
||||
for nid, (score, meta) in ranked:
|
||||
note = meta if (meta and meta.get("content_md")) else await repo.get_note(nid, with_sources=False)
|
||||
if note is None:
|
||||
continue
|
||||
snippet_src = note.get("content_md") or meta.get("_chunk") or note.get("summary") or ""
|
||||
results.append(
|
||||
{
|
||||
"note_id": nid,
|
||||
"chunk_id": None,
|
||||
"title": note.get("title") or meta.get("title"),
|
||||
"summary": note.get("summary"),
|
||||
"snippet": _snippet(snippet_src, terms),
|
||||
"score": round(score / max_score, 4) if max_score else 0.0,
|
||||
"tags": note.get("tags") or [],
|
||||
"source_type": note.get("source_type"),
|
||||
"updated_at": note.get("updated_at"),
|
||||
"sources": [],
|
||||
}
|
||||
)
|
||||
return results
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Sensitive-information redaction for knowledge content (phase 3).
|
||||
|
||||
Best-effort masking of secrets / PII before a note is persisted: API keys,
|
||||
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
|
||||
ID-card numbers and email addresses. Conservative by design — it favours not
|
||||
mangling legitimate prose over catching every possible secret.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_MASK = "[REDACTED]"
|
||||
|
||||
# Order matters: more specific patterns first.
|
||||
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
|
||||
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
|
||||
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
|
||||
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
|
||||
# key/secret/password/token assignment: keep the field name, mask the value.
|
||||
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
|
||||
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
|
||||
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
|
||||
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
|
||||
]
|
||||
|
||||
|
||||
def redact(text: str | None) -> tuple[str, list[str]]:
|
||||
"""Return ``(redacted_text, hit_kinds)``.
|
||||
|
||||
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
|
||||
is falsy it is returned unchanged with no hits.
|
||||
"""
|
||||
if not text:
|
||||
return text or "", []
|
||||
hits: list[str] = []
|
||||
out = text
|
||||
for kind, pattern in _PATTERNS:
|
||||
def _sub(m: re.Match[str], _kind: str = kind) -> str:
|
||||
# For assignments keep the field name + delimiter, mask only the value.
|
||||
if _kind == "assignment":
|
||||
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
|
||||
sep = ":" if ":" in m.group(0) else "="
|
||||
return f"{head}{sep} {_MASK}"
|
||||
if _kind == "bearer":
|
||||
return f"Bearer {_MASK}"
|
||||
return _MASK
|
||||
|
||||
new_out, n = pattern.subn(_sub, out)
|
||||
if n:
|
||||
hits.append(kind)
|
||||
out = new_out
|
||||
return out, hits
|
||||
@@ -0,0 +1,689 @@
|
||||
"""KnowledgeService — high-level orchestration for the knowledge base.
|
||||
|
||||
Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
|
||||
(Markdown vault + index/log/hot/manifest + graph export) and the extractor
|
||||
(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
|
||||
service; everything obsidian-wiki specific stays inside the ``knowledge``
|
||||
module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.knowledge import extractor, llm_extractor
|
||||
from deerflow.knowledge.chunking import chunk_markdown
|
||||
from deerflow.knowledge.embeddings import EmbeddingClient
|
||||
from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target
|
||||
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
|
||||
from deerflow.knowledge.repository import KnowledgeRepository
|
||||
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
|
||||
from deerflow.knowledge.search import search_notes
|
||||
from deerflow.knowledge.sources import from_reference_batches, from_search_results
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EmptyThreadError(ValueError):
|
||||
"""Raised when a thread has no sediment-able content."""
|
||||
|
||||
|
||||
def _resolve_vault_dir(vault_dir: str) -> Path:
|
||||
path = Path(vault_dir)
|
||||
if path.is_absolute():
|
||||
return path.resolve()
|
||||
return (runtime_home() / vault_dir).resolve()
|
||||
|
||||
|
||||
def _inject_related_notes(body: str, links: list[str]) -> str:
|
||||
"""Insert real related-note ``[[wikilinks]]`` into the Related block.
|
||||
|
||||
Notes carry only content-specific related notes now (the structural meta-links
|
||||
were dropped), so this is the sole writer of ``## Related`` links. When the
|
||||
body has no ``## Related`` section yet, one is appended.
|
||||
"""
|
||||
if not links:
|
||||
return body
|
||||
bullets = "\n".join(f"- {link}" for link in links)
|
||||
marker = "## Related\n"
|
||||
idx = body.find(marker)
|
||||
if idx == -1:
|
||||
return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n"
|
||||
insert_at = idx + len(marker)
|
||||
return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:]
|
||||
|
||||
|
||||
class KnowledgeService:
|
||||
"""Application-facing knowledge base API."""
|
||||
|
||||
def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
|
||||
self.repo = repo
|
||||
self.config = config
|
||||
self.project_name = config.project_name
|
||||
vault_root = _resolve_vault_dir(config.vault_dir)
|
||||
self.adapter = ObsidianWikiAdapter(
|
||||
vault_root,
|
||||
project_name=config.project_name,
|
||||
export_html=config.export.graph_html_enabled,
|
||||
export_json=config.export.graph_json_enabled,
|
||||
)
|
||||
self.embedding_client = EmbeddingClient(config.embedding)
|
||||
try:
|
||||
self.adapter.ensure_ready()
|
||||
except Exception: # pragma: no cover - vault init best-effort
|
||||
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
|
||||
|
||||
# -- internal helpers -------------------------------------------------
|
||||
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
|
||||
"""Mask secrets/PII in a draft before persistence (phase 3)."""
|
||||
if not self.config.sensitive_redaction_enabled:
|
||||
return draft
|
||||
from deerflow.knowledge.sensitive import redact
|
||||
|
||||
draft.content_md, body_hits = redact(draft.content_md)
|
||||
draft.summary, _ = redact(draft.summary)
|
||||
for src in draft.sources:
|
||||
if src.snippet:
|
||||
src.snippet, _ = redact(src.snippet)
|
||||
if body_hits:
|
||||
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
|
||||
return draft
|
||||
|
||||
async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]:
|
||||
"""Find existing notes to cross-link under ## Related — by **shared tags**.
|
||||
|
||||
Matching is strictly tag-based: a candidate must share at least one of the
|
||||
draft's *distinctive* tags (the per-note project / ``knowledge-base``
|
||||
defaults are dropped, else everything matches everything). Free-text
|
||||
title/summary matching was removed because CJK tokenization made generic
|
||||
words (助手 / 知识图谱 / 页面) link unrelated notes. No distinctive tags →
|
||||
no related notes (better empty than noisy). Best-effort.
|
||||
"""
|
||||
if not self.config.related_notes_enabled:
|
||||
return []
|
||||
default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "zncm") if t}
|
||||
distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags]
|
||||
if not distinctive:
|
||||
return []
|
||||
own_title = (draft.title or "").strip().lower()
|
||||
# Rank candidates by how many distinctive tags they share with the draft.
|
||||
shared: dict[str, dict[str, Any]] = {}
|
||||
for tag in distinctive:
|
||||
try:
|
||||
items, _total = await self.repo.list_notes(tag=tag, limit=50, offset=0)
|
||||
except Exception:
|
||||
logger.exception("Related-notes tag lookup failed for tag '%s'", tag)
|
||||
continue
|
||||
for it in items:
|
||||
nid = it.get("id")
|
||||
title = (it.get("title") or "").strip()
|
||||
if not nid or not title or title.lower() == own_title:
|
||||
continue
|
||||
entry = shared.setdefault(nid, {"id": nid, "title": title, "count": 0})
|
||||
entry["count"] += 1
|
||||
ranked = sorted(shared.values(), key=lambda e: e["count"], reverse=True)
|
||||
limit = max(1, self.config.related_notes_limit)
|
||||
return [{"id": e["id"], "title": e["title"]} for e in ranked[:limit]]
|
||||
|
||||
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
|
||||
"""Create a DB row, write the vault file, attach sources, return note."""
|
||||
draft = self._redact_draft(draft)
|
||||
related_notes = await self._find_related_notes(draft)
|
||||
if related_notes:
|
||||
real_links = [f"[[{n['title']}]]" for n in related_notes]
|
||||
draft.content_md = _inject_related_notes(draft.content_md, real_links)
|
||||
draft.related = list(dict.fromkeys([*real_links, *draft.related]))
|
||||
note = await self.repo.create_note(
|
||||
title=draft.title,
|
||||
summary=draft.summary,
|
||||
content_md=draft.content_md,
|
||||
source_type=draft.source_type,
|
||||
source_id=draft.source_id,
|
||||
status=draft.status,
|
||||
confidence=draft.confidence,
|
||||
tags=draft.tags,
|
||||
created_by=created_by,
|
||||
)
|
||||
note_id = note["id"]
|
||||
try:
|
||||
rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
|
||||
await self.repo.set_vault_path(note_id, rel)
|
||||
note["vault_path"] = rel
|
||||
except Exception:
|
||||
logger.exception("Failed to write knowledge note %s to vault", note_id)
|
||||
if draft.sources:
|
||||
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
|
||||
await self._reindex_embeddings(note_id, draft.content_md)
|
||||
await self._store_entities(note_id, draft)
|
||||
# Record note→note "related" edges (after _store_entities, which clears
|
||||
# this note's relations) so the graph reflects the cross-links too.
|
||||
for rel_note in related_notes:
|
||||
try:
|
||||
await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related")
|
||||
except Exception:
|
||||
logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"])
|
||||
return await self.repo.get_note(note_id) or note
|
||||
|
||||
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
|
||||
"""Persist extracted entities/relations and link them to the note (phase 4)."""
|
||||
if not self.config.entity_extraction_enabled or not draft.entities:
|
||||
return
|
||||
try:
|
||||
await self.repo.clear_note_relations(note_id)
|
||||
name_to_id: dict[str, str] = {}
|
||||
for ent in draft.entities:
|
||||
name = str(ent.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
row = await self.repo.get_or_create_entity(name, ent.get("type"))
|
||||
name_to_id[name.lower()] = row["id"]
|
||||
# note -> entity "mentions"
|
||||
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
|
||||
for rel in draft.relations:
|
||||
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
|
||||
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
|
||||
if a and b and a != b:
|
||||
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
|
||||
except Exception:
|
||||
logger.exception("Failed to store entities/relations for note %s", note_id)
|
||||
|
||||
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
|
||||
"""Best-effort: (re)build the per-chunk embedding index for a note."""
|
||||
if not self.embedding_client.is_enabled():
|
||||
return
|
||||
try:
|
||||
chunks = chunk_markdown(content_md)
|
||||
if not chunks:
|
||||
await self.repo.delete_embeddings(note_id)
|
||||
return
|
||||
vectors = await self.embedding_client.embed_texts(chunks)
|
||||
if not vectors or len(vectors) != len(chunks):
|
||||
return
|
||||
await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
|
||||
except Exception:
|
||||
logger.exception("Failed to (re)index embeddings for note %s", note_id)
|
||||
|
||||
# -- capture ----------------------------------------------------------
|
||||
async def capture_thread(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
messages: list[dict[str, Any]],
|
||||
thread_title: str | None = None,
|
||||
mode: str = "summary",
|
||||
title: str | None = None,
|
||||
status: str = "approved",
|
||||
tags: list[str] | None = None,
|
||||
include_sources: bool = True,
|
||||
reference_batches: list[dict[str, Any]] | None = None,
|
||||
created_by: str | None = None,
|
||||
skip_existing: bool = False,
|
||||
model_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Sediment a thread into a knowledge note.
|
||||
|
||||
``model_name`` optionally overrides the configured extraction model for
|
||||
this one capture (falls back to ``config.extract_model_name``).
|
||||
|
||||
Returns ``{"note": <note>, "created": bool}``. Raises
|
||||
:class:`EmptyThreadError` when there is nothing worth keeping.
|
||||
"""
|
||||
# Always-on entry log: proves a capture was triggered for this thread,
|
||||
# regardless of how it ends (success / skip / fallback). Lets you tell
|
||||
# "capture never ran" apart from "logs not visible" via one grep.
|
||||
logger.info(
|
||||
"[knowledge] capturing thread %s (msgs=%d, llm_extract=%s, skip_existing=%s)",
|
||||
thread_id,
|
||||
len(messages or []),
|
||||
self.config.llm_extract_enabled,
|
||||
skip_existing,
|
||||
)
|
||||
if skip_existing:
|
||||
existing = await self.repo.find_by_source("thread", thread_id)
|
||||
if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
|
||||
existing = await self.repo.find_by_source("thread", thread_id)
|
||||
if existing:
|
||||
logger.info("[knowledge] thread %s already captured (note %s); skipping", thread_id, existing.get("id"))
|
||||
return {"note": existing, "created": False}
|
||||
|
||||
sources = []
|
||||
if include_sources and reference_batches:
|
||||
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
|
||||
|
||||
drafts: list[NoteDraft] = []
|
||||
# Prefer LLM distillation (wiki-capture style) so notes are declarative
|
||||
# knowledge, not raw chat. A single conversation may split into several
|
||||
# cross-linked notes. Falls back to the rule-based extractor (always one
|
||||
# note) when the LLM is disabled/unavailable or returns nothing usable.
|
||||
llm_attempted = self.config.llm_extract_enabled
|
||||
llm_fallback = False
|
||||
if llm_attempted:
|
||||
try:
|
||||
drafts = await llm_extractor.extract_thread_llm_multi(
|
||||
messages,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
mode=mode,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
sources=sources,
|
||||
max_messages=self.config.max_thread_messages,
|
||||
model_name=model_name or self.config.extract_model_name,
|
||||
max_input_chars=self.config.extract_max_input_chars,
|
||||
allow_multi=self.config.multi_note_enabled,
|
||||
)
|
||||
except llm_extractor.LLMRejectedIngest as exc:
|
||||
# The model judged this conversation worthless (greeting / meta
|
||||
# chatter). Skip ingestion entirely — do NOT dump the raw chat.
|
||||
raise EmptyThreadError(f"Thread {thread_id} judged not worth ingesting: {exc}") from exc
|
||||
if not drafts:
|
||||
# The LLM path was supposed to run but produced nothing usable
|
||||
# (API error / non-JSON). We will fall back to the rule-based
|
||||
# extractor, which yields a low-quality verbatim dump with no
|
||||
# entities/tags — flag it so it lands as a draft.
|
||||
llm_fallback = True
|
||||
if not drafts:
|
||||
single = extractor.extract_thread(
|
||||
messages,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
mode=mode,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
sources=sources,
|
||||
max_messages=self.config.max_thread_messages,
|
||||
)
|
||||
drafts = [single] if single else []
|
||||
if not drafts:
|
||||
logger.info("[knowledge] thread %s has no sediment-able content; skipping", thread_id)
|
||||
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
|
||||
|
||||
# When the LLM was enabled but silently fell back to rule-based extraction,
|
||||
# the note is a verbatim dump (no distillation / entities / tags). Land it
|
||||
# as a *draft* for review instead of silently approving it, and log loudly
|
||||
# so the extraction failure is diagnosable (see also the WARNING/EXCEPTION
|
||||
# logged inside llm_extractor.extract_thread_llm_multi).
|
||||
effective_status = status
|
||||
if llm_fallback:
|
||||
logger.warning(
|
||||
"[knowledge] thread %s — capture fell back to rule-based extraction "
|
||||
"(LLM distillation produced nothing usable); landing as DRAFT. "
|
||||
"Check extraction model '%s' / earlier [knowledge] logs for the cause.",
|
||||
thread_id,
|
||||
self.config.extract_model_name or "models[0]",
|
||||
)
|
||||
if normalize_status(status) == "approved":
|
||||
effective_status = "draft"
|
||||
|
||||
normalized = normalize_status(effective_status)
|
||||
for draft in drafts:
|
||||
draft.status = normalized
|
||||
# Cross-link sibling notes from the same conversation by title, so the
|
||||
# split pages reference each other (resolved once all are persisted).
|
||||
if len(drafts) > 1:
|
||||
for i, draft in enumerate(drafts):
|
||||
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
|
||||
if siblings:
|
||||
draft.content_md = _inject_related_notes(draft.content_md, siblings)
|
||||
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
|
||||
|
||||
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
|
||||
logger.info(
|
||||
"[knowledge] thread %s captured → %d note(s) [%s], status=%s, llm_fallback=%s",
|
||||
thread_id,
|
||||
len(notes),
|
||||
", ".join(n.get("title", "") for n in notes),
|
||||
normalized,
|
||||
llm_fallback,
|
||||
)
|
||||
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
|
||||
|
||||
async def capture_search(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
results: list[dict[str, Any]] | None,
|
||||
thread_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
created_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Sediment search/tool results into a reference note."""
|
||||
sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
|
||||
draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
|
||||
draft.status = normalize_status(status)
|
||||
return await self._persist_draft(draft, created_by=created_by)
|
||||
|
||||
async def create_manual_note(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
content_md: str,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
summary: str | None = None,
|
||||
created_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a note directly from product-edited Markdown."""
|
||||
# Strip any frontmatter the user may have pasted; vault writer adds its own.
|
||||
_, body = MarkdownVault.parse(content_md)
|
||||
body = body or content_md
|
||||
base_tags = list(tags or [])
|
||||
for default in (self.project_name, "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
draft = NoteDraft(
|
||||
title=title.strip() or "知识笔记",
|
||||
summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
|
||||
content_md=body,
|
||||
category="synthesis",
|
||||
source_type="manual",
|
||||
source_id=None,
|
||||
status=normalize_status(status),
|
||||
tags=base_tags,
|
||||
confidence=0.5,
|
||||
related=extractor.related_links(self.project_name),
|
||||
)
|
||||
return await self._persist_draft(draft, created_by=created_by)
|
||||
|
||||
# -- CRUD -------------------------------------------------------------
|
||||
async def list_notes(
|
||||
self,
|
||||
*,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
items, total = await self.repo.list_notes(
|
||||
keyword=keyword,
|
||||
tag=tag,
|
||||
source_type=normalize_source_type(source_type) if source_type else None,
|
||||
status=status,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return {"items": items, "total": total}
|
||||
|
||||
async def get_note(self, note_id: str) -> dict[str, Any] | None:
|
||||
note = await self.repo.get_note(note_id)
|
||||
if note is not None:
|
||||
note["entities"] = await self.repo.list_note_entities(note_id)
|
||||
return note
|
||||
|
||||
async def update_note(
|
||||
self,
|
||||
note_id: str,
|
||||
*,
|
||||
title: str | None = None,
|
||||
content_md: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str | None = None,
|
||||
summary: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
current = await self.repo.get_note(note_id, with_sources=False)
|
||||
if current is None:
|
||||
return None
|
||||
# Snapshot the pre-edit state for version history (phase 3).
|
||||
try:
|
||||
await self.repo.add_version(current, edited_by=updated_by)
|
||||
except Exception:
|
||||
logger.exception("Failed to snapshot version for note %s", note_id)
|
||||
body = content_md
|
||||
if body is not None:
|
||||
_, parsed = MarkdownVault.parse(body)
|
||||
body = parsed or body
|
||||
note = await self.repo.update_note(
|
||||
note_id,
|
||||
title=title,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
status=normalize_status(status) if status else None,
|
||||
tags=tags,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
if note is None:
|
||||
return None
|
||||
try:
|
||||
rel = self.adapter.update_note_file(
|
||||
note_id,
|
||||
current.get("vault_path"),
|
||||
title=note["title"],
|
||||
content_md=note["content_md"],
|
||||
tags=note.get("tags") or [],
|
||||
status=note["status"],
|
||||
summary=note.get("summary") or "",
|
||||
category="synthesis",
|
||||
created=note["created_at"],
|
||||
updated=note["updated_at"],
|
||||
)
|
||||
if rel and rel != current.get("vault_path"):
|
||||
await self.repo.set_vault_path(note_id, rel)
|
||||
except Exception:
|
||||
logger.exception("Failed to update knowledge note %s in vault", note_id)
|
||||
if note.get("content_md") is not None:
|
||||
await self._reindex_embeddings(note_id, note["content_md"])
|
||||
return await self.repo.get_note(note_id)
|
||||
|
||||
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
|
||||
return await self.repo.list_versions(note_id)
|
||||
|
||||
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
|
||||
"""Soft delete (archive) by default; hard delete removes the vault file."""
|
||||
current = await self.repo.get_note(note_id, with_sources=False)
|
||||
if current is None:
|
||||
return False
|
||||
note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
|
||||
try:
|
||||
self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
|
||||
if hard and current.get("vault_path"):
|
||||
self.adapter.vault.delete_note(current["vault_path"])
|
||||
self.adapter.manifest.remove_page(note_id)
|
||||
await self.repo.delete_embeddings(note_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to archive knowledge note %s in vault", note_id)
|
||||
# Drop this note's entity mentions + any now-orphaned entities so the graph
|
||||
# follows the deletion (no lingering nodes from removed notes).
|
||||
try:
|
||||
await self.repo.clear_note_relations(note_id)
|
||||
pruned = await self.repo.prune_orphan_entities()
|
||||
if pruned:
|
||||
logger.info("[knowledge] deleted note %s → pruned %d orphan entities", note_id, pruned)
|
||||
except Exception:
|
||||
logger.exception("Failed to prune relations/entities for deleted note %s", note_id)
|
||||
return True
|
||||
|
||||
# -- wikilinks --------------------------------------------------------
|
||||
async def resolve_wikilink(self, target: str) -> dict[str, Any]:
|
||||
"""Resolve a single ``[[wikilink]]`` target to a navigable destination.
|
||||
|
||||
Returns one of:
|
||||
- ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note
|
||||
- ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page
|
||||
- ``{"kind": "missing", "exists": False}`` — nothing matches
|
||||
|
||||
Resolution order: note id → note vault_path → note title → vault file.
|
||||
"""
|
||||
norm = normalize_wikilink_target(target)
|
||||
base = {"target": target, "normalized": norm}
|
||||
if not norm:
|
||||
return {**base, "kind": "missing", "exists": False}
|
||||
|
||||
# 1) Direct note id (e.g. "kb_abc...").
|
||||
if norm.startswith("kb_"):
|
||||
note = await self.repo.get_note(norm, with_sources=False)
|
||||
if note and note.get("status") != "archived":
|
||||
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
|
||||
|
||||
# 2) Note by vault path (links from index.md / hot.md / structural hub).
|
||||
note = await self.repo.find_by_vault_path(norm)
|
||||
if note:
|
||||
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
|
||||
|
||||
# 3) Note by title (the real related-note cross-links use [[Title]]).
|
||||
note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1])
|
||||
if note:
|
||||
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
|
||||
|
||||
# 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…).
|
||||
page = self.read_vault_page(norm)
|
||||
if page is not None:
|
||||
return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True}
|
||||
|
||||
return {**base, "kind": "missing", "exists": False}
|
||||
|
||||
async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""Resolve many targets at once; returns ``{original_target: resolution}``."""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for target in dict.fromkeys(t for t in targets if t and t.strip()):
|
||||
out[target] = await self.resolve_wikilink(target)
|
||||
return out
|
||||
|
||||
async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None:
|
||||
"""Extract every ``[[wikilink]]`` in a note's body and resolve them.
|
||||
|
||||
Returns ``{target: resolution}`` or ``None`` when the note is missing.
|
||||
Numeric ``[[123]]`` citation badges are skipped (handled by the renderer).
|
||||
"""
|
||||
note = await self.repo.get_note(note_id, with_sources=False)
|
||||
if note is None:
|
||||
return None
|
||||
targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()]
|
||||
return await self.resolve_wikilinks(targets)
|
||||
|
||||
def read_vault_page(self, path: str) -> dict[str, Any] | None:
|
||||
"""Read a raw vault Markdown page (meta/hub pages without a DB note).
|
||||
|
||||
Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does
|
||||
not exist or the path escapes the vault root.
|
||||
"""
|
||||
norm = normalize_wikilink_target(path)
|
||||
if not norm:
|
||||
return None
|
||||
rel = f"{norm}.md"
|
||||
try:
|
||||
parsed = self.adapter.vault.read_note(rel)
|
||||
except VaultPathError:
|
||||
return None
|
||||
if parsed is None:
|
||||
return None
|
||||
fm, body = parsed
|
||||
title = str(fm.get("title") or "").strip() or norm.split("/")[-1]
|
||||
return {"path": norm, "title": title, "content_md": body}
|
||||
|
||||
# -- search & graph ---------------------------------------------------
|
||||
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
|
||||
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
|
||||
|
||||
async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
|
||||
"""Retrieve top knowledge for ``query`` and format an injectable block.
|
||||
|
||||
Returns ``{"text": <system context or "">, "items": [...]}``. Uses hybrid
|
||||
search when embeddings are available, else keyword (wiki-query style).
|
||||
"""
|
||||
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
|
||||
items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
|
||||
if not items:
|
||||
return {"text": "", "items": []}
|
||||
lines = [
|
||||
"以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
|
||||
"",
|
||||
]
|
||||
for i, it in enumerate(items, 1):
|
||||
lines.append(f"[知识 {i}] {it.get('title') or ''}")
|
||||
if it.get("snippet"):
|
||||
lines.append(it["snippet"])
|
||||
lines.append(f"来源:{it['note_id']}")
|
||||
lines.append("")
|
||||
return {"text": "\n".join(lines).strip(), "items": items}
|
||||
|
||||
# -- dedup & merge (phase 3) -----------------------------------------
|
||||
async def find_duplicates(self) -> list[dict[str, Any]]:
|
||||
from deerflow.knowledge.dedup import find_duplicate_groups
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
|
||||
|
||||
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
|
||||
"""Merge ``duplicate_ids`` into ``primary_id``.
|
||||
|
||||
Sources from each duplicate are re-attached to the primary, a wikilink
|
||||
+ "Merged from" note is appended to the primary body, the duplicates are
|
||||
archived (soft delete), and a ``merged_into`` relation is recorded.
|
||||
"""
|
||||
primary = await self.repo.get_note(primary_id)
|
||||
if primary is None:
|
||||
return None
|
||||
merged_titles: list[str] = []
|
||||
for dup_id in duplicate_ids:
|
||||
if dup_id == primary_id:
|
||||
continue
|
||||
dup = await self.repo.get_note(dup_id)
|
||||
if dup is None:
|
||||
continue
|
||||
# Move sources.
|
||||
for src in dup.get("sources") or []:
|
||||
await self.repo.add_sources(
|
||||
primary_id,
|
||||
[
|
||||
SourceDraft(
|
||||
source_type=src.get("source_type") or "tool",
|
||||
thread_id=src.get("thread_id"),
|
||||
message_id=src.get("message_id"),
|
||||
tool_name=src.get("tool_name"),
|
||||
title=src.get("title"),
|
||||
url=src.get("url"),
|
||||
snippet=src.get("snippet"),
|
||||
)
|
||||
],
|
||||
created_by=updated_by,
|
||||
)
|
||||
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
|
||||
await self.delete_note(dup_id, updated_by=updated_by)
|
||||
merged_titles.append(dup.get("title") or dup_id)
|
||||
|
||||
if merged_titles:
|
||||
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
|
||||
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
|
||||
return await self.repo.get_note(primary_id)
|
||||
|
||||
async def reindex_all(self) -> dict[str, int]:
|
||||
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
|
||||
notes = await self.repo.list_all()
|
||||
count = 0
|
||||
for note in notes:
|
||||
await self._reindex_embeddings(note["id"], note.get("content_md") or "")
|
||||
count += 1
|
||||
return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
|
||||
|
||||
async def export_graph(self) -> dict[str, Any]:
|
||||
from deerflow.utils.time import now_iso
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
|
||||
return {"files": result["files"], "stats": result["stats"]}
|
||||
|
||||
async def get_graph(self) -> dict[str, Any]:
|
||||
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
|
||||
from deerflow.knowledge import exporter
|
||||
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
return exporter.build_graph(notes, entities, relations)
|
||||
|
||||
def export_file_path(self, filename: str) -> Path | None:
|
||||
return self.adapter.export_file_path(filename)
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""Built-in obsidian-wiki skill templates.
|
||||
|
||||
Mirrors the key intent of ``ar9av/obsidian-wiki`` skills so the backend can
|
||||
write vault content that stays compatible with the original tooling without
|
||||
shelling out to its CLI:
|
||||
|
||||
- ``wiki-capture`` — how a conversation becomes a *declarative* knowledge note
|
||||
(frontmatter schema + Context/Finding/Reasoning/Implications/Related body).
|
||||
- ``wiki-query`` — layered retrieval (hot.md → index.md → frontmatter → body).
|
||||
- ``wiki-export`` — graph artifacts (graph.json / graph.html / graphml / cypher).
|
||||
|
||||
These are intentionally lightweight: the structures here are what
|
||||
:mod:`deerflow.knowledge.markdown_vault` and :mod:`deerflow.knowledge.extractor`
|
||||
produce, and what the first-release LLM prompt (future enhancement) should
|
||||
target.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Category → vault sub-directory (wiki-capture page taxonomy, §8 of the dev doc).
|
||||
CATEGORY_DIRS = {
|
||||
"synthesis": "synthesis",
|
||||
"concepts": "concepts",
|
||||
"concept": "concepts",
|
||||
"references": "references",
|
||||
"reference": "references",
|
||||
"source": "references",
|
||||
"decision": "synthesis",
|
||||
"journal": "journal",
|
||||
"session": "journal",
|
||||
}
|
||||
|
||||
# Body section headings (wiki-capture). Declarative knowledge, never a chat log.
|
||||
BODY_SECTIONS = ("Context", "Finding / Decision", "Reasoning", "Implications", "Related")
|
||||
|
||||
# Inline provenance markers reused from wiki-capture.
|
||||
INFERRED_MARKER = "^[inferred]"
|
||||
AMBIGUOUS_MARKER = "^[ambiguous]"
|
||||
|
||||
WIKI_CAPTURE_GUIDANCE = """\
|
||||
Capture *declarative* knowledge, not a chat transcript. Classify the note as one
|
||||
of: synthesis, concept, source, decision, session. Prefer at least two
|
||||
[[wikilinks]]. Mark inferred content with ^[inferred] and uncertain/conflicting
|
||||
content with ^[ambiguous]. Project-specific knowledge belongs under
|
||||
projects/<project-name>/...
|
||||
"""
|
||||
|
||||
|
||||
def category_to_dir(category: str | None) -> str:
|
||||
"""Map a wiki-capture category to its vault sub-directory."""
|
||||
return CATEGORY_DIRS.get((category or "").strip().lower(), "synthesis")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Normalize knowledge sources from threads / searches / tool results.
|
||||
|
||||
Source items come from a few shapes: backend reference-batches (built by
|
||||
``deerflow.runtime.references``), raw search-result payloads, and tool-call
|
||||
messages. They are all normalized into :class:`SourceDraft` so the repository
|
||||
and frontend see one consistent structure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from deerflow.knowledge.schemas import SourceDraft
|
||||
|
||||
|
||||
def _clip(value: Any, limit: int = 2000) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
|
||||
"""Convert backend reference-batches into :class:`SourceDraft`."""
|
||||
out: list[SourceDraft] = []
|
||||
seen: set[str] = set()
|
||||
for batch in batches or []:
|
||||
msg_id = batch.get("assistant_message_id") or batch.get("id")
|
||||
for src in batch.get("sources", []) or []:
|
||||
if not isinstance(src, dict):
|
||||
continue
|
||||
url = _clip(src.get("url"), 1024)
|
||||
title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
|
||||
dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
|
||||
if dedup_key in seen:
|
||||
continue
|
||||
seen.add(dedup_key)
|
||||
out.append(
|
||||
SourceDraft(
|
||||
source_type="search" if url else "tool",
|
||||
thread_id=thread_id,
|
||||
message_id=_clip(msg_id, 128),
|
||||
tool_name=_clip(src.get("skillName"), 128),
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=_clip(src.get("snippet")),
|
||||
raw=None,
|
||||
)
|
||||
)
|
||||
if len(out) >= max_items:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
def from_search_results(
|
||||
query: str,
|
||||
results: list[dict[str, Any]] | None,
|
||||
*,
|
||||
thread_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
max_items: int = 20,
|
||||
) -> list[SourceDraft]:
|
||||
"""Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
|
||||
out: list[SourceDraft] = []
|
||||
for item in (results or [])[:max_items]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
out.append(
|
||||
SourceDraft(
|
||||
source_type="search",
|
||||
thread_id=thread_id,
|
||||
message_id=message_id,
|
||||
tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
|
||||
title=_clip(item.get("title"), 512) or query[:200],
|
||||
url=_clip(item.get("url"), 1024),
|
||||
snippet=_clip(item.get("snippet") or item.get("content")),
|
||||
raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
|
||||
)
|
||||
)
|
||||
return out
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
"""Pure scheduling helpers for the admin leaderboard daily snapshots.
|
||||
|
||||
Kept import-light on purpose (only stdlib ``datetime``) so the decision logic
|
||||
is unit-testable without standing up the FastAPI app, the DB engine, or the
|
||||
ORM. The background scheduler in ``app/gateway/app.py`` wires this to the real
|
||||
snapshot rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def should_finalize_previous_day(
|
||||
*,
|
||||
status: str | None,
|
||||
generated_at: datetime | None,
|
||||
now: datetime,
|
||||
finalize_hour: int,
|
||||
) -> bool:
|
||||
"""Decide whether *yesterday's* snapshot should be force-recomputed once.
|
||||
|
||||
The previous day's snapshot is normally built before midnight, so it can
|
||||
miss runs / tool calls whose rows only land in the database shortly before
|
||||
or after the day boundary. Once per day, on/after ``finalize_hour`` (local
|
||||
Beijing time), we recompute it a single time so the full day is captured.
|
||||
|
||||
``now`` and ``generated_at`` are expected to be timezone-aware Beijing
|
||||
datetimes (``generated_at`` may be ``None`` for legacy rows).
|
||||
|
||||
Returns ``True`` only when all hold:
|
||||
- it is already on/after ``finalize_hour`` today;
|
||||
- a snapshot row exists (``status`` is not ``None`` — a missing day is
|
||||
left to the normal prewarm/queue path);
|
||||
- that snapshot is not already ``queued``/``running`` (a finalize already
|
||||
in flight, or normal processing);
|
||||
- it was generated before today (i.e. not yet finalized for this day).
|
||||
"""
|
||||
if status is None:
|
||||
return False
|
||||
if now.hour < finalize_hour:
|
||||
return False
|
||||
if status in ("queued", "running"):
|
||||
return False
|
||||
if generated_at is not None and generated_at.date() >= now.date():
|
||||
return False
|
||||
return True
|
||||
+45
-42
@@ -119,18 +119,19 @@ class AgentStore(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return one page of agents plus the total matching count.
|
||||
|
||||
``scope``:
|
||||
@@ -142,13 +143,15 @@ class AgentStore(ABC):
|
||||
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
|
||||
|
||||
``search`` matches the raw name OR its desensitized display form.
|
||||
``tag_ids`` keeps agents carrying ANY of the tags (OR). Both filters,
|
||||
the COUNT, and LIMIT/OFFSET all execute in SQL.
|
||||
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
|
||||
only agents published by that user (``user_id`` equality) — built-in
|
||||
agents (no owner) never match. All filters, the COUNT, and LIMIT/OFFSET
|
||||
all execute in SQL.
|
||||
|
||||
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
|
||||
for the 1-based ``page``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
for the 1-based ``page``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Listing cache (skills / tool_groups / model) ---
|
||||
|
||||
@@ -199,28 +202,28 @@ class AgentStore(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Per-user pins ("置顶") ---
|
||||
|
||||
@abstractmethod
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
"""Remove a per-user pin. Returns ``False`` when none existed."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
"""Return the set of agent ids pinned by ``user_id``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
|
||||
raise NotImplementedError
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# --- Per-user pins ("置顶") ---
|
||||
|
||||
@abstractmethod
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
"""Remove a per-user pin. Returns ``False`` when none existed."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
"""Return the set of agent ids pinned by ``user_id``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
|
||||
raise NotImplementedError
|
||||
+62
-59
@@ -21,9 +21,9 @@ def _copy(data: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
class MemoryAgentStore(AgentStore):
|
||||
def __init__(self) -> None:
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
self._agents: dict[str, dict[str, Any]] = {}
|
||||
self._favorites: set[tuple[str, str]] = set()
|
||||
self._pins: set[tuple[str, str]] = set()
|
||||
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
|
||||
self._skills: dict[str, list[str]] = {}
|
||||
self._extras: dict[str, dict[str, Any]] = {}
|
||||
@@ -186,51 +186,54 @@ class MemoryAgentStore(AgentStore):
|
||||
return bool(agent.get("is_private"))
|
||||
return self._is_visible(agent, user_id)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
elif sort == "updated_desc":
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "updated_asc":
|
||||
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
|
||||
elif sort == "name_asc":
|
||||
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
|
||||
elif sort == "name_desc":
|
||||
matched.sort(
|
||||
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
matched.sort(
|
||||
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
|
||||
total = len(matched)
|
||||
start = max(page - 1, 0) * page_size
|
||||
page_rows = matched[start : start + page_size]
|
||||
return [_copy(a) for a in page_rows], total
|
||||
|
||||
@@ -275,23 +278,23 @@ class MemoryAgentStore(AgentStore):
|
||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
self._pins.add((user_id, agent_id))
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
key = (user_id, agent_id)
|
||||
if key not in self._pins:
|
||||
return False
|
||||
self._pins.remove(key)
|
||||
return True
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
return {agent_id for uid, agent_id in self._pins if uid == user_id}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
|
||||
+90
-86
@@ -6,19 +6,19 @@ from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.config.agent_name_desensitize import source_terms_for_needle
|
||||
from deerflow.persistence.agents.base import AgentStore
|
||||
from deerflow.persistence.agents.model import (
|
||||
AgentExtrasRow,
|
||||
AgentFavoriteRow,
|
||||
AgentPinRow,
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
AgentExtrasRow,
|
||||
AgentFavoriteRow,
|
||||
AgentPinRow,
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow
|
||||
|
||||
|
||||
@@ -278,18 +278,19 @@ class AgentRepository(AgentStore):
|
||||
)
|
||||
return AgentRow.id.in_(subq)
|
||||
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
async def list_paginated(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
scope: str | None,
|
||||
search: str | None,
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
tag_ids: list[str] | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
filters = []
|
||||
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
|
||||
if scope_filter is not None:
|
||||
@@ -298,30 +299,33 @@ class AgentRepository(AgentStore):
|
||||
filters.append(self._search_clause(search))
|
||||
if tag_ids:
|
||||
filters.append(self._tag_clause(tag_ids))
|
||||
if owner and owner.strip():
|
||||
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
|
||||
filters.append(AgentRow.user_id == owner.strip())
|
||||
|
||||
count_stmt = select(func.count()).select_from(AgentRow)
|
||||
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
|
||||
|
||||
if sort == "time_asc":
|
||||
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
|
||||
elif sort == "updated_desc":
|
||||
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
|
||||
elif sort == "updated_asc":
|
||||
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
|
||||
elif sort == "name_asc":
|
||||
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
|
||||
elif sort == "name_desc":
|
||||
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
|
||||
else:
|
||||
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
|
||||
|
||||
page_stmt = (
|
||||
select(AgentRow)
|
||||
.order_by(pin_rank.asc(), *order_by)
|
||||
.limit(page_size)
|
||||
.offset(max(page - 1, 0) * page_size)
|
||||
)
|
||||
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
|
||||
|
||||
if sort == "time_asc":
|
||||
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
|
||||
elif sort == "updated_desc":
|
||||
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
|
||||
elif sort == "updated_asc":
|
||||
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
|
||||
elif sort == "name_asc":
|
||||
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
|
||||
elif sort == "name_desc":
|
||||
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
|
||||
else:
|
||||
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
|
||||
|
||||
page_stmt = (
|
||||
select(AgentRow)
|
||||
.order_by(pin_rank.asc(), *order_by)
|
||||
.limit(page_size)
|
||||
.offset(max(page - 1, 0) * page_size)
|
||||
)
|
||||
if filters:
|
||||
where = and_(*filters)
|
||||
count_stmt = count_stmt.where(where)
|
||||
@@ -441,49 +445,49 @@ class AgentRepository(AgentStore):
|
||||
)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(AgentPinRow.id)
|
||||
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return
|
||||
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(AgentPinRow).where(
|
||||
AgentPinRow.user_id == user_id,
|
||||
AgentPinRow.agent_id == agent_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
async def delete_favorites_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
|
||||
# --- Pins ("置顶") ---
|
||||
|
||||
async def add_pin(self, user_id: str, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(AgentPinRow.id)
|
||||
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return
|
||||
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
|
||||
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(AgentPinRow).where(
|
||||
AgentPinRow.user_id == user_id,
|
||||
AgentPinRow.agent_id == agent_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def list_pin_ids(self, user_id: str) -> set[str]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
)
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
async def delete_pins_for_agent(self, agent_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Add name_zh column to skills table.
|
||||
|
||||
Revision ID: 20260609_01
|
||||
Revises: 20260608_01
|
||||
Create Date: 2026-06-09
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260609_01"
|
||||
down_revision = "20260608_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ``skills`` may not exist yet on a brand-new DB — create_all handles that
|
||||
# case with the column already present, so only patch existing tables.
|
||||
if _has_table("skills") and not _has_column("skills", "name_zh"):
|
||||
op.add_column("skills", sa.Column("name_zh", sa.String(length=128), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("skills", "name_zh"):
|
||||
op.drop_column("skills", "name_zh")
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Merge the skill_name_zh and chain-stages migration heads.
|
||||
|
||||
Two lineages both descend from ``20260608_01`` (admin_leaderboard_daily_stats):
|
||||
``20260608_02`` … ``20260608_06`` (the chain-stages / parallel-agent branch) and
|
||||
``20260609_01`` (skills.name_zh), which branched directly off ``20260608_01``
|
||||
instead of off the latest head. That left the graph with two heads and made
|
||||
``alembic upgrade head`` fail with "Multiple head revisions are present". This
|
||||
no-op merge revision joins both lineages so ``upgrade head`` resolves to a single
|
||||
head again.
|
||||
|
||||
Revision ID: 20260609_02
|
||||
Revises: 20260608_06, 20260609_01
|
||||
Create Date: 2026-06-10
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
revision = "20260609_02"
|
||||
down_revision = ("20260608_06", "20260609_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+18
@@ -25,6 +25,16 @@ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.knowledge.models import (
|
||||
KnowledgeEmbeddingRow,
|
||||
KnowledgeEntityRow,
|
||||
KnowledgeNoteRow,
|
||||
KnowledgeNoteTagRow,
|
||||
KnowledgeNoteVersionRow,
|
||||
KnowledgeRelationRow,
|
||||
KnowledgeSourceRow,
|
||||
KnowledgeTagRow,
|
||||
)
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
@@ -63,6 +73,14 @@ __all__ = [
|
||||
"AgentSkillRow",
|
||||
"AdminLeaderboardDailyStatRow",
|
||||
"BusinessMappingRow",
|
||||
"KnowledgeEmbeddingRow",
|
||||
"KnowledgeEntityRow",
|
||||
"KnowledgeNoteRow",
|
||||
"KnowledgeNoteTagRow",
|
||||
"KnowledgeNoteVersionRow",
|
||||
"KnowledgeRelationRow",
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"LlmCallMetricRow",
|
||||
"NotificationRow",
|
||||
"ParallelAgentTaskRow",
|
||||
|
||||
+6
@@ -70,6 +70,7 @@ class MemorySkillStore(SkillStore):
|
||||
"owner_user_id": data.get("owner_user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"detail": data.get("detail"),
|
||||
"name_zh": data.get("name_zh"),
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -85,6 +86,7 @@ class MemorySkillStore(SkillStore):
|
||||
"owner_user_id": data.get("owner_user_id"),
|
||||
"published": bool(data.get("published", True)),
|
||||
"detail": data.get("detail"),
|
||||
"name_zh": data.get("name_zh"),
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -102,6 +104,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["published"] = bool(data["published"])
|
||||
if "detail" in data:
|
||||
record["detail"] = data["detail"]
|
||||
if "name_zh" in data:
|
||||
record["name_zh"] = data["name_zh"]
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
@@ -115,6 +119,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["owner_user_id"] = data["owner_user_id"]
|
||||
if "detail" in data:
|
||||
record["detail"] = data["detail"]
|
||||
if "name_zh" in data:
|
||||
record["name_zh"] = data["name_zh"]
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@@ -29,6 +29,10 @@ class SkillRow(Base):
|
||||
# kept in the DB so admins can annotate built-in skills without rewriting
|
||||
# the shared on-disk SKILL.md file.
|
||||
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
# User-configured or LLM-generated Chinese display name. Kept in the DB (not
|
||||
# the on-disk SKILL.md) so it can be set for built-in skills too, and so the
|
||||
# shared source file is never rewritten.
|
||||
name_zh: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
@@ -64,6 +64,7 @@ class SkillRepository(SkillStore):
|
||||
owner_user_id=data.get("owner_user_id"),
|
||||
published=bool(data.get("published", False)),
|
||||
detail=data.get("detail"),
|
||||
name_zh=data.get("name_zh"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -83,6 +84,7 @@ class SkillRepository(SkillStore):
|
||||
owner_user_id=data.get("owner_user_id"),
|
||||
published=bool(data.get("published", True)),
|
||||
detail=data.get("detail"),
|
||||
name_zh=data.get("name_zh"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -105,6 +107,8 @@ class SkillRepository(SkillStore):
|
||||
row.published = bool(data["published"])
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
@@ -121,6 +125,8 @@ class SkillRepository(SkillStore):
|
||||
row.owner_user_id = data["owner_user_id"]
|
||||
if "detail" in data:
|
||||
row.detail = data["detail"]
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
+302
-295
@@ -1,295 +1,302 @@
|
||||
"""Async checkpointer factory.
|
||||
|
||||
Provides an **async context manager** for long-running async servers that need
|
||||
proper resource cleanup.
|
||||
|
||||
Supported backends: memory, sqlite, postgres.
|
||||
|
||||
Usage (e.g. FastAPI lifespan)::
|
||||
|
||||
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
|
||||
|
||||
async with make_checkpointer() as checkpointer:
|
||||
app.state.checkpointer = checkpointer # InMemorySaver if not configured
|
||||
|
||||
For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from langgraph.types import Checkpointer
|
||||
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.runtime.checkpointer.provider import (
|
||||
POSTGRES_CONN_REQUIRED,
|
||||
POSTGRES_INSTALL,
|
||||
SQLITE_INSTALL,
|
||||
)
|
||||
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _csv_env(name: str) -> list[str]:
|
||||
raw = os.getenv(name, "")
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def _dual_read_enabled(config) -> bool:
|
||||
if getattr(config, "dual_read_enabled", False):
|
||||
return True
|
||||
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _use_postgres_enabled(config) -> bool:
|
||||
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
|
||||
if raw:
|
||||
return raw.strip().lower() not in {"0", "false", "no", "off"}
|
||||
return bool(getattr(config, "use_postgres", True))
|
||||
|
||||
|
||||
def _dual_read_paths(config) -> list[str]:
|
||||
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
|
||||
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
conn_str = resolve_sqlite_conn_str(path)
|
||||
if conn_str == ":memory:" or conn_str.startswith("file:"):
|
||||
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
|
||||
continue
|
||||
if conn_str not in seen:
|
||||
resolved.append(conn_str)
|
||||
seen.add(conn_str)
|
||||
return resolved
|
||||
|
||||
|
||||
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
|
||||
configured = getattr(config, "sqlite_connection_string", None)
|
||||
if configured:
|
||||
return configured
|
||||
paths = _dual_read_paths(config)
|
||||
return paths[0] if paths else "store.db"
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _pg_pool_saver(conn_str: str) -> AsyncIterator[Checkpointer]:
|
||||
"""Pool-backed ``AsyncPostgresSaver``, resilient to idle-closed connections.
|
||||
|
||||
``AsyncPostgresSaver.from_conn_string`` opens **one** long-lived connection.
|
||||
对远程 Postgres(本项目 47.88.25.99 这类),空闲几分钟后连接会被服务端 / NAT /
|
||||
连接池中间件断开,而 psycopg **不会自动重连** —— 下一次 checkpoint 操作就抛
|
||||
``OperationalError: the connection is closed``,整个 run 直接失败(实测:圆桌第二步
|
||||
完成后闲置 ~6 分钟,用户再人工干预 → leader run 读 checkpoint 时撞死连接 → 前端
|
||||
"(无文本输出)")。
|
||||
|
||||
用带健康检查的连接池根治:``check=AsyncConnectionPool.check_connection`` 在把连接交给
|
||||
调用方前先验活,坏连接直接丢弃重建;``max_idle`` 主动回收久闲连接,赶在远程 idle 超时
|
||||
之前。这样上层永远拿不到失效连接。``prepare_threshold=0`` 关预备语句,兼容 pgbouncer
|
||||
一类连接池(8432 非标准端口很可能就是)。
|
||||
"""
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(POSTGRES_INSTALL) from exc
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
try:
|
||||
max_size = int(os.getenv("DEER_FLOW_PG_POOL_MAX", "20"))
|
||||
except ValueError:
|
||||
max_size = 20
|
||||
|
||||
pool = AsyncConnectionPool(
|
||||
conninfo=conn_str,
|
||||
min_size=1,
|
||||
max_size=max_size,
|
||||
open=False,
|
||||
kwargs={"autocommit": True, "prepare_threshold": 0},
|
||||
check=AsyncConnectionPool.check_connection,
|
||||
max_idle=120.0,
|
||||
max_lifetime=1800.0,
|
||||
)
|
||||
await pool.open(wait=True)
|
||||
try:
|
||||
saver = AsyncPostgresSaver(pool)
|
||||
await saver.setup()
|
||||
logger.info("Checkpointer: Postgres connection pool ready (max_size=%d, health-checked)", max_size)
|
||||
yield saver
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
|
||||
try:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(SQLITE_INSTALL) from exc
|
||||
|
||||
conn_str = resolve_sqlite_conn_str(raw)
|
||||
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
|
||||
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
|
||||
"""Async context manager that constructs and tears down a checkpointer."""
|
||||
if config.type == "memory":
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
yield InMemorySaver()
|
||||
return
|
||||
|
||||
if config.type == "sqlite":
|
||||
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
if config.type == "postgres":
|
||||
if not _use_postgres_enabled(config):
|
||||
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
|
||||
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
|
||||
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
# 连接池 + 健康检查,根治远程 Postgres 单连接空闲被关导致的 run 崩溃(见 _pg_pool_saver)。
|
||||
async with _pg_pool_saver(config.connection_string) as saver:
|
||||
if not _dual_read_enabled(config):
|
||||
yield saver
|
||||
return
|
||||
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from deerflow.runtime.checkpointer.checkpoint_migration import (
|
||||
DualReadCheckpointer,
|
||||
SQLiteCheckpointSource,
|
||||
)
|
||||
|
||||
sources: list[SQLiteCheckpointSource] = []
|
||||
for sqlite_path in _dual_read_paths(config):
|
||||
if not Path(sqlite_path).exists():
|
||||
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
|
||||
continue
|
||||
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
|
||||
source_saver = await ctx.__aenter__()
|
||||
await source_saver.setup()
|
||||
sources.append(
|
||||
SQLiteCheckpointSource(
|
||||
path=sqlite_path,
|
||||
saver=source_saver,
|
||||
context=ctx,
|
||||
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
|
||||
)
|
||||
)
|
||||
|
||||
if not sources:
|
||||
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
|
||||
yield saver
|
||||
return
|
||||
|
||||
proxy = DualReadCheckpointer(
|
||||
saver,
|
||||
sources,
|
||||
)
|
||||
try:
|
||||
yield proxy
|
||||
finally:
|
||||
await proxy.aclose_sources()
|
||||
return
|
||||
|
||||
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public async context manager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
|
||||
"""Async context manager that constructs a checkpointer from unified DatabaseConfig."""
|
||||
if db_config.backend == "memory":
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
yield InMemorySaver()
|
||||
return
|
||||
|
||||
if db_config.backend == "sqlite":
|
||||
try:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(SQLITE_INSTALL) from exc
|
||||
|
||||
conn_str = db_config.checkpointer_sqlite_path
|
||||
ensure_sqlite_parent_dir(conn_str)
|
||||
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
return
|
||||
|
||||
if db_config.backend == "postgres":
|
||||
if not db_config.postgres_url:
|
||||
raise ValueError("database.postgres_url is required for the postgres backend")
|
||||
|
||||
# 连接池 + 健康检查(见 _pg_pool_saver),避免单连接空闲被关。
|
||||
async with _pg_pool_saver(db_config.postgres_url) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
raise ValueError(f"Unknown database backend: {db_config.backend!r}")
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
|
||||
"""Async context manager that yields a checkpointer for the caller's lifetime.
|
||||
Resources are opened on enter and closed on exit -- no global state::
|
||||
|
||||
async with make_checkpointer(app_config) as checkpointer:
|
||||
app.state.checkpointer = checkpointer
|
||||
|
||||
Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
|
||||
|
||||
Priority:
|
||||
1. Legacy ``checkpointer:`` config section (backward compatible)
|
||||
2. Unified ``database:`` config section
|
||||
3. Default InMemorySaver
|
||||
"""
|
||||
|
||||
if app_config is None:
|
||||
app_config = get_app_config()
|
||||
|
||||
# Legacy: standalone checkpointer config takes precedence
|
||||
if app_config.checkpointer is not None:
|
||||
async with _async_checkpointer(app_config.checkpointer) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
# Unified database config
|
||||
db_config = getattr(app_config, "database", None)
|
||||
if db_config is not None and db_config.backend != "memory":
|
||||
async with _async_checkpointer_from_database(db_config) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
# Default: in-memory
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
yield InMemorySaver()
|
||||
"""Async checkpointer factory.
|
||||
|
||||
Provides an **async context manager** for long-running async servers that need
|
||||
proper resource cleanup.
|
||||
|
||||
Supported backends: memory, sqlite, postgres.
|
||||
|
||||
Usage (e.g. FastAPI lifespan)::
|
||||
|
||||
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
|
||||
|
||||
async with make_checkpointer() as checkpointer:
|
||||
app.state.checkpointer = checkpointer # InMemorySaver if not configured
|
||||
|
||||
For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from langgraph.types import Checkpointer
|
||||
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.runtime.checkpointer.provider import (
|
||||
POSTGRES_CONN_REQUIRED,
|
||||
POSTGRES_INSTALL,
|
||||
SQLITE_INSTALL,
|
||||
)
|
||||
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _csv_env(name: str) -> list[str]:
|
||||
raw = os.getenv(name, "")
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def _dual_read_enabled(config) -> bool:
|
||||
if getattr(config, "dual_read_enabled", False):
|
||||
return True
|
||||
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _use_postgres_enabled(config) -> bool:
|
||||
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
|
||||
if raw:
|
||||
return raw.strip().lower() not in {"0", "false", "no", "off"}
|
||||
return bool(getattr(config, "use_postgres", True))
|
||||
|
||||
|
||||
def _dual_read_paths(config) -> list[str]:
|
||||
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
|
||||
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
conn_str = resolve_sqlite_conn_str(path)
|
||||
if conn_str == ":memory:" or conn_str.startswith("file:"):
|
||||
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
|
||||
continue
|
||||
if conn_str not in seen:
|
||||
resolved.append(conn_str)
|
||||
seen.add(conn_str)
|
||||
return resolved
|
||||
|
||||
|
||||
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
|
||||
configured = getattr(config, "sqlite_connection_string", None)
|
||||
if configured:
|
||||
return configured
|
||||
paths = _dual_read_paths(config)
|
||||
return paths[0] if paths else "store.db"
|
||||
|
||||
|
||||
def _pool_size_env(name: str, default: int) -> int:
|
||||
raw = os.getenv(name, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except ValueError:
|
||||
logger.warning("Invalid %s=%r; using default %d", name, raw, default)
|
||||
return default
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _postgres_saver_from_pool(conn_string: str) -> AsyncIterator[Checkpointer]:
|
||||
"""Yield an ``AsyncPostgresSaver`` backed by a self-healing connection pool.
|
||||
|
||||
``AsyncPostgresSaver.from_conn_string`` opens a *single* long-lived
|
||||
connection. If that one connection dies (Postgres restart, idle timeout,
|
||||
network blip, or a prior broken pipeline), every later checkpointer call
|
||||
fails permanently with "the connection is closed" until the process is
|
||||
restarted. A pool with ``check=check_connection`` validates each connection
|
||||
on checkout and transparently replaces dead ones, so the runtime recovers
|
||||
on its own.
|
||||
"""
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
pool = AsyncConnectionPool(
|
||||
conninfo=conn_string,
|
||||
min_size=_pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
|
||||
max_size=_pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
|
||||
max_idle=300.0,
|
||||
max_lifetime=3600.0,
|
||||
check=AsyncConnectionPool.check_connection,
|
||||
kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row},
|
||||
open=False,
|
||||
)
|
||||
await pool.open(wait=True)
|
||||
try:
|
||||
saver = AsyncPostgresSaver(pool)
|
||||
await saver.setup()
|
||||
yield saver
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
|
||||
try:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(SQLITE_INSTALL) from exc
|
||||
|
||||
conn_str = resolve_sqlite_conn_str(raw)
|
||||
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
|
||||
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
|
||||
"""Async context manager that constructs and tears down a checkpointer."""
|
||||
if config.type == "memory":
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
yield InMemorySaver()
|
||||
return
|
||||
|
||||
if config.type == "sqlite":
|
||||
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
if config.type == "postgres":
|
||||
if not _use_postgres_enabled(config):
|
||||
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
|
||||
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
|
||||
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ImportError(POSTGRES_INSTALL) from exc
|
||||
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
async with _postgres_saver_from_pool(config.connection_string) as saver:
|
||||
if not _dual_read_enabled(config):
|
||||
yield saver
|
||||
return
|
||||
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from deerflow.runtime.checkpointer.checkpoint_migration import (
|
||||
DualReadCheckpointer,
|
||||
SQLiteCheckpointSource,
|
||||
)
|
||||
|
||||
sources: list[SQLiteCheckpointSource] = []
|
||||
for sqlite_path in _dual_read_paths(config):
|
||||
if not Path(sqlite_path).exists():
|
||||
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
|
||||
continue
|
||||
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
|
||||
source_saver = await ctx.__aenter__()
|
||||
await source_saver.setup()
|
||||
sources.append(
|
||||
SQLiteCheckpointSource(
|
||||
path=sqlite_path,
|
||||
saver=source_saver,
|
||||
context=ctx,
|
||||
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
|
||||
)
|
||||
)
|
||||
|
||||
if not sources:
|
||||
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
|
||||
yield saver
|
||||
return
|
||||
|
||||
proxy = DualReadCheckpointer(
|
||||
saver,
|
||||
sources,
|
||||
)
|
||||
try:
|
||||
yield proxy
|
||||
finally:
|
||||
await proxy.aclose_sources()
|
||||
return
|
||||
|
||||
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public async context manager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
|
||||
"""Async context manager that constructs a checkpointer from unified DatabaseConfig."""
|
||||
if db_config.backend == "memory":
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
yield InMemorySaver()
|
||||
return
|
||||
|
||||
if db_config.backend == "sqlite":
|
||||
try:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(SQLITE_INSTALL) from exc
|
||||
|
||||
conn_str = db_config.checkpointer_sqlite_path
|
||||
ensure_sqlite_parent_dir(conn_str)
|
||||
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
return
|
||||
|
||||
if db_config.backend == "postgres":
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ImportError(POSTGRES_INSTALL) from exc
|
||||
|
||||
if not db_config.postgres_url:
|
||||
raise ValueError("database.postgres_url is required for the postgres backend")
|
||||
|
||||
async with _postgres_saver_from_pool(db_config.postgres_url) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
raise ValueError(f"Unknown database backend: {db_config.backend!r}")
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
|
||||
"""Async context manager that yields a checkpointer for the caller's lifetime.
|
||||
Resources are opened on enter and closed on exit -- no global state::
|
||||
|
||||
async with make_checkpointer(app_config) as checkpointer:
|
||||
app.state.checkpointer = checkpointer
|
||||
|
||||
Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
|
||||
|
||||
Priority:
|
||||
1. Legacy ``checkpointer:`` config section (backward compatible)
|
||||
2. Unified ``database:`` config section
|
||||
3. Default InMemorySaver
|
||||
"""
|
||||
|
||||
if app_config is None:
|
||||
app_config = get_app_config()
|
||||
|
||||
# Legacy: standalone checkpointer config takes precedence
|
||||
if app_config.checkpointer is not None:
|
||||
async with _async_checkpointer(app_config.checkpointer) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
# Unified database config
|
||||
db_config = getattr(app_config, "database", None)
|
||||
if db_config is not None and db_config.backend != "memory":
|
||||
async with _async_checkpointer_from_database(db_config) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
# Default: in-memory
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
yield InMemorySaver()
|
||||
+38
-2
@@ -28,6 +28,42 @@ from deerflow.runtime.store.provider import POSTGRES_CONN_REQUIRED, POSTGRES_STO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _pool_size_env(name: str, default: int) -> int:
|
||||
import os
|
||||
|
||||
raw = os.getenv(name, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except ValueError:
|
||||
logger.warning("Invalid %s=%r; using default %d", name, raw, default)
|
||||
return default
|
||||
|
||||
|
||||
def _postgres_pool_config() -> dict:
|
||||
"""Pool config for the Postgres store.
|
||||
|
||||
A single long-lived connection (the default ``from_conn_string`` behavior)
|
||||
dies permanently on a server restart / idle timeout / network blip — every
|
||||
later store call then fails with "server closed the connection
|
||||
unexpectedly". A pool with ``check=check_connection`` validates each
|
||||
connection on checkout and replaces dead ones, so the runtime self-heals.
|
||||
Extra keys (``check``/``max_idle``/``max_lifetime``) are passed straight
|
||||
through to ``AsyncConnectionPool`` by ``from_conn_string``.
|
||||
"""
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
return {
|
||||
"min_size": _pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
|
||||
"max_size": _pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
|
||||
"check": AsyncConnectionPool.check_connection,
|
||||
"max_idle": 300.0,
|
||||
"max_lifetime": 3600.0,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal backend factory
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -71,9 +107,9 @@ async def _async_store(config) -> AsyncIterator[BaseStore]:
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
async with AsyncPostgresStore.from_conn_string(config.connection_string) as store:
|
||||
async with AsyncPostgresStore.from_conn_string(config.connection_string, pool_config=_postgres_pool_config()) as store:
|
||||
await store.setup()
|
||||
logger.info("Store: using AsyncPostgresStore")
|
||||
logger.info("Store: using AsyncPostgresStore (pooled, self-healing)")
|
||||
yield store
|
||||
return
|
||||
|
||||
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""Interactive browser action tool backed by the user's Chrome extension.
|
||||
|
||||
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
|
||||
*observe -> act* loop on a live page the extension keeps open:
|
||||
|
||||
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
|
||||
2. The extension performs the action on the session tab, then re-scans the page
|
||||
and returns an *observation*: the final URL/title, an indexed list of
|
||||
interactive elements, and the visible page text.
|
||||
3. The model reads the observation, picks the next element by its index, and
|
||||
calls ``browser_act`` again with the same ``session_id``.
|
||||
|
||||
The element ``index`` values are only valid for the most recent observation of a
|
||||
session; always act on indices from the latest result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
|
||||
_NEEDS_INDEX = {"click", "type"}
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
|
||||
if not elements:
|
||||
return "(no interactive elements detected)"
|
||||
lines: list[str] = []
|
||||
for el in elements:
|
||||
idx = el.get("index")
|
||||
tag = str(el.get("tag") or "?")
|
||||
el_type = str(el.get("type") or "").strip()
|
||||
label = str(el.get("label") or "").strip().replace("\n", " ")
|
||||
if len(label) > 120:
|
||||
label = label[:120] + "…"
|
||||
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
|
||||
extra = ""
|
||||
if el.get("value"):
|
||||
value = str(el["value"]).strip().replace("\n", " ")
|
||||
extra = f' value="{value[:60]}"'
|
||||
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
|
||||
observation = observation or {}
|
||||
status = str(observation.get("action_status") or "ok")
|
||||
message = str(observation.get("message") or "").strip()
|
||||
url = str(observation.get("url") or "").strip() or "(unknown)"
|
||||
title = str(observation.get("title") or "").strip() or "(untitled)"
|
||||
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
|
||||
text = str(observation.get("text") or "").strip()
|
||||
text_limit = max(500, int(settings.observation_text_max_chars))
|
||||
text_truncated = len(text) > text_limit
|
||||
if text_truncated:
|
||||
text = text[:text_limit]
|
||||
|
||||
parts = [
|
||||
f"Browser session: {session_id}",
|
||||
f"Last action: {action} -> {status}",
|
||||
]
|
||||
if message:
|
||||
parts.append(f"Note: {message}")
|
||||
parts.extend(
|
||||
[
|
||||
f"URL: {url}",
|
||||
f"Title: {title}",
|
||||
"",
|
||||
"Interactive elements (use the [index] with click/type):",
|
||||
_format_elements(elements),
|
||||
"",
|
||||
"Page text:",
|
||||
text or "(empty page text)",
|
||||
]
|
||||
)
|
||||
if text_truncated:
|
||||
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
|
||||
parts.append(
|
||||
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
|
||||
"When the task is finished, call browser_act with action=\"close\" to release the tab."
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@tool("browser_act", parse_docstring=True)
|
||||
async def browser_act_tool(
|
||||
action: str,
|
||||
session_id: str = "",
|
||||
url: str = "",
|
||||
index: int = -1,
|
||||
text: str = "",
|
||||
submit: bool = False,
|
||||
wait_for: str = "",
|
||||
wait_seconds: int = 0,
|
||||
) -> str:
|
||||
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
|
||||
|
||||
Use this to operate a real web page on the user's behalf — log in, fill forms,
|
||||
click through navigation, and read pages that need the user's session or local
|
||||
network. Start every task with ``action="navigate"`` and a URL to open a tab
|
||||
and get the first observation, then keep calling this tool with the same
|
||||
``session_id``, choosing elements by the ``index`` shown in the observation.
|
||||
Call ``action="close"`` when done to free the tab.
|
||||
|
||||
Args:
|
||||
action: One of navigate, click, type, scroll, wait, extract, observe, close.
|
||||
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
|
||||
url: Target http(s) URL. Required for navigate (and to start a new session).
|
||||
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
|
||||
text: Text to enter for the type action.
|
||||
submit: For type, press Enter / submit the form after typing.
|
||||
wait_for: For wait, a CSS selector to wait for before observing.
|
||||
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
|
||||
if not settings.actions_enabled:
|
||||
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
|
||||
|
||||
action = (action or "").strip().lower()
|
||||
if action not in _ACTIONS:
|
||||
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
|
||||
|
||||
session_id = (session_id or "").strip()
|
||||
safe_url = _validate_url(url) if url else None
|
||||
|
||||
if action == "navigate" and safe_url is None:
|
||||
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
if not session_id:
|
||||
if action != "navigate":
|
||||
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
|
||||
session_id = uuid4().hex
|
||||
|
||||
if action in _NEEDS_INDEX and index < 0:
|
||||
return f"browser_act {action} requires a non-negative element index from the latest observation."
|
||||
if action == "type" and not text:
|
||||
return "browser_act type requires the text to enter."
|
||||
|
||||
action_payload: dict[str, Any] = {"type": action}
|
||||
if safe_url is not None:
|
||||
action_payload["url"] = safe_url
|
||||
if index >= 0:
|
||||
action_payload["index"] = int(index)
|
||||
if text:
|
||||
action_payload["text"] = text
|
||||
if submit:
|
||||
action_payload["submit"] = True
|
||||
if wait_for.strip():
|
||||
action_payload["wait_for"] = wait_for.strip()[:300]
|
||||
|
||||
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url or "",
|
||||
instruction="",
|
||||
kind="act",
|
||||
session_id=session_id,
|
||||
action=action_payload,
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_act_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
|
||||
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
if action == "close":
|
||||
return f"browser_act session {session_id}: tab closed."
|
||||
|
||||
result = completed.get("result") or {}
|
||||
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
|
||||
return _format_observation(action, observation, session_id, settings)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""Browser extension backed page fetch tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
|
||||
def _settings() -> BrowserContextConfig:
|
||||
value = getattr(get_app_config(), "browser_context", None)
|
||||
if isinstance(value, BrowserContextConfig):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return BrowserContextConfig.model_validate(value)
|
||||
return BrowserContextConfig()
|
||||
|
||||
|
||||
def _validate_url(raw_url: str) -> str | None:
|
||||
url = (raw_url or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
return None
|
||||
return url
|
||||
|
||||
|
||||
@tool("browser_fetch_page", parse_docstring=True)
|
||||
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
|
||||
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
|
||||
|
||||
Use this when the user explicitly asks to use the browser extension or asks
|
||||
for data from a page that may require their browser session, login state, or
|
||||
local network access. The URL must be an exact http(s) URL provided by the
|
||||
user. If the user did not provide a URL, ask for the URL instead of guessing.
|
||||
|
||||
Args:
|
||||
url: Exact http(s) URL to open in the user's Chrome browser.
|
||||
question: Optional instruction describing what data the user wants from the page.
|
||||
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
|
||||
"""
|
||||
|
||||
settings = _settings()
|
||||
if not settings.enabled:
|
||||
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
|
||||
|
||||
safe_url = _validate_url(url)
|
||||
if safe_url is None:
|
||||
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
|
||||
|
||||
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
|
||||
requested_timeout = int(wait_seconds or configured_timeout)
|
||||
timeout = min(max(5, requested_timeout), configured_timeout)
|
||||
user_id = get_effective_user_id()
|
||||
|
||||
task = create_browser_task(
|
||||
user_id=user_id,
|
||||
url=safe_url,
|
||||
instruction=(question or "").strip(),
|
||||
timeout_seconds=timeout + int(settings.task_lease_seconds),
|
||||
max_content_chars=settings.max_content_chars,
|
||||
metadata={"source": "browser_fetch_page_tool"},
|
||||
)
|
||||
|
||||
completed = await wait_for_browser_task_result(
|
||||
task_id=task["id"],
|
||||
user_id=user_id,
|
||||
timeout_seconds=timeout,
|
||||
poll_interval_seconds=settings.task_poll_interval_seconds,
|
||||
)
|
||||
if completed is None:
|
||||
return f"Browser task {task['id']} was not found after it was created."
|
||||
|
||||
status = completed.get("status")
|
||||
if status == "error":
|
||||
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
|
||||
if status == "expired":
|
||||
return (
|
||||
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
|
||||
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
|
||||
)
|
||||
if status != "done":
|
||||
return (
|
||||
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
|
||||
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
|
||||
)
|
||||
|
||||
result = completed.get("result") or {}
|
||||
content = str(result.get("content") or "").strip()
|
||||
tool_limit = max(1_000, int(settings.tool_result_max_chars))
|
||||
tool_truncated = len(content) > tool_limit
|
||||
if tool_truncated:
|
||||
content = content[:tool_limit]
|
||||
|
||||
notes: list[str] = []
|
||||
if result.get("truncated"):
|
||||
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
|
||||
if tool_truncated:
|
||||
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
|
||||
|
||||
parts = [
|
||||
"Browser page fetched through the user's Chrome extension.",
|
||||
f"Task ID: {task['id']}",
|
||||
f"Title: {result.get('title') or '(untitled)'}",
|
||||
f"URL: {result.get('url') or safe_url}",
|
||||
]
|
||||
if question:
|
||||
parts.extend(["", "Requested extraction:", question.strip()])
|
||||
if notes:
|
||||
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
|
||||
parts.extend(["", "Page content:", content or "(empty page text)"])
|
||||
return "\n".join(parts)
|
||||
|
||||
@@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool"
|
||||
BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool"
|
||||
BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE}
|
||||
BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"}
|
||||
|
||||
BUILTIN_TOOLS = [
|
||||
present_file_tool,
|
||||
ask_clarification_tool,
|
||||
@@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _is_browser_context_tool(tool: object) -> bool:
|
||||
"""Return True if the tool config represents the browser extension bridge."""
|
||||
|
||||
use = getattr(tool, "use", None)
|
||||
name = getattr(tool, "name", None)
|
||||
return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
|
||||
|
||||
|
||||
def get_available_tools(
|
||||
groups: list[str] | None = None,
|
||||
include_mcp: bool = True,
|
||||
@@ -64,6 +77,18 @@ def get_available_tools(
|
||||
config = app_config or get_app_config()
|
||||
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
|
||||
|
||||
browser_context_config = getattr(config, "browser_context", None)
|
||||
if not getattr(browser_context_config, "enabled", False):
|
||||
tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
|
||||
elif not getattr(browser_context_config, "actions_enabled", True):
|
||||
# browser_context is on, but interactive page actions are disabled — keep
|
||||
# the read-only browser_fetch_page tool, drop only browser_act.
|
||||
tool_configs = [
|
||||
tool
|
||||
for tool in tool_configs
|
||||
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
|
||||
]
|
||||
|
||||
# Do not expose host bash by default when LocalSandboxProvider is active.
|
||||
if not is_host_bash_allowed(config):
|
||||
tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)]
|
||||
|
||||
Reference in new issue
Block a user