Merge remote-tracking branch 'origin/master' into xuhd

# Conflicts:
#	frontend-web/.env.production
#	frontend-web/.env.sit
#	frontend-web/package.json
#	offline-backend-20260512/backend/config.yaml
This commit is contained in:
xuhd committed 2026-06-11 14:19:01 +08:00
commit 1668c1da87
177 files changed
+21030 -2738

No files matched your search

@@ -299,6 +299,16 @@ def _build_middlewares(
if not is_bootstrap and not is_writing_setup:
middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory, injection_enabled=memory_injection_enabled))
# KnowledgeRagMiddleware — inject knowledge-base recall (phase 2). Gated
# per-conversation by the ``knowledge_rag_enabled`` runtime flag, defaulting
# to the global ``knowledge.rag_enabled`` config. No-op when knowledge is
# disabled. Skipped in bootstrap/writing-setup like MemoryMiddleware.
knowledge_cfg = getattr(resolved_app_config, "knowledge", None)
if not is_bootstrap and not is_writing_setup and knowledge_cfg is not None and knowledge_cfg.enabled:
from deerflow.agents.middlewares.knowledge_rag_middleware import KnowledgeRagMiddleware
middlewares.append(KnowledgeRagMiddleware(limit=knowledge_cfg.rag_limit, default_enabled=knowledge_cfg.rag_enabled))
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
# ingests raw image data). In delegated mode the view_image tool already
# returns a text description and stores nothing in `viewed_images`, so the
@@ -0,0 +1,190 @@
"""KnowledgeRagMiddleware — inject knowledge-base recall before the model call.
Phase 2 of the product knowledge base. When enabled (per-conversation runtime
flag ``knowledge_rag_enabled``, falling back to the global
``knowledge.rag_enabled`` config), the latest user question is used to retrieve
the most relevant knowledge notes, which are injected as a transient
``<knowledge-context>`` system message — not persisted to state, so it never
leaks to the frontend transcript.
Retrieval is hybrid (vector + keyword) when an embedding endpoint is
configured, else keyword/wiki-query layered search. Failures are swallowed: RAG
must never break a conversation.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain_core.messages import HumanMessage
from langgraph.runtime import Runtime
logger = logging.getLogger(__name__)
_TAG = "knowledge-context"
def _coerce_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict) and block.get("type") == "text":
parts.append(str(block.get("text", "")))
return " ".join(parts)
return str(content or "")
def _latest_human_text(messages: list) -> str:
for msg in reversed(messages or []):
if isinstance(msg, HumanMessage):
return _coerce_text(msg.content).strip()
return ""
class KnowledgeRagMiddleware(AgentMiddleware[AgentState]):
"""Inject knowledge-base recall as transient system context."""
state_schema = AgentState
def __init__(self, *, limit: int = 5, default_enabled: bool = False) -> None:
super().__init__()
self._limit = limit
self._default_enabled = default_enabled
self._service = None # lazy
def _get_service(self):
if self._service is not None:
return self._service
try:
from deerflow.config import get_app_config
from deerflow.knowledge import make_knowledge_service
from deerflow.persistence.engine import get_session_factory
config = get_app_config()
self._service = make_knowledge_service(get_session_factory(), config.knowledge)
except Exception:
logger.exception("KnowledgeRagMiddleware: failed to build knowledge service")
self._service = None
return self._service
def _enabled_for(self, request: Any) -> bool:
runtime = getattr(request, "runtime", None)
if runtime is not None and getattr(runtime, "context", None):
flag = runtime.context.get("knowledge_rag_enabled")
if flag is not None:
return bool(flag)
return self._default_enabled
async def _build_context_message(self, request: Any) -> HumanMessage | None:
if not self._enabled_for(request):
return None
service = self._get_service()
if service is None:
return None
query = _latest_human_text(getattr(request, "messages", None) or [])
if not query:
return None
try:
result = await service.build_rag_context(query, limit=self._limit)
except Exception:
logger.exception("KnowledgeRagMiddleware: recall failed (non-fatal)")
return None
text = result.get("text") if isinstance(result, dict) else ""
if not text:
return None
fenced = (
f"<{_TAG}>\n"
"[系统提示:以下是知识库召回的背景资料,不是用户的新输入。回答时优先参考,"
"并在使用到时标注来源 note_id。]\n\n"
f"{text}\n"
f"</{_TAG}>"
)
return HumanMessage(content=fenced)
@override
async def awrap_model_call(self, request: Any, handler: Any) -> Any:
msg = await self._build_context_message(request)
if msg is not None:
request = request.override(messages=[*request.messages, msg])
return await handler(request)
@override
def wrap_model_call(self, request: Any, handler: Any) -> Any:
# Sync path: best-effort. The gateway runtime is async, so this is rarely
# hit; run the async recall in a private loop when there is none.
if not self._enabled_for(request):
return handler(request)
try:
loop = asyncio.get_event_loop()
if loop.is_running():
# Cannot block a running loop from sync code; skip injection.
return handler(request)
msg = loop.run_until_complete(self._build_context_message(request))
except Exception:
msg = None
if msg is not None:
request = request.override(messages=[*request.messages, msg])
return handler(request)
# ---- after_agent: enqueue background auto-sedimentation -------------
def _resolve_thread_id(self, runtime: Runtime | None) -> str | None:
if runtime is not None and getattr(runtime, "context", None):
tid = runtime.context.get("thread_id")
if tid:
return str(tid)
try:
from langgraph.config import get_config
return get_config().get("configurable", {}).get("thread_id")
except Exception:
return None
@override
def after_agent(self, state: AgentState, runtime: Runtime) -> dict | None:
"""When auto-ingest is on, hand the finished turn to the background queue."""
try:
from deerflow.config import get_app_config
knowledge_cfg = getattr(get_app_config(), "knowledge", None)
if knowledge_cfg is None or not knowledge_cfg.enabled or not knowledge_cfg.auto_ingest_enabled:
return None
from deerflow.knowledge.auto_ingest import AutoIngestItem, get_auto_ingest_queue
queue = get_auto_ingest_queue()
if queue is None:
return None
messages = state.get("messages") if isinstance(state, dict) else getattr(state, "messages", None)
if not messages:
return None
from deerflow.runtime.references import build_reference_batches
from deerflow.runtime.serialization import serialize_lc_object
from deerflow.runtime.user_context import get_effective_user_id
serialized = [serialize_lc_object(m) for m in messages]
thread_id = self._resolve_thread_id(runtime)
if not thread_id:
return None
title = state.get("title") if isinstance(state, dict) else getattr(state, "title", None)
queue.enqueue(
AutoIngestItem(
thread_id=thread_id,
user_id=get_effective_user_id(),
messages=serialized,
thread_title=title if isinstance(title, str) else None,
reference_batches=build_reference_batches(serialized),
)
)
except Exception:
logger.exception("KnowledgeRagMiddleware.after_agent enqueue failed (non-fatal)")
return None
@@ -0,0 +1,2 @@
"""Browser-backed page context helpers."""
@@ -0,0 +1,303 @@
"""Small file-backed task queue for browser page fetch requests."""
from __future__ import annotations
import asyncio
import json
import logging
import re
import threading
import time
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import uuid4
from deerflow.config.runtime_paths import runtime_home
logger = logging.getLogger(__name__)
_LOCK = threading.RLock()
_TASK_ID_RE = re.compile(r"^[a-f0-9]{32}$")
_TERMINAL_STATUSES = {"done", "error", "expired"}
def tasks_dir() -> Path:
"""Return the browser-context task directory, creating it if needed."""
path = runtime_home() / "browser-context" / "tasks"
path.mkdir(parents=True, exist_ok=True)
return path
def _now() -> datetime:
return datetime.now(UTC)
def _iso(value: datetime | None = None) -> str:
return (value or _now()).isoformat()
def _parse_iso(value: Any) -> datetime | None:
if not isinstance(value, str) or not value:
return None
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def _task_path(task_id: str) -> Path | None:
if not _TASK_ID_RE.fullmatch(task_id or ""):
return None
return tasks_dir() / f"{task_id}.json"
def _read_task(path: Path) -> dict[str, Any] | None:
try:
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
except FileNotFoundError:
return None
except Exception:
logger.warning("Ignoring unreadable browser task file: %s", path, exc_info=True)
return None
return data if isinstance(data, dict) else None
def _write_task(path: Path, data: dict[str, Any]) -> None:
tmp = path.with_name(f"{path.name}.{uuid4().hex}.tmp")
with tmp.open("w", encoding="utf-8") as fh:
json.dump(data, fh, ensure_ascii=False, separators=(",", ":"))
tmp.replace(path)
def _public_task(data: dict[str, Any]) -> dict[str, Any]:
return {
"id": data.get("id"),
"kind": data.get("kind") or "fetch",
"url": data.get("url") or "",
"instruction": data.get("instruction") or "",
"session_id": data.get("session_id") or "",
"action": data.get("action") or None,
"created_at": data.get("created_at") or "",
"updated_at": data.get("updated_at") or "",
"max_content_chars": int(data.get("max_content_chars") or 200_000),
}
def _expire_if_needed(path: Path, data: dict[str, Any], now: datetime) -> dict[str, Any]:
if data.get("status") in _TERMINAL_STATUSES:
return data
expires_at = _parse_iso(data.get("expires_at"))
if expires_at is not None and now >= expires_at:
data = dict(data)
data["status"] = "expired"
data["error"] = "Browser task expired before the extension returned a result."
data["updated_at"] = _iso(now)
_write_task(path, data)
return data
def prune_browser_tasks(max_age_seconds: int = 86_400) -> int:
"""Delete old terminal tasks and return the number removed."""
cutoff = _now() - timedelta(seconds=max(60, int(max_age_seconds)))
removed = 0
with _LOCK:
for path in tasks_dir().glob("*.json"):
data = _read_task(path)
if not data or data.get("status") not in _TERMINAL_STATUSES:
continue
updated_at = _parse_iso(data.get("updated_at")) or _parse_iso(data.get("created_at"))
if updated_at is None or updated_at < cutoff:
try:
path.unlink()
removed += 1
except OSError:
logger.debug("Failed to delete old browser task: %s", path, exc_info=True)
return removed
def create_browser_task(
*,
user_id: str,
url: str,
instruction: str = "",
timeout_seconds: int = 60,
max_content_chars: int = 200_000,
metadata: dict[str, Any] | None = None,
kind: str = "fetch",
session_id: str = "",
action: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Create a browser task for the current user's extension to pick up.
``kind`` is ``"fetch"`` for one-shot page captures (the legacy behavior) or
``"act"`` for an interactive observe/act step driven by ``action`` within a
long-lived ``session_id`` (a tab kept open by the extension).
"""
now = _now()
task_id = uuid4().hex
task = {
"id": task_id,
"user_id": str(user_id or "default"),
"status": "pending",
"kind": kind if kind in {"fetch", "act"} else "fetch",
"url": url,
"instruction": instruction or "",
"session_id": str(session_id or ""),
"action": action or None,
"created_at": _iso(now),
"updated_at": _iso(now),
"expires_at": _iso(now + timedelta(seconds=max(5, int(timeout_seconds)))),
"max_content_chars": max(1_000, int(max_content_chars)),
"worker_id": None,
"claimed_at": None,
"result": None,
"error": None,
"metadata": metadata or {},
}
path = tasks_dir() / f"{task_id}.json"
with _LOCK:
_write_task(path, task)
return task
def get_browser_task(*, task_id: str, user_id: str) -> dict[str, Any] | None:
"""Read a task if it belongs to the requested user."""
path = _task_path(task_id)
if path is None:
return None
with _LOCK:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
return None
return _expire_if_needed(path, data, _now())
def claim_next_browser_task(
*,
user_id: str,
worker_id: str,
lease_seconds: int = 120,
) -> dict[str, Any] | None:
"""Claim the oldest pending task for a user."""
now = _now()
user_id = str(user_id or "default")
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
lease_seconds = max(10, int(lease_seconds))
with _LOCK:
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
for path in paths:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != user_id:
continue
data = _expire_if_needed(path, data, now)
status = data.get("status")
if status == "running":
updated_at = _parse_iso(data.get("updated_at")) or now
if (now - updated_at).total_seconds() <= lease_seconds:
continue
status = "pending"
data = dict(data)
data["status"] = status
if status != "pending":
continue
data = dict(data)
data["status"] = "running"
data["worker_id"] = worker_id
data["claimed_at"] = _iso(now)
data["updated_at"] = _iso(now)
_write_task(path, data)
return _public_task(data)
return None
def complete_browser_task(
*,
task_id: str,
user_id: str,
title: str = "",
url: str = "",
content: str = "",
content_type: str = "markdown",
error: str = "",
metadata: dict[str, Any] | None = None,
observation: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Mark a browser task complete or failed.
For ``act`` tasks the extension returns a structured ``observation`` (final
URL/title, interactive element list, page text) instead of a captured
markdown document; it is stored alongside the result verbatim.
"""
path = _task_path(task_id)
if path is None:
return None
now = _now()
with _LOCK:
data = _read_task(path)
if not data or str(data.get("user_id") or "") != str(user_id or "default"):
return None
data = dict(data)
if error:
data["status"] = "error"
data["error"] = str(error).strip()[:4000]
data["result"] = None
else:
raw_content = str(content or "").strip()
max_content_chars = max(1_000, int(data.get("max_content_chars") or 200_000))
truncated = len(raw_content) > max_content_chars
data["status"] = "done"
data["error"] = None
data["result"] = {
"title": str(title or "").strip()[:500],
"url": str(url or data.get("url") or "").strip()[:4096],
"content": raw_content[:max_content_chars] if truncated else raw_content,
"content_type": content_type if content_type in {"markdown", "text"} else "markdown",
"truncated": truncated,
"observation": observation if isinstance(observation, dict) else None,
"captured_at": _iso(now),
"metadata": metadata or {},
}
data["updated_at"] = _iso(now)
_write_task(path, data)
return data
async def wait_for_browser_task_result(
*,
task_id: str,
user_id: str,
timeout_seconds: int,
poll_interval_seconds: float = 1.0,
) -> dict[str, Any] | None:
"""Wait for a browser task to reach a terminal state."""
deadline = time.monotonic() + max(1, int(timeout_seconds))
interval = max(0.25, float(poll_interval_seconds))
while True:
task = get_browser_task(task_id=task_id, user_id=user_id)
if task is None:
return None
if task.get("status") in _TERMINAL_STATUSES:
return task
if time.monotonic() >= deadline:
return task
await asyncio.sleep(interval)
@@ -10,11 +10,13 @@ from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
@@ -133,6 +135,8 @@ class AppConfig(BaseModel):
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
@classmethod
def resolve_config_path(cls, config_path: str | None = None) -> Path:
@@ -0,0 +1,74 @@
"""Configuration for browser page context capture."""
from pydantic import BaseModel, Field
class BrowserContextConfig(BaseModel):
"""Controls the optional Chrome-extension page capture integration."""
enabled: bool = Field(
default=False,
description="Enable /api/browser-context endpoints for Chrome extension page capture.",
)
max_content_chars: int = Field(
default=200_000,
ge=1_000,
description="Maximum captured page text stored per request. Longer content is truncated.",
)
max_title_chars: int = Field(
default=300,
ge=20,
description="Maximum title length accepted from the browser extension.",
)
max_url_chars: int = Field(
default=2_048,
ge=100,
description="Maximum source URL length accepted from the browser extension.",
)
filename_prefix: str = Field(
default="webpage",
description="Prefix used for Markdown files created from captured pages.",
)
route_base: str = Field(
default="/page/workspace",
description="Frontend route base used when returning chat paths to the extension.",
)
task_result_timeout_seconds: int = Field(
default=60,
ge=5,
description="Maximum time the browser_fetch_page tool waits for the extension result.",
)
task_poll_interval_seconds: float = Field(
default=1.0,
ge=0.25,
description="How often the backend tool checks for a returned browser task result.",
)
task_lease_seconds: int = Field(
default=120,
ge=10,
description="How long a claimed browser task is reserved before another extension may retry it.",
)
tool_result_max_chars: int = Field(
default=60_000,
ge=1_000,
description="Maximum page characters returned from browser_fetch_page into the model context.",
)
actions_enabled: bool = Field(
default=True,
description="Enable the interactive browser_act tool (click/type/scroll/wait/navigate on the user's page).",
)
action_result_timeout_seconds: int = Field(
default=45,
ge=5,
description="Maximum time the browser_act tool waits for the extension to perform one step and return an observation.",
)
max_observed_elements: int = Field(
default=150,
ge=10,
description="Maximum number of interactive elements returned in a browser_act observation.",
)
observation_text_max_chars: int = Field(
default=12_000,
ge=500,
description="Maximum page text characters returned alongside a browser_act observation.",
)
@@ -3,7 +3,7 @@
import json
import os
from pathlib import Path
from typing import Any, Literal
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -53,115 +53,115 @@ class McpServerConfig(BaseModel):
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayTableColumn(BaseModel):
"""A configured table column for structured skill result display."""
label: str = Field(description="Column label shown in the frontend table")
field: str = Field(description="Field name or dot path read from each result item")
width: str | None = Field(default=None, description="Optional CSS width for this column")
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayResultSetConfig(BaseModel):
"""Display adapter for one result array inside a skill response."""
key: str = Field(description="Stable key for this result set")
label: str | None = Field(default=None, description="Human-readable result set label")
enabled: bool = Field(default=True, description="Whether this result set is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="citation",
description="How the frontend should render this result set",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="singleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayConfig(BaseModel):
"""Frontend display adapter for structured skill results."""
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="none",
description="How the frontend should render this skill's structured results",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="doubleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
result_sets: list[SkillDisplayResultSetConfig] = Field(
default_factory=list,
description="Optional per-result-array display adapters",
alias="resultSets",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
"""Configuration for a single skill's state."""
enabled: bool = Field(default=True, description="Whether this skill is enabled")
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
class SkillDisplayTableColumn(BaseModel):
"""A configured table column for structured skill result display."""
label: str = Field(description="Column label shown in the frontend table")
field: str = Field(description="Field name or dot path read from each result item")
width: str | None = Field(default=None, description="Optional CSS width for this column")
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayResultSetConfig(BaseModel):
"""Display adapter for one result array inside a skill response."""
key: str = Field(description="Stable key for this result set")
label: str | None = Field(default=None, description="Human-readable result set label")
enabled: bool = Field(default=True, description="Whether this result set is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="citation",
description="How the frontend should render this result set",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="singleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=100, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillDisplayConfig(BaseModel):
"""Frontend display adapter for structured skill results."""
enabled: bool = Field(default=False, description="Whether result display adaptation is enabled")
mode: Literal["citation", "list", "table", "card", "custom", "none"] = Field(
default="none",
description="How the frontend should render this skill's structured results",
)
result_path: str = Field(default="results", description="Dot path to the array of result items", alias="resultPath")
title_field: str | None = Field(default=None, description="Field used as item title", alias="titleField")
summary_field: str | None = Field(default=None, description="Field used as short summary/snippet", alias="summaryField")
content_field: str | None = Field(default=None, description="Field used as full item content", alias="contentField")
source_field: str | None = Field(default=None, description="Field used as source label", alias="sourceField")
author_field: str | None = Field(default=None, description="Field used as author/byline label", alias="authorField")
type_field: str | None = Field(default=None, description="Field used as source/type badge", alias="typeField")
time_field: str | None = Field(default=None, description="Field used as publication/update time", alias="timeField")
score_field: str | None = Field(default=None, description="Field used as relevance score", alias="scoreField")
url_field: str | None = Field(default=None, description="Field containing a direct URL", alias="urlField")
id_field: str | None = Field(default=None, description="Field containing an ID for template URLs", alias="idField")
link_strategy: Literal["auto", "url", "id"] = Field(
default="auto",
description="Whether links use a URL field, an ID template, or automatic fallback",
alias="linkStrategy",
)
link_template: str | None = Field(default=None, description="Template used when linkStrategy is id", alias="linkTemplate")
citation_token: Literal["doubleBracket", "singleBracket"] = Field(
default="doubleBracket",
description="Citation token style expected in model output",
alias="citationToken",
)
max_items: int = Field(default=8, ge=1, le=100, description="Maximum displayed items", alias="maxItems")
custom_instruction: str = Field(
default="",
description="User-authored instruction for custom rendering schema generation",
alias="customInstruction",
)
table_columns: list[SkillDisplayTableColumn] = Field(
default_factory=list,
description="Columns used when mode is table",
alias="tableColumns",
)
result_sets: list[SkillDisplayResultSetConfig] = Field(
default_factory=list,
description="Optional per-result-array display adapters",
alias="resultSets",
)
model_config = ConfigDict(extra="allow", populate_by_name=True)
class SkillStateConfig(BaseModel):
"""Configuration for a single skill's state."""
enabled: bool = Field(default=True, description="Whether this skill is enabled")
display: SkillDisplayConfig | None = Field(default=None, description="Optional frontend display adapter")
class ExtensionsConfig(BaseModel):
@@ -0,0 +1,75 @@
"""Configuration for the product knowledge base (obsidian-wiki compatible)."""
from pydantic import BaseModel, Field
class KnowledgeExportConfig(BaseModel):
"""Knowledge graph export switches (``wiki-export`` compatible output)."""
enabled: bool = Field(default=True, description="Master switch for graph export.")
graph_html_enabled: bool = Field(default=True, description="Generate wiki-export/graph.html.")
graph_json_enabled: bool = Field(default=True, description="Generate wiki-export/graph.json.")
class KnowledgeQueryConfig(BaseModel):
"""Layered query strategy flags (``wiki-query`` style)."""
use_hot_md: bool = Field(default=True, description="Consult hot.md when searching.")
use_index_md: bool = Field(default=True, description="Consult index.md when searching.")
use_frontmatter_scan: bool = Field(default=True, description="Scan frontmatter when searching.")
class KnowledgeEmbeddingConfig(BaseModel):
"""Vector embedding config (phase 2).
Vector RAG is opt-in: set ``enabled: true`` and point ``base_url`` at an
OpenAI-compatible ``/embeddings`` endpoint. When disabled or unreachable,
search transparently falls back to keyword scoring, so the feature degrades
gracefully on offline deployments without an embedding model.
"""
enabled: bool = Field(default=False, description="Enable vector embeddings.")
provider: str = Field(default="openai", description="Embedding provider (openai-compatible).")
model: str = Field(default="", description="Embedding model name, e.g. text-embedding-3-small / bge-m3.")
base_url: str = Field(default="", description="OpenAI-compatible base URL (…/v1). Empty disables vector mode.")
api_key: str = Field(default="", description="API key for the embedding endpoint ($ENV resolved like other config).")
dimensions: int | None = Field(default=None, description="Optional embedding dimensions hint sent to the endpoint.")
batch_size: int = Field(default=32, description="Texts per embedding request.")
timeout_seconds: int = Field(default=30, description="HTTP timeout per embedding request.")
class KnowledgeConfig(BaseModel):
"""Product knowledge base configuration.
The first release ships a global, shared, obsidian-wiki compatible vault.
Knowledge metadata is mirrored into the database for fast listing/search,
while ``content_md`` is written to a Markdown vault that stays compatible
with ``ar9av/obsidian-wiki`` (index.md / log.md / hot.md / .manifest.json,
frontmatter, wikilinks, wiki-export graph artifacts).
"""
enabled: bool = Field(default=True, description="Enable the knowledge base feature.")
provider: str = Field(default="obsidian_wiki", description="Knowledge organization provider.")
vault_dir: str = Field(
default="data/knowledge/obsidian-vault",
description="Vault directory (relative to DEER_FLOW_HOME, or absolute).",
)
project_name: str = Field(default="zncm", description="Project namespace for projects/<name>/ pages.")
auto_ingest_enabled: bool = Field(default=False, description="Auto-sediment conversations in the background (phase 3).")
auto_ingest_min_chars: int = Field(default=300, description="Minimum characters before auto-ingest considers a turn.")
llm_extract_enabled: bool = Field(default=True, description="Distill notes with an LLM (wiki-capture style) instead of the rule-based fallback.")
extract_model_name: str | None = Field(default=None, description="Model (from models[]) used to distill knowledge. Falls back to models[0] when unset.")
extract_max_input_chars: int = Field(default=12000, description="Max characters of conversation transcript fed to the extraction LLM.")
max_thread_messages: int = Field(default=80, description="Maximum thread messages to scan when capturing.")
max_source_items: int = Field(default=20, description="Maximum source items recorded per note.")
related_notes_enabled: bool = Field(default=True, description="On capture, find existing notes on the same topic and link them under ## Related (real cross-references instead of placeholders).")
related_notes_limit: int = Field(default=3, description="Maximum real related notes linked into each captured note.")
multi_note_enabled: bool = Field(default=True, description="Allow the LLM extractor to split one conversation into multiple cross-linked knowledge notes.")
rag_enabled: bool = Field(default=False, description="Enable RAG injection before agent runs (phase 2).")
rag_limit: int = Field(default=5, description="Maximum knowledge snippets injected per query.")
sensitive_redaction_enabled: bool = Field(default=True, description="Redact secrets/PII (tokens, keys, phone, id-card, email) before persisting notes (phase 3).")
dedup_similarity_threshold: float = Field(default=0.82, description="Title/summary similarity above which two notes are flagged as duplicates (phase 3).")
entity_extraction_enabled: bool = Field(default=True, description="Extract entities/relations during LLM distillation and feed them into the graph (phase 4).")
export: KnowledgeExportConfig = Field(default_factory=KnowledgeExportConfig, description="Graph export configuration.")
query: KnowledgeQueryConfig = Field(default_factory=KnowledgeQueryConfig, description="Layered query configuration.")
embedding: KnowledgeEmbeddingConfig = Field(default_factory=KnowledgeEmbeddingConfig, description="Embedding configuration.")
@@ -0,0 +1,41 @@
"""Product knowledge base module (obsidian-wiki compatible).
Self-contained: knowledge ingestion, storage (DB mirror + Markdown vault),
search and graph export all live here. The Gateway only depends on
:class:`KnowledgeService` via :func:`make_knowledge_service`.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.service import EmptyThreadError, KnowledgeService
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.knowledge_config import KnowledgeConfig
__all__ = [
"EmptyThreadError",
"KnowledgeRepository",
"KnowledgeService",
"ObsidianWikiAdapter",
"make_knowledge_service",
]
def make_knowledge_service(
session_factory: async_sessionmaker[AsyncSession] | None,
config: KnowledgeConfig,
) -> KnowledgeService | None:
"""Build the knowledge service.
Returns ``None`` when persistence is memory-only (no session factory) or
the feature is disabled — the router then responds 503.
"""
if session_factory is None or not config.enabled:
return None
return KnowledgeService(KnowledgeRepository(session_factory), config)
@@ -0,0 +1,125 @@
"""Background auto-sediment queue (phase 3).
A single asyncio worker drains a bounded queue of finished conversations and
sediments the worthwhile ones into the knowledge base — entirely off the chat
request path, so it never adds latency to a reply. Enqueue happens from
``KnowledgeRagMiddleware.after_agent`` when ``knowledge.auto_ingest_enabled``.
Value judgment is intentionally cheap (min assistant chars + dedup via
``skip_existing``); the LLM still distills the kept conversations via the normal
``capture_thread`` path.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
logger = logging.getLogger(__name__)
_QUEUE: AutoIngestQueue | None = None
@dataclass
class _QueueUser:
"""Minimal CurrentUser stand-in (only ``.id`` is needed for context)."""
id: str
@dataclass
class AutoIngestItem:
thread_id: str
user_id: str
messages: list[dict[str, Any]]
thread_title: str | None = None
reference_batches: list[dict[str, Any]] = field(default_factory=list)
class AutoIngestQueue:
"""Bounded background queue + single worker for auto-sedimentation."""
def __init__(self, service_provider: Callable[[], Any], *, min_chars: int = 300, maxsize: int = 256) -> None:
self._provider = service_provider
self._min_chars = min_chars
self._queue: asyncio.Queue[AutoIngestItem] = asyncio.Queue(maxsize=maxsize)
self._task: asyncio.Task | None = None
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._worker(), name="knowledge-auto-ingest")
logger.info("Knowledge auto-ingest worker started")
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._task
self._task = None
def enqueue(self, item: AutoIngestItem) -> None:
try:
self._queue.put_nowait(item)
except asyncio.QueueFull:
logger.warning("Knowledge auto-ingest queue full; dropping thread %s", item.thread_id)
def _worth_ingesting(self, item: AutoIngestItem) -> bool:
chars = 0
for m in item.messages:
if str(m.get("type") or m.get("role") or "").lower() == "ai":
content = m.get("content")
chars += len(content) if isinstance(content, str) else len(str(content or ""))
return chars >= self._min_chars
async def _process(self, item: AutoIngestItem) -> None:
service = self._provider()
if service is None:
return
if not self._worth_ingesting(item):
logger.debug("Auto-ingest skipped (below min chars) thread=%s", item.thread_id)
return
from deerflow.runtime.user_context import reset_current_user, set_current_user
token = set_current_user(_QueueUser(id=item.user_id))
try:
result = await service.capture_thread(
thread_id=item.thread_id,
messages=item.messages,
thread_title=item.thread_title,
status="draft", # auto-sedimented notes land as drafts for review
include_sources=True,
reference_batches=item.reference_batches,
created_by=item.user_id,
skip_existing=True,
)
if result.get("created"):
logger.info("Auto-ingested thread %s -> note %s", item.thread_id, result["note"]["id"])
except Exception:
logger.exception("Auto-ingest failed for thread %s", item.thread_id)
finally:
reset_current_user(token)
async def _worker(self) -> None:
while True:
item = await self._queue.get()
try:
await self._process(item)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Auto-ingest worker error")
finally:
self._queue.task_done()
def set_auto_ingest_queue(queue: AutoIngestQueue | None) -> None:
global _QUEUE
_QUEUE = queue
def get_auto_ingest_queue() -> AutoIngestQueue | None:
return _QUEUE
@@ -0,0 +1,60 @@
"""Markdown chunking for embedding/index (phase 2).
Splits a note body into overlapping chunks along Markdown structure (headings →
paragraphs) so each embedded chunk is semantically coherent and bounded in
size. Frontmatter is assumed already stripped by the caller.
"""
from __future__ import annotations
import re
_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
def _split_paragraphs(text: str) -> list[str]:
return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
"""Return a list of text chunks for ``content``.
Sections are cut at Markdown headings; long sections are further split into
~``max_chars`` windows with ``overlap`` characters of carry-over.
"""
text = (content or "").strip()
if not text:
return []
# Split into heading-delimited sections, keeping the heading with its body.
sections: list[str] = []
last = 0
for m in _HEADING_RE.finditer(text):
if m.start() > last:
sections.append(text[last : m.start()].strip())
last = m.start()
sections.append(text[last:].strip())
sections = [s for s in sections if s]
chunks: list[str] = []
for section in sections:
if len(section) <= max_chars:
chunks.append(section)
continue
# Pack paragraphs into windows, then hard-split anything still too long.
buf = ""
for para in _split_paragraphs(section):
if len(buf) + len(para) + 2 <= max_chars:
buf = f"{buf}\n\n{para}".strip()
else:
if buf:
chunks.append(buf)
if len(para) <= max_chars:
buf = para
else:
for i in range(0, len(para), max_chars - overlap):
chunks.append(para[i : i + max_chars])
buf = ""
if buf:
chunks.append(buf)
return [c for c in chunks if c.strip()]
@@ -0,0 +1,75 @@
"""Duplicate detection for knowledge notes (phase 3).
First-release heuristic: token Jaccard over ``title + summary + tags`` plus an
exact ``content_md`` hash match. Cheap, deterministic and embedding-free so it
works on offline deployments. Returns *suggestions*; merging is an explicit,
user-driven action.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.manifest import content_hash
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
def _tokens(*texts: str) -> set[str]:
out: set[str] = set()
for t in texts:
out.update(m.lower() for m in _TOKEN_RE.findall(t or ""))
return out
def _jaccard(a: set[str], b: set[str]) -> float:
if not a or not b:
return 0.0
inter = len(a & b)
union = len(a | b)
return inter / union if union else 0.0
def find_duplicate_groups(notes: list[dict[str, Any]], *, threshold: float = 0.82, max_notes: int = 800) -> list[dict[str, Any]]:
"""Return groups ``{primary, duplicates:[{id,title,score,reason}]}``.
``primary`` is the most recently updated note in each cluster.
"""
notes = notes[:max_notes]
feats: list[tuple[dict[str, Any], set[str], str]] = []
for n in notes:
toks = _tokens(n.get("title") or "", n.get("summary") or "", " ".join(n.get("tags") or []))
feats.append((n, toks, content_hash(n.get("content_md") or "")))
used: set[str] = set()
groups: list[dict[str, Any]] = []
for i in range(len(feats)):
ni, ti, hi = feats[i]
if ni["id"] in used:
continue
dups: list[dict[str, Any]] = []
for j in range(i + 1, len(feats)):
nj, tj, hj = feats[j]
if nj["id"] in used:
continue
if hi and hi == hj:
score, reason = 1.0, "identical-content"
else:
score = _jaccard(ti, tj)
reason = "similar-title-summary"
if score >= threshold:
dups.append({"id": nj["id"], "title": nj.get("title"), "score": round(score, 4), "reason": reason})
used.add(nj["id"])
if dups:
used.add(ni["id"])
# Primary = newest by updated_at.
cluster = [ni, *[next(f[0] for f in feats if f[0]["id"] == d["id"]) for d in dups]]
primary = max(cluster, key=lambda x: x.get("updated_at") or "")
others = [
{"id": c["id"], "title": c.get("title"), "score": next((d["score"] for d in dups if d["id"] == c["id"]), 1.0)}
for c in cluster
if c["id"] != primary["id"]
]
groups.append({"primary": {"id": primary["id"], "title": primary.get("title")}, "duplicates": others})
return groups
@@ -0,0 +1,79 @@
"""Embedding generation for vector RAG (phase 2).
Talks to an OpenAI-compatible ``/embeddings`` endpoint configured under
``knowledge.embedding``. Everything here is **best-effort**: if embeddings are
disabled, unconfigured, or the endpoint errors, callers get an empty result and
the search layer falls back to keyword scoring — so vector RAG never breaks a
deployment that has no embedding model.
"""
from __future__ import annotations
import logging
import math
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deerflow.config.knowledge_config import KnowledgeEmbeddingConfig
logger = logging.getLogger(__name__)
def cosine_similarity(a: list[float], b: list[float]) -> float:
"""Cosine similarity of two equal-length vectors (0 when degenerate)."""
if not a or not b or len(a) != len(b):
return 0.0
dot = sum(x * y for x, y in zip(a, b, strict=False))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
if na == 0 or nb == 0:
return 0.0
return dot / (na * nb)
class EmbeddingClient:
"""Thin async client over an OpenAI-compatible embeddings endpoint."""
def __init__(self, config: KnowledgeEmbeddingConfig) -> None:
self.config = config
def is_enabled(self) -> bool:
return bool(self.config.enabled and self.config.base_url and self.config.model)
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
"""Return one vector per input text; ``[]`` on any failure/disabled."""
if not self.is_enabled() or not texts:
return []
try:
import httpx
except Exception: # pragma: no cover - httpx is a core dep
logger.warning("httpx unavailable; knowledge embeddings disabled")
return []
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
if self.config.api_key:
headers["Authorization"] = f"Bearer {self.config.api_key}"
vectors: list[list[float]] = []
batch = max(1, self.config.batch_size)
try:
async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
for start in range(0, len(texts), batch):
chunk = texts[start : start + batch]
payload: dict = {"model": self.config.model, "input": chunk}
if self.config.dimensions:
payload["dimensions"] = self.config.dimensions
resp = await client.post(url, headers=headers, json=payload)
resp.raise_for_status()
data = resp.json()
items = sorted(data.get("data", []), key=lambda d: d.get("index", 0))
vectors.extend([item.get("embedding", []) for item in items])
except Exception:
logger.exception("Knowledge embedding request failed; falling back to keyword search")
return []
return vectors
async def embed_query(self, text: str) -> list[float]:
vecs = await self.embed_texts([text])
return vecs[0] if vecs else []
@@ -0,0 +1,227 @@
"""obsidian-wiki ``wiki-export`` compatible graph generation.
Builds a knowledge graph from the notes (nodes) plus their wikilinks and shared
tags (edges) and writes the four native artifacts into ``<vault>/wiki-export/``:
``graph.json``, ``graph.graphml``, ``cypher.txt`` and a self-contained
``graph.html`` (no external CDN, safe to iframe offline).
"""
from __future__ import annotations
import html
import json
from pathlib import Path
from typing import Any
from deerflow.knowledge.markdown_vault import extract_wikilinks
def _vault_key(vault_path: str | None) -> str | None:
if not vault_path:
return None
key = vault_path.replace("\\", "/")
if key.endswith(".md"):
key = key[:-3]
return key
def build_graph(
notes: list[dict[str, Any]],
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
Note nodes come from ``notes`` (id, title, tags, vault_path, content_md);
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
note→entity / entity→entity edges are added on top of the wikilink and
shared-tag edges.
"""
nodes = []
by_key: dict[str, str] = {}
note_ids: set[str] = set()
for n in notes:
nodes.append({"id": n["id"], "type": "note", "label": n.get("title") or n["id"], "category": n.get("source_type")})
note_ids.add(n["id"])
key = _vault_key(n.get("vault_path"))
if key:
by_key[key] = n["id"]
# Only surface entities that are still mentioned by a LIVE note. Entities whose
# only referencing notes were deleted/archived would otherwise linger forever
# as orphan nodes ("graph doesn't follow my deletions").
mentioned: set[str] = set()
for rel in relations or []:
if rel.get("from_note_id") in note_ids and rel.get("to_entity_id"):
mentioned.add(rel["to_entity_id"])
entity_ids: set[str] = set()
for ent in entities or []:
if ent["id"] not in mentioned:
continue
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
entity_ids.add(ent["id"])
edges: list[dict[str, Any]] = []
edge_seen: set[tuple[str, str, str]] = set()
def add_edge(a: str, b: str, label: str, weight: float) -> None:
if a == b:
return
key = (min(a, b), max(a, b), label)
if key in edge_seen:
return
edge_seen.add(key)
edges.append({"id": f"rel_{len(edges) + 1}", "source": a, "target": b, "label": label, "weight": weight})
# Wikilink edges.
for n in notes:
for target in extract_wikilinks(n.get("content_md") or ""):
tid = by_key.get(target.replace("\\", "/").removesuffix(".md"))
if tid:
add_edge(n["id"], tid, "links", 2)
# Shared-tag edges.
tag_to_notes: dict[str, list[str]] = {}
for n in notes:
for tag in n.get("tags") or []:
tag_to_notes.setdefault(tag.lower(), []).append(n["id"])
for tag, ids in tag_to_notes.items():
if len(ids) < 2 or len(ids) > 50: # skip ubiquitous tags (e.g. knowledge-base)
continue
for i in range(len(ids)):
for j in range(i + 1, len(ids)):
add_edge(ids[i], ids[j], "related", 1)
# Entity / relation edges (phase 4) — only when both endpoints exist as nodes.
valid = note_ids | entity_ids
for rel in relations or []:
src = rel.get("from_note_id") or rel.get("from_entity_id")
tgt = rel.get("to_note_id") or rel.get("to_entity_id")
if src in valid and tgt in valid:
add_edge(src, tgt, rel.get("relation_type") or "related", float(rel.get("weight") or 1))
return {"nodes": nodes, "edges": edges}
def to_graphml(graph: dict[str, Any]) -> str:
lines = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
' <key id="label" for="node" attr.name="label" attr.type="string"/>',
' <key id="elabel" for="edge" attr.name="label" attr.type="string"/>',
' <graph edgedefault="undirected">',
]
for n in graph["nodes"]:
lines.append(f' <node id="{html.escape(str(n["id"]))}"><data key="label">{html.escape(str(n["label"]))}</data></node>')
for e in graph["edges"]:
lines.append(
f' <edge id="{html.escape(str(e["id"]))}" source="{html.escape(str(e["source"]))}" '
f'target="{html.escape(str(e["target"]))}"><data key="elabel">{html.escape(str(e["label"]))}</data></edge>'
)
lines += [" </graph>", "</graphml>"]
return "\n".join(lines)
def to_cypher(graph: dict[str, Any]) -> str:
lines = []
for n in graph["nodes"]:
label = str(n["label"]).replace("'", "\\'")
lines.append(f"CREATE (:Note {{id:'{n['id']}', title:'{label}'}});")
for e in graph["edges"]:
rel = str(e["label"]).upper()
lines.append(f"MATCH (a:Note {{id:'{e['source']}'}}),(b:Note {{id:'{e['target']}'}}) CREATE (a)-[:{rel} {{weight:{e['weight']}}}]->(b);")
return "\n".join(lines)
def to_html(graph: dict[str, Any]) -> str:
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
data_json = json.dumps(graph, ensure_ascii=False)
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
def export_all(
vault_root: Path | str,
notes: list[dict[str, Any]],
*,
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
write_html: bool = True,
write_json: bool = True,
) -> dict[str, Any]:
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
out_dir = Path(vault_root).resolve() / "wiki-export"
out_dir.mkdir(parents=True, exist_ok=True)
graph = build_graph(notes, entities, relations)
written: list[str] = []
if write_json:
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
written.append("graph.json")
(out_dir / "graph.graphml").write_text(to_graphml(graph), encoding="utf-8")
written.append("graph.graphml")
(out_dir / "cypher.txt").write_text(to_cypher(graph), encoding="utf-8")
written.append("cypher.txt")
if write_html:
(out_dir / "graph.html").write_text(to_html(graph), encoding="utf-8")
written.append("graph.html")
return {
"files": written,
"stats": {"nodes": len(graph["nodes"]), "edges": len(graph["edges"])},
"graph": graph,
}
_HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Knowledge Graph</title>
<style>
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
canvas{display:block}
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
</style>
</head>
<body>
<div id="hud"></div>
<div id="tip"></div>
<canvas id="c"></canvas>
<script>
const GRAPH = __GRAPH_DATA__;
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
let W=0,H=0;
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
addEventListener('resize',resize);resize();
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
let cam={x:0,y:0,k:1},drag=null,pan=null;
function step(){
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
}
function draw(){
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
ctx.strokeStyle='rgba(120,150,255,.25)';
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
}
function loop(){step();draw();requestAnimationFrame(loop);}loop();
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
addEventListener('mouseup',()=>{drag=null;pan=null;});
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
</script>
</body>
</html>"""
@@ -0,0 +1,262 @@
"""Extract knowledge notes from threads / searches / tool results.
First-release strategy is rule-based (wiki-capture aligned, §11 of the dev doc):
keep high-value content (user questions, final assistant answers, cited
sources), drop noise (empty/greeting/intermediate reasoning), and write a
*declarative* note rather than a chat transcript. An LLM extractor can later
replace :func:`extract_thread` while keeping the same :class:`NoteDraft` output.
"""
from __future__ import annotations
import re
from typing import Any
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
# Greeting / low-value lines filtered from extracted turns.
_GREETING_RE = re.compile(r"^(你好|您好|hi|hello|hey|thanks?|谢谢|好的|ok|okay)[\s!!。.,,~]*$", re.IGNORECASE)
_WS_RE = re.compile(r"\s+")
def _one_line(text: str, limit: int = 300) -> str:
"""Collapse whitespace and clip ``text`` to a single line."""
collapsed = _WS_RE.sub(" ", text or "").strip()
return collapsed[:limit]
def message_text(msg: dict[str, Any]) -> str:
"""Return the plain-text content of a serialized LangChain message."""
content = msg.get("content")
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
if block.get("type") in (None, "text") and isinstance(block.get("text"), str):
parts.append(block["text"])
return "\n".join(p for p in parts if p).strip()
return ""
def _msg_type(msg: dict[str, Any]) -> str:
return str(msg.get("type") or msg.get("role") or "").lower()
def split_turns(messages: list[dict[str, Any]], *, max_messages: int = 80) -> list[dict[str, str]]:
"""Split messages into ``{user, ai}`` turns keyed by human messages.
For each turn we keep the last substantial AI message (with text body).
"""
msgs = messages[-max_messages:] if max_messages and len(messages) > max_messages else list(messages)
turns: list[dict[str, str]] = []
current: dict[str, str] | None = None
for msg in msgs:
mtype = _msg_type(msg)
text = message_text(msg)
if mtype == "human":
if current and (current.get("user") or current.get("ai")):
turns.append(current)
current = {"user": text, "ai": ""}
elif mtype == "ai" and text:
if current is None:
current = {"user": "", "ai": text}
else:
current["ai"] = text # last substantial AI message wins
if current and (current.get("user") or current.get("ai")):
turns.append(current)
# Drop turns that are pure greetings with no answer.
cleaned = []
for t in turns:
user = (t.get("user") or "").strip()
ai = (t.get("ai") or "").strip()
if not ai and _GREETING_RE.match(user):
continue
if not user and not ai:
continue
cleaned.append({"user": user, "ai": ai})
return cleaned
def _auto_title(thread_title: str | None, turns: list[dict[str, str]]) -> str:
if thread_title and thread_title.strip() and thread_title.strip().lower() != "untitled":
return thread_title.strip()[:200]
for t in turns:
if t.get("user"):
first = t["user"].splitlines()[0].strip()
return (first[:40] + ("…" if len(first) > 40 else "")) or "知识笔记"
return "知识笔记"
def _summary(turns: list[dict[str, str]], *, limit: int = 280) -> str:
"""Simple rule-based summary: the first answer, clipped."""
for t in turns:
if t.get("ai"):
text = re.sub(r"\s+", " ", t["ai"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
for t in turns:
if t.get("user"):
text = re.sub(r"\s+", " ", t["user"]).strip()
return text[:limit] + ("…" if len(text) > limit else "")
return ""
def related_links(project_name: str | None) -> list[str]: # noqa: ARG001
"""Structural wikilinks every note used to carry.
Product decision: notes no longer auto-link the vault meta/hub pages
(``index`` / ``_meta/taxonomy`` / the project hub) — they were noise in every
note's ``## Related``. Only real, content-specific related notes (injected by
the service layer, which has DB access) are linked now. Kept as a function
(returns empty) so existing callers and the wiki-capture contract stay intact.
"""
return []
# Backwards-compatible internal alias.
_related_links = related_links
def _related_section_md(related_lines: list[str], source_lines: list[str]) -> str:
"""Render the ``## Related`` section, or ``""`` when there is nothing to show."""
if not related_lines and not source_lines:
return ""
inner = "\n".join(related_lines)
if source_lines:
sources_block = "### Sources\n\n" + "\n".join(source_lines)
inner = (inner + "\n\n" + sources_block) if inner else sources_block
return f"## Related\n\n{inner}\n"
def _build_body(
*,
title: str,
turns: list[dict[str, str]],
mode: str,
project_name: str | None,
sources: list[SourceDraft],
) -> str:
"""Render the wiki-capture 5-section declarative body."""
full = mode == "full"
answer_clip = 4000 if full else 800
context_lines = []
for t in turns:
if t.get("user"):
q = re.sub(r"\s+", " ", t["user"]).strip()
context_lines.append(f"- {q[:300]}")
context = "\n".join(context_lines[:8]) or "- 来自一次对话沉淀。^[inferred]"
finding_lines = []
for i, t in enumerate(turns, 1):
if t.get("ai"):
ans = t["ai"].strip()
ans = ans if full else (ans[:answer_clip] + ("…" if len(ans) > answer_clip else ""))
finding_lines.append(f"### 结论 {i}\n\n{ans}")
finding = "\n\n".join(finding_lines) or "_暂无可提炼的结论。_"
reasoning = "本页由对话自动沉淀而成,结论来自助手最终回答;过程性推理已省略。判断与归纳部分标记为 ^[inferred]。"
implications = "- 后续可在相关问答中参考本页知识。\n- 如内容过期或冲突,请编辑或归档。"
related_lines = [f"- {link}" for link in _related_links(project_name)]
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
related_section = _related_section_md(related_lines, source_lines)
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{finding}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"{related_section}"
)
def extract_thread(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
) -> NoteDraft | None:
"""Build a :class:`NoteDraft` from thread messages, or ``None`` if empty."""
turns = split_turns(messages, max_messages=max_messages)
if not turns:
return None
final_title = (title or "").strip() or _auto_title(thread_title, turns)
summary = _summary(turns)
sources = sources or []
body = _build_body(title=final_title, turns=turns, mode=mode, project_name=project_name, sources=sources)
base_tags = list(tags or [])
for default in (project_name or "zncm", "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=summary,
content_md=body,
category="synthesis",
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=0.6,
related=_related_links(project_name),
sources=sources,
source_key=f"thread:{thread_id}",
)
def extract_search(
query: str,
sources: list[SourceDraft],
*,
thread_id: str | None = None,
project_name: str | None = "zncm",
title: str | None = None,
tags: list[str] | None = None,
) -> NoteDraft:
"""Build a reference-category :class:`NoteDraft` from search/tool results."""
final_title = (title or "").strip() or f"检索:{query[:40]}"
body_lines = [f"# {final_title}\n", "## Context\n", f"- 检索关键词:{query}\n", "## Finding / Decision\n"]
for s in sources:
label = s.title or s.url or "来源"
line = f"- **{label}**"
if s.url:
line += f" — {s.url}"
if s.snippet:
line += "\n - " + _one_line(s.snippet)
body_lines.append(line)
body_lines += [
"\n## Reasoning\n",
"本页汇总外部检索来源,供后续问答引用。来源可信度需结合原文判断。^[ambiguous]\n",
"## Implications\n",
"- 引用前请核对原始来源。\n",
"## Related\n",
*[f"- {link}" for link in _related_links(project_name)],
]
base_tags = list(tags or [])
for default in (project_name or "zncm", "knowledge-base", "reference"):
if default and default not in base_tags:
base_tags.append(default)
return NoteDraft(
title=final_title,
summary=f"检索“{query}”的来源汇总,共 {len(sources)} 条。",
content_md="\n".join(body_lines),
category="references",
source_type="search",
source_id=f"search:{query[:120]}",
tags=base_tags,
confidence=0.4,
related=_related_links(project_name),
sources=sources,
source_key=None,
)
@@ -0,0 +1,354 @@
"""LLM-based knowledge extraction (wiki-capture style).
Turns a conversation into a *declarative* knowledge note instead of dumping the
raw chat. Follows the obsidian-wiki ``wiki-capture`` skill: classify the note,
keep only high-value findings/decisions, drop greetings / intermediate
reasoning / verbatim transcripts, mark inferred content, and emit structured
JSON that we render into the Context / Finding / Reasoning / Implications /
Related body.
Falls back to the rule-based :func:`deerflow.knowledge.extractor.extract_thread`
when the LLM is unavailable or returns something unusable, so capture never
hard-fails.
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any
from deerflow.knowledge import extractor
from deerflow.knowledge.schemas import NoteDraft, SourceDraft
from deerflow.knowledge.skill_templates import WIKI_CAPTURE_GUIDANCE
logger = logging.getLogger(__name__)
_VALID_CATEGORIES = {"synthesis", "concepts", "references", "decision", "journal"}
class LLMRejectedIngest(Exception):
"""The LLM judged the conversation as having no knowledge value.
Raised (instead of returning ``[]``) when the model explicitly sets
``should_ingest: false`` with no notes — so callers can *skip* ingestion
entirely rather than falling back to the rule-based verbatim dump.
"""
def _is_falsey(value: Any) -> bool:
if value is False:
return True
if isinstance(value, str):
return value.strip().lower() in ("false", "0", "no")
return False
_SYSTEM_PROMPT = """\
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一次对话提炼成\
**陈述式知识(declarative knowledge)**,而不是聊天流水账。
要求:
- 只保留高价值内容:明确结论、方案、决策、关键事实、被引用的来源。
- 丢弃:寒暄、客套、中间推理过程、重复内容、无意义的原文堆砌。
- 不要写“用户问了什么、AI 回答了什么”,要写“结论是什么、为什么成立”。
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
references(外部来源汇总)、decision(架构/设计决策)、journal(会话纪要)。
- 用中文输出。
一次对话可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
- 默认只产出 **1 页**;只有当对话明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
- 每一页聚焦单一主题,标题与 summary 必须彼此区分,不要把同一结论拆到多页。
- 不值得沉淀时返回空的 notes 数组。
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构如下:
{
"should_ingest": true,
"reason": "为什么值得/不值得沉淀",
"notes": [
{
"title": "简洁的知识标题(不超过 40 字)",
"summary": "1-2 句话概括这页知识",
"category": "synthesis",
"context": "问题背景或触发场景,1-3 句",
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
"reasoning": "为什么成立、取舍、不确定性",
"implications": ["后续建议/风险/下一步", "..."],
"tags": ["标签1", "标签2"],
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
"confidence": 0.0
}
]
}"""
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
parts: list[str] = []
for i, t in enumerate(turns, 1):
if t.get("user"):
parts.append(f"[用户 {i}] {t['user']}")
if t.get("ai"):
parts.append(f"[助手 {i}] {t['ai']}")
if sources:
parts.append("\n[来源]")
for s in sources[:20]:
label = s.title or s.url or s.tool_name or "来源"
parts.append(f"- {label}" + (f" ({s.url})" if s.url else ""))
text = "\n\n".join(parts)
if len(text) > max_chars:
text = text[:max_chars] + "\n…(已截断)"
return text
def _parse_json(raw: str) -> dict[str, Any] | None:
if not raw:
return None
text = raw.strip()
# Strip ```json fences if present.
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
if fence:
text = fence.group(1).strip()
# Otherwise grab the outermost { ... }.
if not text.startswith("{"):
start = text.find("{")
end = text.rfind("}")
if start >= 0 and end > start:
text = text[start : end + 1]
try:
data = json.loads(text)
return data if isinstance(data, dict) else None
except json.JSONDecodeError:
logger.warning("Knowledge LLM returned non-JSON output; falling back to rule-based")
return None
def _render_body(data: dict[str, Any], title: str, related: list[str], sources: list[SourceDraft]) -> str:
def _bullets(items: Any) -> str:
if isinstance(items, str):
items = [items]
if not isinstance(items, list):
return ""
return "\n".join(f"- {str(x).strip()}" for x in items if str(x).strip())
context = str(data.get("context") or "").strip() or "- 来自一次对话沉淀。^[inferred]"
findings = _bullets(data.get("findings")) or "_暂无可提炼的结论。_"
reasoning = str(data.get("reasoning") or "").strip() or "(未提供推理说明)"
implications = _bullets(data.get("implications")) or "- 如内容过期或冲突,请编辑或归档。"
related_lines = [f"- {link}" for link in related]
source_lines = [f"- {s.title or s.url or '来源'}" + (f" — {s.url}" if s.url else "") for s in sources[:20]]
related_section = extractor._related_section_md(related_lines, source_lines)
return (
f"# {title}\n\n"
f"## Context\n\n{context}\n\n"
f"## Finding / Decision\n\n{findings}\n\n"
f"## Reasoning\n\n{reasoning}\n\n"
f"## Implications\n\n{implications}\n\n"
f"{related_section}"
)
def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
"""Normalize the LLM JSON into a list of per-note dicts.
Accepts the multi-note shape ``{"notes": [...]}`` as well as the legacy
single-note shape (``{"title": ..., "findings": ...}``) for backward
compatibility. Notes without any title/findings are dropped.
"""
raw_notes = data.get("notes")
if isinstance(raw_notes, list):
candidates = [n for n in raw_notes if isinstance(n, dict)]
elif data.get("title") or data.get("findings") or data.get("context"):
candidates = [data] # legacy single-object output
else:
candidates = []
return [n for n in candidates if str(n.get("title") or "").strip() or n.get("findings")]
def _build_note_draft(
note_data: dict[str, Any],
*,
thread_id: str,
thread_title: str | None,
turns: list[dict[str, str]],
title_override: str | None,
tags: list[str] | None,
project_name: str | None,
sources: list[SourceDraft],
source_key_suffix: str = "",
) -> NoteDraft:
"""Render one note dict into a :class:`NoteDraft`."""
category = str(note_data.get("category") or "synthesis").strip().lower()
if category not in _VALID_CATEGORIES:
category = "synthesis"
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
summary = str(note_data.get("summary") or "").strip()
related = extractor.related_links(project_name)
body = _render_body(note_data, final_title, related, sources)
base_tags = list(tags or [])
llm_tags = note_data.get("tags") if isinstance(note_data.get("tags"), list) else []
for tag in [*llm_tags, project_name or "zncm", "knowledge-base"]:
tag = str(tag).strip()
if tag and tag not in base_tags:
base_tags.append(tag)
try:
confidence = float(note_data.get("confidence") or 0.7)
except (TypeError, ValueError):
confidence = 0.7
entities = note_data.get("entities") if isinstance(note_data.get("entities"), list) else []
relations = note_data.get("relations") if isinstance(note_data.get("relations"), list) else []
return NoteDraft(
title=final_title,
summary=summary or final_title,
content_md=body,
category=category,
source_type="thread",
source_id=thread_id,
tags=base_tags,
confidence=max(0.0, min(confidence, 1.0)),
related=related,
sources=sources,
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
source_key=f"thread:{thread_id}{source_key_suffix}",
)
async def extract_thread_llm_multi(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
allow_multi: bool = True,
max_notes: int = 4,
) -> list[NoteDraft]:
"""Distill a thread into one *or several* cross-linkable knowledge notes.
The LLM may split a conversation that covers multiple independent topics into
several declarative notes. Returns ``[]`` when there is nothing to sediment
or the LLM path fails — callers should fall back to the rule-based extractor.
When ``allow_multi`` is false only the first note is kept.
"""
turns = extractor.split_turns(messages, max_messages=max_messages)
if not turns:
return []
sources = sources or []
try:
from langchain_core.messages import HumanMessage, SystemMessage
from deerflow.models import create_chat_model
model = create_chat_model(name=model_name or None, thinking_enabled=False)
transcript = _build_transcript(turns, sources, max_chars=max_input_chars)
prompt = [
SystemMessage(content=_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
HumanMessage(content=f"以下是一次对话,请按规范提炼为知识 JSON:\n\n{transcript}"),
]
response = await model.ainvoke(prompt)
raw = response.content if isinstance(response.content, str) else str(response.content)
data = _parse_json(raw)
except Exception:
logger.exception("[knowledge] thread %s — LLM extraction call failed; using rule-based fallback", thread_id)
return []
if not data:
logger.warning("[knowledge] thread %s — LLM returned no parsable JSON; using rule-based fallback", thread_id)
return []
note_dicts = _notes_from_data(data)
if not note_dicts:
reason = str(data.get("reason") or "")[:200]
if _is_falsey(data.get("should_ingest")):
# The model deliberately judged this conversation worthless (greeting,
# meta chatter, no findings). Skip ingestion entirely — do NOT fall
# back to the rule-based verbatim dump.
logger.warning("[knowledge] thread %s SKIPPED — model judged not worth ingesting: %s", thread_id, reason)
raise LLMRejectedIngest(reason or "no knowledge value")
logger.warning(
"[knowledge] thread %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
thread_id,
data.get("should_ingest"),
reason,
)
return []
if not allow_multi:
note_dicts = note_dicts[:1]
else:
note_dicts = note_dicts[: max(1, max_notes)]
drafts: list[NoteDraft] = []
for i, nd in enumerate(note_dicts):
# The user-supplied title override only applies to a single-note result;
# multi-note splits keep each note's own LLM title.
override = title if (len(note_dicts) == 1) else None
suffix = "" if len(note_dicts) == 1 else f"#{i}"
drafts.append(
_build_note_draft(
nd,
thread_id=thread_id,
thread_title=thread_title,
turns=turns,
title_override=override,
tags=tags,
project_name=project_name,
sources=sources,
source_key_suffix=suffix,
)
)
return drafts
async def extract_thread_llm(
messages: list[dict[str, Any]],
*,
thread_id: str,
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
tags: list[str] | None = None,
project_name: str | None = "zncm",
sources: list[SourceDraft] | None = None,
max_messages: int = 80,
model_name: str | None = None,
max_input_chars: int = 12000,
) -> NoteDraft | None:
"""Distill a thread into a single :class:`NoteDraft` (backward-compatible).
Thin wrapper over :func:`extract_thread_llm_multi` that returns the first
note, or ``None`` when nothing was produced.
"""
try:
drafts = await extract_thread_llm_multi(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=project_name,
sources=sources,
max_messages=max_messages,
model_name=model_name,
max_input_chars=max_input_chars,
allow_multi=False,
)
except LLMRejectedIngest:
return None
return drafts[0] if drafts else None
@@ -0,0 +1,74 @@
""".manifest.json read/write for the vault.
The manifest records, per page, the source key, content hash, vault path and
update time — so batch ingest can skip already-sedimented threads (delta/dedup)
exactly like obsidian-wiki's native ``.manifest.json``.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Any
def content_hash(text: str) -> str:
"""Stable sha256 hex digest of ``text`` (used for dedup)."""
return hashlib.sha256((text or "").encode("utf-8")).hexdigest()
class Manifest:
"""JSON manifest stored at ``<vault>/.manifest.json``."""
def __init__(self, root: Path | str) -> None:
self.path = Path(root).resolve() / ".manifest.json"
def load(self) -> dict[str, Any]:
if not self.path.exists():
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
try:
data = json.loads(self.path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return {"version": 1, "updated": None, "pages": {}, "sources": {}}
data.setdefault("pages", {})
data.setdefault("sources", {})
return data
def save(self, data: dict[str, Any]) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def record_page(
self,
*,
note_id: str,
vault_path: str,
source_key: str | None,
hash_value: str,
updated: str,
) -> None:
data = self.load()
data["pages"][note_id] = {
"vault_path": vault_path,
"source": source_key,
"hash": hash_value,
"updated": updated,
}
if source_key:
data["sources"][source_key] = {"note_id": note_id, "hash": hash_value, "updated": updated}
data["updated"] = updated
self.save(data)
def remove_page(self, note_id: str) -> None:
data = self.load()
page = data["pages"].pop(note_id, None)
if page and page.get("source"):
data["sources"].pop(page["source"], None)
self.save(data)
def has_source(self, source_key: str) -> bool:
return source_key in self.load().get("sources", {})
def get_source(self, source_key: str) -> dict[str, Any] | None:
return self.load().get("sources", {}).get(source_key)
@@ -0,0 +1,213 @@
"""obsidian-wiki compatible Markdown vault writer/reader.
Responsibilities (dev doc §10 ``markdown_vault.py``):
- Generate obsidian-wiki compatible frontmatter.
- Produce safe, path-traversal-proof file names/paths.
- Write/update/read Markdown note files.
- Parse frontmatter + extract ``[[wikilinks]]``.
The vault layout follows §8 of the dev doc and stays compatible with
``ar9av/obsidian-wiki`` (concepts/ references/ synthesis/ journal/ projects/…,
plus index.md / log.md / hot.md / .manifest.json / _meta/taxonomy.md /
wiki-export/).
"""
from __future__ import annotations
import re
import unicodedata
from datetime import datetime
from pathlib import Path
from typing import Any
import yaml
_WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
_UNSAFE_CHARS_RE = re.compile(r"[^\w一-鿿-]+")
# Top-level vault directories that must exist (obsidian-wiki native layout).
_VAULT_DIRS = ("concepts", "entities", "skills", "references", "synthesis", "journal", "projects", "_meta", "wiki-export")
def slugify(title: str, *, max_len: int = 60) -> str:
"""Return a filesystem-safe slug (ASCII + CJK), never empty."""
text = unicodedata.normalize("NFKC", (title or "").strip()).lower()
text = text.replace(" ", "-")
text = _UNSAFE_CHARS_RE.sub("-", text).strip("-")
text = re.sub(r"-{2,}", "-", text)
if len(text) > max_len:
text = text[:max_len].rstrip("-")
return text or "note"
def extract_wikilinks(text: str) -> list[str]:
"""Return the targets of every ``[[wikilink]]`` in ``text`` (de-duplicated)."""
seen: list[str] = []
for match in _WIKILINK_RE.finditer(text or ""):
target = match.group(1).strip()
if target and target not in seen:
seen.append(target)
return seen
def normalize_wikilink_target(target: str) -> str:
"""Canonicalize a ``[[wikilink]]`` target for resolution/comparison.
Drops any ``|alias`` part, normalizes slashes, strips a leading ``/`` and a
trailing ``.md`` — so ``[[index]]``, ``index.md`` and ``/index`` all map to
the same canonical ``index`` key used by the index/graph/resolver layers.
"""
t = (target or "").strip()
if not t:
return ""
t = t.split("|", 1)[0].strip()
t = t.replace("\\", "/").lstrip("/")
if t.lower().endswith(".md"):
t = t[:-3]
return t
class VaultPathError(ValueError):
"""Raised when a requested vault path escapes the vault root."""
class MarkdownVault:
"""Reads/writes Markdown files inside a single vault root directory."""
def __init__(self, root: Path | str) -> None:
self.root = Path(root).resolve()
# -- path safety ------------------------------------------------------
def resolve(self, rel_path: str) -> Path:
"""Resolve ``rel_path`` under the vault root, blocking traversal."""
rel = str(rel_path).replace("\\", "/").lstrip("/")
candidate = (self.root / rel).resolve()
try:
candidate.relative_to(self.root)
except ValueError as exc:
raise VaultPathError(f"Path escapes vault root: {rel_path!r}") from exc
return candidate
def note_relpath(self, *, category: str, project_name: str | None, slug: str, note_id: str) -> str:
"""Build a vault-relative path for a note.
Project knowledge goes under ``projects/<project>/<category>/`` per
wiki-capture; a short id suffix guarantees uniqueness.
"""
from deerflow.knowledge.skill_templates import category_to_dir
cat_dir = category_to_dir(category)
safe_slug = slugify(slug)
suffix = note_id.split("_")[-1][:8]
filename = f"{safe_slug}-{suffix}.md"
if project_name:
return f"projects/{slugify(project_name)}/{cat_dir}/{filename}"
return f"{cat_dir}/{filename}"
# -- scaffold ---------------------------------------------------------
def ensure_scaffold(self, *, project_name: str | None = None) -> None:
"""Create the vault directory tree + base files if missing."""
self.root.mkdir(parents=True, exist_ok=True)
for d in _VAULT_DIRS:
(self.root / d).mkdir(parents=True, exist_ok=True)
if project_name:
base = self.root / "projects" / slugify(project_name)
for d in ("concepts", "references", "synthesis", "journal"):
(base / d).mkdir(parents=True, exist_ok=True)
# Base vault files.
index = self.root / "index.md"
if not index.exists():
index.write_text("# Knowledge Index\n\n_页面索引。每次写入新页面后自动更新。_\n\n", encoding="utf-8")
log = self.root / "log.md"
if not log.exists():
log.write_text("# Activity Log\n\n_CAPTURE / INGEST / QUERY / EXPORT 记录。_\n\n", encoding="utf-8")
hot = self.root / "hot.md"
if not hot.exists():
hot.write_text("# Hot\n\n_近期活动摘要。_\n\n", encoding="utf-8")
taxonomy = self.root / "_meta" / "taxonomy.md"
if not taxonomy.exists():
taxonomy.write_text(
"# Taxonomy\n\n- synthesis — 多步骤分析、方案、结论\n- concepts — 概念、框架、模型\n"
"- references — 外部来源、搜索结果、文章资料\n- decision — 架构或设计决策\n- journal — 完整会话摘要\n",
encoding="utf-8",
)
# -- frontmatter ------------------------------------------------------
@staticmethod
def build_frontmatter(
*,
note_id: str,
title: str,
category: str,
tags: list[str],
sources: list[str],
summary: str,
created: str,
updated: str,
base_confidence: float,
lifecycle: str,
provenance: dict[str, float] | None = None,
relationships: list[dict[str, str]] | None = None,
) -> dict[str, Any]:
"""Build an obsidian-wiki / wiki-capture compatible frontmatter dict."""
created_date = (created or "")[:10]
return {
"id": note_id,
"title": title,
"category": category,
"tags": list(tags or []),
"sources": list(sources or []),
"created": created,
"updated": updated,
"summary": summary or "",
"provenance": provenance or {"extracted": 0.7, "inferred": 0.2, "ambiguous": 0.1},
"base_confidence": round(float(base_confidence or 0.0), 2),
"lifecycle": lifecycle,
"lifecycle_changed": created_date,
"relationships": relationships or [],
}
# -- IO ---------------------------------------------------------------
def write_note(self, rel_path: str, frontmatter: dict[str, Any], body: str) -> Path:
"""Write ``frontmatter`` + ``body`` to ``rel_path`` (creates dirs)."""
path = self.resolve(rel_path)
path.parent.mkdir(parents=True, exist_ok=True)
fm = yaml.safe_dump(frontmatter, allow_unicode=True, sort_keys=False, default_flow_style=False)
path.write_text(f"---\n{fm}---\n\n{body.strip()}\n", encoding="utf-8")
return path
def read_note(self, rel_path: str) -> tuple[dict[str, Any], str] | None:
"""Return ``(frontmatter, body)`` for a note, or ``None`` if missing."""
path = self.resolve(rel_path)
if not path.exists():
return None
return self.parse(path.read_text(encoding="utf-8"))
@staticmethod
def parse(text: str) -> tuple[dict[str, Any], str]:
"""Split raw Markdown into ``(frontmatter dict, body)``."""
match = _FRONTMATTER_RE.match(text or "")
if not match:
return {}, text or ""
try:
fm = yaml.safe_load(match.group(1)) or {}
except yaml.YAMLError:
fm = {}
if not isinstance(fm, dict):
fm = {}
return fm, match.group(2) or ""
def delete_note(self, rel_path: str) -> None:
"""Remove a note file if present (used on hard delete; archive keeps it)."""
path = self.resolve(rel_path)
if path.exists():
path.unlink()
def now_iso_with_tz(dt: datetime) -> str:
"""ISO-8601 string with timezone offset (frontmatter ``created``/``updated``)."""
if dt.tzinfo is None:
return dt.isoformat()
return dt.isoformat(timespec="seconds")
@@ -0,0 +1,141 @@
"""ORM models for the product knowledge base.
Tables mirror the obsidian-wiki vault into the database so the product can
list, filter, search and audit notes quickly without re-parsing Markdown. The
vault Markdown remains the source-of-truth for ``content_md`` / frontmatter /
graph; these rows are an index + audit layer.
The knowledge base is **global / shared** — there is no ``user_id`` isolation.
Writes record ``created_by`` / ``updated_by`` for auditing only.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class KnowledgeNoteRow(Base):
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
__tablename__ = "knowledge_notes"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
source_type: Mapped[str] = mapped_column(String(64), index=True)
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
confidence: Mapped[float] = mapped_column(Float, default=0.0)
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
updated_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), index=True)
class KnowledgeSourceRow(Base):
"""Knowledge source table (mirrors ``knowledge_sources``)."""
__tablename__ = "knowledge_sources"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
source_type: Mapped[str] = mapped_column(String(64))
thread_id: Mapped[str | None] = mapped_column(String(128), index=True)
message_id: Mapped[str | None] = mapped_column(String(128))
tool_name: Mapped[str | None] = mapped_column(String(128))
title: Mapped[str | None] = mapped_column(String(512))
url: Mapped[str | None] = mapped_column(Text)
snippet: Mapped[str | None] = mapped_column(Text)
raw_json: Mapped[str | None] = mapped_column(PortableLongText())
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeTagRow(Base):
"""Tag table (mirrors ``knowledge_tags``)."""
__tablename__ = "knowledge_tags"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(128), unique=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeNoteTagRow(Base):
"""Note ↔ tag join table (mirrors ``knowledge_note_tags``)."""
__tablename__ = "knowledge_note_tags"
__table_args__ = (UniqueConstraint("note_id", "tag_id", name="uq_knowledge_note_tags"),)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), primary_key=True)
tag_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_tags.id", ondelete="CASCADE"), primary_key=True)
class KnowledgeEmbeddingRow(Base):
"""Per-chunk embedding index (phase 2, mirrors ``knowledge_embeddings``).
The vector is stored as a JSON float array; first-release similarity is
computed in Python (no external vector DB). ``vector_id`` is reserved for a
future external store mapping.
"""
__tablename__ = "knowledge_embeddings"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
chunk_index: Mapped[int] = mapped_column(Integer, default=0)
content: Mapped[str] = mapped_column(Text)
vector: Mapped[list] = mapped_column(PortableJSON(), default=list)
vector_id: Mapped[str | None] = mapped_column(String(256))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeNoteVersionRow(Base):
"""Immutable snapshot of a note before each update (phase 3 versioning)."""
__tablename__ = "knowledge_note_versions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
note_id: Mapped[str] = mapped_column(String(64), ForeignKey("knowledge_notes.id", ondelete="CASCADE"), index=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
status: Mapped[str] = mapped_column(String(32))
tags_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
edited_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
class KnowledgeEntityRow(Base):
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
__tablename__ = "knowledge_entities"
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), index=True)
entity_type: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class KnowledgeRelationRow(Base):
"""Relation between notes/entities (phase 4, mirrors ``knowledge_relations``)."""
__tablename__ = "knowledge_relations"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
from_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
to_note_id: Mapped[str | None] = mapped_column(String(64), index=True)
from_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
to_entity_id: Mapped[str | None] = mapped_column(String(64), index=True)
relation_type: Mapped[str] = mapped_column(String(128))
weight: Mapped[float] = mapped_column(Float, default=1.0)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,254 @@
"""ObsidianWikiAdapter — the obsidian-wiki vault facade.
Owns the Markdown vault: writes notes with wiki-capture frontmatter/body, keeps
``index.md`` / ``log.md`` / ``hot.md`` / ``.manifest.json`` in sync after every
write, and produces wiki-export graph artifacts. Output stays compatible with
``ar9av/obsidian-wiki`` without ever invoking its CLI.
This class is storage-only (no DB, no LLM). The :class:`KnowledgeService`
combines it with the repository + extractor.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from deerflow.knowledge import exporter
from deerflow.knowledge.manifest import Manifest, content_hash
from deerflow.knowledge.markdown_vault import MarkdownVault
from deerflow.knowledge.schemas import NoteDraft
_INDEX_PAGES_HEADER = "## Pages"
_HOT_HEADER = "## Recent"
_HOT_MAX = 20
class ObsidianWikiAdapter:
"""Read/write an obsidian-wiki compatible vault."""
def __init__(self, vault_root: Path | str, *, project_name: str = "zncm", export_html: bool = True, export_json: bool = True) -> None:
self.vault = MarkdownVault(vault_root)
self.root = self.vault.root
self.manifest = Manifest(self.root)
self.project_name = project_name
self.export_html = export_html
self.export_json = export_json
def ensure_ready(self) -> None:
self.vault.ensure_scaffold(project_name=self.project_name)
# -- writes -----------------------------------------------------------
def write_note(
self,
note_id: str,
draft: NoteDraft,
*,
created: str,
updated: str,
status: str,
) -> str:
"""Write the note file, sync vault metadata, return the vault path."""
self.ensure_ready()
rel = self.vault.note_relpath(category=draft.category, project_name=self.project_name, slug=draft.title, note_id=note_id)
sources_fm = self._frontmatter_sources(draft)
relationships = [{"target": link, "type": "related"} for link in (draft.related or [])[:8]]
frontmatter = self.vault.build_frontmatter(
note_id=note_id,
title=draft.title,
category=draft.category,
tags=draft.tags,
sources=sources_fm,
summary=draft.summary,
created=created,
updated=updated,
base_confidence=draft.confidence,
lifecycle=status,
relationships=relationships,
)
self.vault.write_note(rel, frontmatter, draft.content_md)
self.manifest.record_page(
note_id=note_id,
vault_path=rel,
source_key=draft.source_key,
hash_value=content_hash(draft.content_md),
updated=updated,
)
self._update_index(note_id, rel, draft.title, draft.summary)
self._append_log("CAPTURE" if draft.source_type in ("thread", "search", "tool") else "INGEST", note_id, draft.title, updated)
self._update_hot(draft.title, rel, updated)
return rel
def update_note_file(
self,
note_id: str,
vault_path: str | None,
*,
title: str,
content_md: str,
tags: list[str],
status: str,
summary: str,
category: str,
created: str,
updated: str,
) -> str:
"""Rewrite a note file in place (frontmatter refreshed), sync metadata."""
self.ensure_ready()
rel = vault_path
if not rel:
rel = self.vault.note_relpath(category=category, project_name=self.project_name, slug=title, note_id=note_id)
# Preserve original `sources`/`provenance`/`category` frontmatter when present.
existing = self.vault.read_note(rel)
sources_fm = existing[0].get("sources", []) if existing else []
provenance = existing[0].get("provenance") if existing else None
relationships = existing[0].get("relationships") if existing else []
effective_category = (existing[0].get("category") if existing else None) or category
frontmatter = self.vault.build_frontmatter(
note_id=note_id,
title=title,
category=effective_category,
tags=tags,
sources=sources_fm,
summary=summary,
created=created,
updated=updated,
base_confidence=float((existing[0].get("base_confidence") if existing else 0.0) or 0.0),
lifecycle=status,
provenance=provenance,
relationships=relationships,
)
self.vault.write_note(rel, frontmatter, content_md)
self.manifest.record_page(
note_id=note_id,
vault_path=rel,
source_key=self.manifest.load().get("pages", {}).get(note_id, {}).get("source"),
hash_value=content_hash(content_md),
updated=updated,
)
self._update_index(note_id, rel, title, summary)
self._append_log("UPDATE", note_id, title, updated)
self._update_hot(title, rel, updated)
return rel
def archive_note(self, note_id: str, vault_path: str | None, *, updated: str) -> None:
"""Mark a note archived in its frontmatter + log (soft delete)."""
if vault_path:
parsed = self.vault.read_note(vault_path)
if parsed:
fm, body = parsed
fm["lifecycle"] = "archived"
fm["updated"] = updated
self.vault.write_note(vault_path, fm, body)
self._remove_from_index(note_id)
self._append_log("ARCHIVE", note_id, "", updated)
def export_graph(
self,
notes: list[dict[str, Any]],
*,
entities: list[dict[str, Any]] | None = None,
relations: list[dict[str, Any]] | None = None,
updated: str,
) -> dict[str, Any]:
self.ensure_ready()
result = exporter.export_all(
self.root,
notes,
entities=entities,
relations=relations,
write_html=self.export_html,
write_json=self.export_json,
)
self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
return result
def export_file_path(self, filename: str) -> Path | None:
allowed = {"graph.json", "graph.graphml", "cypher.txt", "graph.html"}
if filename not in allowed:
return None
path = (self.root / "wiki-export" / filename).resolve()
try:
path.relative_to(self.root)
except ValueError:
return None
return path if path.exists() else None
# -- frontmatter source serialization ---------------------------------
@staticmethod
def _frontmatter_sources(draft: NoteDraft) -> list[str]:
out: list[str] = []
if draft.source_type == "thread" and draft.source_id:
out.append(f"thread:{draft.source_id}")
for s in draft.sources[:20]:
if s.url:
out.append(f"url:{s.url}")
elif s.thread_id:
out.append(f"thread:{s.thread_id}")
elif s.tool_name:
out.append(f"tool:{s.tool_name}")
# De-dup preserving order.
seen: set[str] = set()
deduped = []
for item in out:
if item not in seen:
seen.add(item)
deduped.append(item)
return deduped
# -- index.md ---------------------------------------------------------
def _index_path(self) -> Path:
return self.root / "index.md"
def _update_index(self, note_id: str, rel: str, title: str, summary: str | None) -> None:
path = self._index_path()
text = path.read_text(encoding="utf-8") if path.exists() else "# Knowledge Index\n\n"
link = rel[:-3] if rel.endswith(".md") else rel
summary_txt = (summary or "").strip().replace("\n", " ")
if len(summary_txt) > 120:
summary_txt = summary_txt[:120] + "…"
line = f"- [[{link}|{title}]]" + (f" — {summary_txt}" if summary_txt else "") + f" <!-- id:{note_id} -->"
lines = text.splitlines()
# Drop any existing line for this note id.
lines = [ln for ln in lines if f"<!-- id:{note_id} -->" not in ln]
if _INDEX_PAGES_HEADER not in text:
lines.append("")
lines.append(_INDEX_PAGES_HEADER)
lines.append("")
# Insert after the Pages header.
try:
insert_at = lines.index(_INDEX_PAGES_HEADER) + 1
except ValueError:
insert_at = len(lines)
lines.insert(insert_at, line)
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
def _remove_from_index(self, note_id: str) -> None:
path = self._index_path()
if not path.exists():
return
lines = [ln for ln in path.read_text(encoding="utf-8").splitlines() if f"<!-- id:{note_id} -->" not in ln]
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
# -- log.md -----------------------------------------------------------
def _append_log(self, action: str, note_id: str, label: str, updated: str) -> None:
path = self.root / "log.md"
text = path.read_text(encoding="utf-8") if path.exists() else "# Activity Log\n\n"
entry = f"- {updated} {action} {note_id}" + (f" — {label}" if label else "")
path.write_text(text.rstrip() + "\n" + entry + "\n", encoding="utf-8")
# -- hot.md -----------------------------------------------------------
def _update_hot(self, title: str, rel: str, updated: str) -> None:
path = self.root / "hot.md"
link = rel[:-3] if rel.endswith(".md") else rel
new_line = f"- {updated} [[{link}|{title}]]"
existing_lines: list[str] = []
if path.exists():
for ln in path.read_text(encoding="utf-8").splitlines():
if ln.startswith("- ") and "[[" in ln:
existing_lines.append(ln)
existing_lines = [ln for ln in existing_lines if f"[[{link}" not in ln]
existing_lines.insert(0, new_line)
existing_lines = existing_lines[:_HOT_MAX]
body = "# Hot\n\n_近期活动摘要。_\n\n" + _HOT_HEADER + "\n\n" + "\n".join(existing_lines) + "\n"
path.write_text(body, encoding="utf-8")
@@ -0,0 +1,528 @@
"""Database read/write for the knowledge base.
This layer never calls an LLM and never touches the Markdown vault — it only
persists/queries the relational mirror (notes, sources, tags). The vault and
extraction live in :mod:`deerflow.knowledge.markdown_vault` and
:mod:`deerflow.knowledge.extractor`; orchestration lives in the service.
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.knowledge.models import (
KnowledgeEmbeddingRow,
KnowledgeEntityRow,
KnowledgeNoteRow,
KnowledgeNoteTagRow,
KnowledgeNoteVersionRow,
KnowledgeRelationRow,
KnowledgeSourceRow,
KnowledgeTagRow,
)
from deerflow.knowledge.schemas import SourceDraft
def _new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:24]}"
def _iso(value: Any) -> Any:
if isinstance(value, datetime):
return value.isoformat()
return value
class KnowledgeRepository:
"""SQLAlchemy-backed repository for knowledge notes/sources/tags."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# -- serialization ----------------------------------------------------
@staticmethod
def _note_to_dict(row: KnowledgeNoteRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
d["updated_at"] = _iso(d.get("updated_at"))
return d
@staticmethod
def _source_to_dict(row: KnowledgeSourceRow) -> dict[str, Any]:
d = row.to_dict()
d["created_at"] = _iso(d.get("created_at"))
return d
# -- tags -------------------------------------------------------------
async def _get_or_create_tags(self, session: AsyncSession, names: list[str]) -> list[KnowledgeTagRow]:
cleaned = []
seen: set[str] = set()
for raw in names or []:
name = (raw or "").strip()
if not name or name.lower() in seen:
continue
seen.add(name.lower())
cleaned.append(name)
if not cleaned:
return []
existing = {
r.name.lower(): r
for r in (await session.execute(select(KnowledgeTagRow).where(func.lower(KnowledgeTagRow.name).in_([n.lower() for n in cleaned])))).scalars()
}
result: list[KnowledgeTagRow] = []
for name in cleaned:
row = existing.get(name.lower())
if row is None:
row = KnowledgeTagRow(id=_new_id("kbtag"), name=name, created_at=datetime.now(UTC))
session.add(row)
await session.flush()
existing[name.lower()] = row
result.append(row)
return result
async def _tags_for(self, session: AsyncSession, note_id: str) -> list[str]:
stmt = (
select(KnowledgeTagRow.name)
.join(KnowledgeNoteTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(KnowledgeNoteTagRow.note_id == note_id)
.order_by(KnowledgeTagRow.name)
)
return list((await session.execute(stmt)).scalars())
async def _tags_for_many(self, session: AsyncSession, note_ids: list[str]) -> dict[str, list[str]]:
if not note_ids:
return {}
stmt = (
select(KnowledgeNoteTagRow.note_id, KnowledgeTagRow.name)
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(KnowledgeNoteTagRow.note_id.in_(note_ids))
.order_by(KnowledgeTagRow.name)
)
out: dict[str, list[str]] = {nid: [] for nid in note_ids}
for note_id, name in (await session.execute(stmt)).all():
out.setdefault(note_id, []).append(name)
return out
async def _replace_note_tags(self, session: AsyncSession, note_id: str, names: list[str]) -> None:
await session.execute(delete(KnowledgeNoteTagRow).where(KnowledgeNoteTagRow.note_id == note_id))
tags = await self._get_or_create_tags(session, names)
for tag in tags:
session.add(KnowledgeNoteTagRow(note_id=note_id, tag_id=tag.id))
# -- notes ------------------------------------------------------------
async def create_note(
self,
*,
title: str,
summary: str | None,
content_md: str,
source_type: str,
source_id: str | None,
status: str,
confidence: float = 0.0,
vault_path: str | None = None,
tags: list[str] | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
note_id = _new_id("kb")
now = datetime.now(UTC)
row = KnowledgeNoteRow(
id=note_id,
title=title[:512],
summary=summary,
content_md=content_md,
vault_path=vault_path,
source_type=source_type,
source_id=source_id,
status=status,
confidence=confidence,
created_by=created_by,
updated_by=created_by,
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.flush()
await self._replace_note_tags(session, note_id, tags or [])
await session.commit()
await session.refresh(row)
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
return d
async def set_vault_path(self, note_id: str, vault_path: str) -> None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return
row.vault_path = vault_path
await session.commit()
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
summary: str | None = None,
content_md: str | None = None,
status: str | None = None,
tags: list[str] | None = None,
vault_path: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return None
if title is not None:
row.title = title[:512]
if summary is not None:
row.summary = summary
if content_md is not None:
row.content_md = content_md
if status is not None:
row.status = status
if vault_path is not None:
row.vault_path = vault_path
if updated_by is not None:
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
if tags is not None:
await self._replace_note_tags(session, note_id, tags)
await session.commit()
await session.refresh(row)
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
return d
async def set_status(self, note_id: str, status: str, *, updated_by: str | None = None) -> dict[str, Any] | None:
return await self.update_note(note_id, status=status, updated_by=updated_by)
async def get_note(self, note_id: str, *, with_sources: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(KnowledgeNoteRow, note_id)
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, note_id)
if with_sources:
stmt = select(KnowledgeSourceRow).where(KnowledgeSourceRow.note_id == note_id).order_by(KnowledgeSourceRow.created_at)
d["sources"] = [self._source_to_dict(r) for r in (await session.execute(stmt)).scalars()]
return d
async def find_by_vault_path(self, vault_path: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note whose vault_path matches.
``vault_path`` is matched both with and without the ``.md`` suffix, since
wikilink targets are stored without it while ``vault_path`` keeps it.
"""
stripped = vault_path[:-3] if vault_path.endswith(".md") else vault_path
candidates = {vault_path, stripped, f"{stripped}.md"}
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(KnowledgeNoteRow.vault_path.in_(list(candidates)))
.where(KnowledgeNoteRow.status != "archived")
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, row.id)
return d
async def find_by_title(self, title: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note with a case-insensitive title match."""
name = (title or "").strip()
if not name:
return None
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(func.lower(KnowledgeNoteRow.title) == name.lower())
.where(KnowledgeNoteRow.status != "archived")
.order_by(KnowledgeNoteRow.updated_at.desc())
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
d = self._note_to_dict(row)
d["tags"] = await self._tags_for(session, row.id)
return d
async def find_by_source(self, source_type: str, source_id: str) -> dict[str, Any] | None:
"""Return the first (non-archived) note for a (source_type, source_id)."""
async with self._sf() as session:
stmt = (
select(KnowledgeNoteRow)
.where(KnowledgeNoteRow.source_type == source_type, KnowledgeNoteRow.source_id == source_id)
.where(KnowledgeNoteRow.status != "archived")
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
return self._note_to_dict(row) if row else None
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
source_type: str | None = None,
status: str | None = None,
include_archived: bool = False,
limit: int = 20,
offset: int = 0,
) -> tuple[list[dict[str, Any]], int]:
async with self._sf() as session:
stmt = select(KnowledgeNoteRow)
count_stmt = select(func.count()).select_from(KnowledgeNoteRow)
conditions = []
if status:
conditions.append(KnowledgeNoteRow.status == status)
elif not include_archived:
conditions.append(KnowledgeNoteRow.status != "archived")
if source_type:
conditions.append(KnowledgeNoteRow.source_type == source_type)
if keyword:
like = f"%{keyword}%"
conditions.append(or_(KnowledgeNoteRow.title.like(like), KnowledgeNoteRow.summary.like(like), KnowledgeNoteRow.content_md.like(like)))
if tag:
tag_subq = (
select(KnowledgeNoteTagRow.note_id)
.join(KnowledgeTagRow, KnowledgeNoteTagRow.tag_id == KnowledgeTagRow.id)
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
)
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
for cond in conditions:
stmt = stmt.where(cond)
count_stmt = count_stmt.where(cond)
total = (await session.execute(count_stmt)).scalar_one()
stmt = stmt.order_by(KnowledgeNoteRow.updated_at.desc()).limit(limit).offset(offset)
rows = list((await session.execute(stmt)).scalars())
ids = [r.id for r in rows]
tags_map = await self._tags_for_many(session, ids)
items = []
for r in rows:
d = self._note_to_dict(r)
d["tags"] = tags_map.get(r.id, [])
items.append(d)
return items, int(total)
async def list_all(self, *, include_archived: bool = False) -> list[dict[str, Any]]:
"""Return every note (light fields + tags) for graph export."""
async with self._sf() as session:
stmt = select(KnowledgeNoteRow)
if not include_archived:
stmt = stmt.where(KnowledgeNoteRow.status != "archived")
rows = list((await session.execute(stmt)).scalars())
tags_map = await self._tags_for_many(session, [r.id for r in rows])
out = []
for r in rows:
d = self._note_to_dict(r)
d["tags"] = tags_map.get(r.id, [])
out.append(d)
return out
# -- embeddings (phase 2) --------------------------------------------
async def replace_embeddings(self, note_id: str, chunks: list[tuple[int, str, list[float]]]) -> None:
"""Replace all embedding rows for a note with ``(index, content, vector)``."""
async with self._sf() as session:
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
for idx, content, vector in chunks:
session.add(
KnowledgeEmbeddingRow(
id=_new_id("kbemb"),
note_id=note_id,
chunk_index=idx,
content=content,
vector=list(vector),
created_at=datetime.now(UTC),
)
)
await session.commit()
async def delete_embeddings(self, note_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(KnowledgeEmbeddingRow).where(KnowledgeEmbeddingRow.note_id == note_id))
await session.commit()
async def all_embeddings(self) -> list[dict[str, Any]]:
"""Return every embedding chunk joined to its (non-archived) note."""
async with self._sf() as session:
stmt = (
select(KnowledgeEmbeddingRow, KnowledgeNoteRow.title, KnowledgeNoteRow.status)
.join(KnowledgeNoteRow, KnowledgeNoteRow.id == KnowledgeEmbeddingRow.note_id)
.where(KnowledgeNoteRow.status != "archived")
)
out = []
for emb, title, _status in (await session.execute(stmt)).all():
out.append({"note_id": emb.note_id, "chunk_index": emb.chunk_index, "content": emb.content, "vector": emb.vector or [], "title": title})
return out
async def has_embeddings(self) -> bool:
async with self._sf() as session:
row = (await session.execute(select(KnowledgeEmbeddingRow.id).limit(1))).first()
return row is not None
# -- versions (phase 3) ----------------------------------------------
async def add_version(self, note: dict[str, Any], *, edited_by: str | None) -> None:
async with self._sf() as session:
session.add(
KnowledgeNoteVersionRow(
id=_new_id("kbver"),
note_id=note["id"],
title=note.get("title") or "",
summary=note.get("summary"),
content_md=note.get("content_md") or "",
status=note.get("status") or "approved",
tags_json=note.get("tags") or [],
edited_by=edited_by,
created_at=datetime.now(UTC),
)
)
await session.commit()
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = select(KnowledgeNoteVersionRow).where(KnowledgeNoteVersionRow.note_id == note_id).order_by(KnowledgeNoteVersionRow.created_at.desc())
out = []
for r in (await session.execute(stmt)).scalars():
d = r.to_dict()
d["tags"] = d.pop("tags_json", [])
d["created_at"] = _iso(d.get("created_at"))
out.append(d)
return out
# -- entities & relations (phase 4) ----------------------------------
async def get_or_create_entity(self, name: str, entity_type: str | None) -> dict[str, Any]:
name = (name or "").strip()
async with self._sf() as session:
stmt = select(KnowledgeEntityRow).where(func.lower(KnowledgeEntityRow.name) == name.lower())
row = (await session.execute(stmt)).scalars().first()
if row is None:
row = KnowledgeEntityRow(id=_new_id("kbent"), name=name, entity_type=entity_type, created_at=datetime.now(UTC))
session.add(row)
await session.commit()
await session.refresh(row)
return {"id": row.id, "name": row.name, "entity_type": row.entity_type}
async def add_relation(
self,
*,
from_note_id: str | None = None,
to_note_id: str | None = None,
from_entity_id: str | None = None,
to_entity_id: str | None = None,
relation_type: str,
weight: float = 1.0,
) -> None:
async with self._sf() as session:
session.add(
KnowledgeRelationRow(
id=_new_id("kbrel"),
from_note_id=from_note_id,
to_note_id=to_note_id,
from_entity_id=from_entity_id,
to_entity_id=to_entity_id,
relation_type=relation_type,
weight=weight,
created_at=datetime.now(UTC),
)
)
await session.commit()
async def clear_note_relations(self, note_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(KnowledgeRelationRow).where(KnowledgeRelationRow.from_note_id == note_id))
await session.commit()
async def list_note_entities(self, note_id: str) -> list[dict[str, Any]]:
"""Return the entities mentioned by a note (for note-detail display)."""
async with self._sf() as session:
stmt = (
select(KnowledgeEntityRow.name, KnowledgeEntityRow.entity_type)
.join(KnowledgeRelationRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
.where(KnowledgeRelationRow.from_note_id == note_id)
.order_by(KnowledgeEntityRow.name)
)
seen: set[str] = set()
out: list[dict[str, Any]] = []
for name, etype in (await session.execute(stmt)).all():
key = (name or "").lower()
if not name or key in seen:
continue
seen.add(key)
out.append({"name": name, "entity_type": etype})
return out
async def prune_orphan_entities(self) -> int:
"""Delete entity rows no longer referenced by any relation. Returns count."""
async with self._sf() as session:
referenced: set[str] = set()
for col in (KnowledgeRelationRow.from_entity_id, KnowledgeRelationRow.to_entity_id):
for (eid,) in (await session.execute(select(col).where(col.isnot(None)))).all():
referenced.add(eid)
all_ids = list((await session.execute(select(KnowledgeEntityRow.id))).scalars())
orphans = [eid for eid in all_ids if eid not in referenced]
if orphans:
await session.execute(delete(KnowledgeEntityRow).where(KnowledgeEntityRow.id.in_(orphans)))
await session.commit()
return len(orphans)
async def list_entities(self) -> list[dict[str, Any]]:
async with self._sf() as session:
rows = (await session.execute(select(KnowledgeEntityRow))).scalars()
return [{"id": r.id, "name": r.name, "entity_type": r.entity_type} for r in rows]
async def list_relations(self) -> list[dict[str, Any]]:
async with self._sf() as session:
rows = (await session.execute(select(KnowledgeRelationRow))).scalars()
return [
{
"id": r.id,
"from_note_id": r.from_note_id,
"to_note_id": r.to_note_id,
"from_entity_id": r.from_entity_id,
"to_entity_id": r.to_entity_id,
"relation_type": r.relation_type,
"weight": r.weight,
}
for r in rows
]
# -- sources ----------------------------------------------------------
async def add_sources(self, note_id: str, sources: list[SourceDraft], *, created_by: str | None = None) -> None:
if not sources:
return
import json
async with self._sf() as session:
for src in sources:
session.add(
KnowledgeSourceRow(
id=_new_id("kbsrc"),
note_id=note_id,
source_type=src.source_type,
thread_id=src.thread_id,
message_id=src.message_id,
tool_name=src.tool_name,
title=(src.title or "")[:512] or None,
url=src.url,
snippet=src.snippet,
raw_json=json.dumps(src.raw, ensure_ascii=False) if src.raw else None,
created_by=created_by,
created_at=datetime.now(UTC),
)
)
await session.commit()
@@ -0,0 +1,75 @@
"""Internal domain models for the knowledge module.
These are plain dataclasses passed between the extractor, the vault writer and
the repository. The Gateway router defines its own Pydantic request/response
models on top of the dicts the service returns.
"""
from __future__ import annotations
from dataclasses import dataclass, field
# Allowed note source types (see product-knowledge-base-dev.md §7.1).
SOURCE_TYPES = ("thread", "search", "tool", "file", "manual", "report")
# Allowed lifecycle states.
STATUSES = ("draft", "approved", "archived")
# obsidian-wiki wiki-capture categories.
CATEGORIES = ("synthesis", "concepts", "references", "decision", "journal")
@dataclass
class SourceDraft:
"""A normalized knowledge source (search result, tool call, thread, …)."""
source_type: str
thread_id: str | None = None
message_id: str | None = None
tool_name: str | None = None
title: str | None = None
url: str | None = None
snippet: str | None = None
raw: dict | None = None
@dataclass
class NoteDraft:
"""A knowledge note ready to be persisted + written to the vault.
Produced by the extractor (from a thread / search / tool result) or built
directly for a manual note. ``content_md`` here is the *body* — the vault
writer prepends obsidian-wiki frontmatter.
"""
title: str
summary: str
content_md: str
category: str = "synthesis"
source_type: str = "manual"
source_id: str | None = None
status: str = "approved"
confidence: float = 0.0
tags: list[str] = field(default_factory=list)
key_points: list[str] = field(default_factory=list)
actions: list[str] = field(default_factory=list)
related: list[str] = field(default_factory=list)
sources: list[SourceDraft] = field(default_factory=list)
# Phase 4: extracted graph signal. entities=[{name,type}],
# relations=[{from,to,type}] (names refer to entities or this note's title).
entities: list[dict] = field(default_factory=list)
relations: list[dict] = field(default_factory=list)
# Stable identity hint used for manifest dedup (e.g. ``thread:thread_id``).
source_key: str | None = None
def normalize_status(status: str | None) -> str:
"""Coerce an arbitrary status string to a valid lifecycle state."""
if status in STATUSES:
return status
return "approved"
def normalize_source_type(source_type: str | None) -> str:
"""Coerce an arbitrary source-type string to a valid value."""
if source_type in SOURCE_TYPES:
return source_type
return "manual"
@@ -0,0 +1,160 @@
"""Knowledge search: keyword / vector / hybrid (phase 1 + 2).
- ``keyword`` — title>tags>summary>body scoring over the DB mirror (always works).
- ``vector`` — cosine over per-chunk embeddings (requires a configured
embedding endpoint; otherwise transparently falls back to ``keyword``).
- ``hybrid`` — min-max-normalized blend of keyword + vector.
"""
from __future__ import annotations
import re
from typing import TYPE_CHECKING, Any
from deerflow.knowledge.embeddings import cosine_similarity
from deerflow.knowledge.repository import KnowledgeRepository
if TYPE_CHECKING:
from deerflow.knowledge.embeddings import EmbeddingClient
_TOKEN_RE = re.compile(r"[\w一-鿿]+", re.UNICODE)
def _tokens(text: str) -> list[str]:
return [t.lower() for t in _TOKEN_RE.findall(text or "")]
def _snippet(content: str, terms: list[str], *, width: int = 160) -> str:
body = re.sub(r"\s+", " ", content or "").strip()
low = body.lower()
for term in terms:
idx = low.find(term)
if idx >= 0:
start = max(0, idx - width // 3)
end = min(len(body), start + width)
prefix = "…" if start > 0 else ""
suffix = "…" if end < len(body) else ""
return f"{prefix}{body[start:end]}{suffix}"
return body[:width] + ("…" if len(body) > width else "")
async def _keyword_scores(repo: KnowledgeRepository, query: str, *, candidate_cap: int) -> dict[str, tuple[float, dict[str, Any]]]:
terms = _tokens(query)
if not terms:
return {}
candidates, _total = await repo.list_notes(limit=candidate_cap, offset=0)
scored: dict[str, tuple[float, dict[str, Any]]] = {}
for note in candidates:
title = (note.get("title") or "").lower()
summary = (note.get("summary") or "").lower()
content = (note.get("content_md") or "").lower()
tags = " ".join(note.get("tags") or []).lower()
score = 0.0
for term in terms:
if term in title:
score += 3.0
if term in tags:
score += 2.0
if term in summary:
score += 2.0
score += min(content.count(term), 5) * 0.5
if score > 0:
scored[note["id"]] = (score, note)
return scored
async def _vector_scores(
repo: KnowledgeRepository,
query: str,
embedding_client: EmbeddingClient,
) -> dict[str, tuple[float, dict[str, Any]]]:
qvec = await embedding_client.embed_query(query)
if not qvec:
return {}
embeddings = await repo.all_embeddings()
best: dict[str, tuple[float, str, str]] = {} # note_id -> (score, title, snippet)
for emb in embeddings:
sim = cosine_similarity(qvec, emb.get("vector") or [])
if sim <= 0:
continue
nid = emb["note_id"]
if nid not in best or sim > best[nid][0]:
best[nid] = (sim, emb.get("title") or "", emb.get("content") or "")
return {nid: (score, {"id": nid, "title": title, "_chunk": chunk}) for nid, (score, title, chunk) in best.items()}
def _normalize(scores: dict[str, float]) -> dict[str, float]:
if not scores:
return {}
mx = max(scores.values())
if mx <= 0:
return {k: 0.0 for k in scores}
return {k: v / mx for k, v in scores.items()}
async def search_notes(
repo: KnowledgeRepository,
query: str,
*,
mode: str = "keyword",
limit: int = 8,
candidate_cap: int = 500,
embedding_client: EmbeddingClient | None = None,
) -> list[dict[str, Any]]:
"""Score notes against ``query`` and return the top ``limit`` matches."""
terms = _tokens(query)
if not terms:
return []
want_vector = mode in ("vector", "hybrid") and embedding_client is not None and embedding_client.is_enabled()
keyword = await _keyword_scores(repo, query, candidate_cap=candidate_cap) if mode != "vector" or not want_vector else {}
vector = await _vector_scores(repo, query, embedding_client) if want_vector else {}
# Pure vector (when available); else keyword; hybrid blends both.
if mode == "vector" and want_vector:
combined = {nid: (sc, meta) for nid, (sc, meta) in vector.items()}
ranked = sorted(combined.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
return await _materialize(repo, ranked, terms)
if mode == "hybrid" and want_vector:
kw_norm = _normalize({k: v[0] for k, v in keyword.items()})
vec_norm = _normalize({k: v[0] for k, v in vector.items()})
ids = set(kw_norm) | set(vec_norm)
blended: dict[str, tuple[float, dict[str, Any]]] = {}
for nid in ids:
score = 0.5 * kw_norm.get(nid, 0.0) + 0.5 * vec_norm.get(nid, 0.0)
meta = keyword.get(nid, (0, None))[1] or vector.get(nid, (0, {}))[1]
blended[nid] = (score, meta)
ranked = sorted(blended.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
return await _materialize(repo, ranked, terms)
# keyword (default, and fallback when vector unavailable)
ranked = sorted(keyword.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
if not ranked and vector: # mode=vector requested but keyword empty: use vector
ranked = sorted(vector.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
return await _materialize(repo, ranked, terms)
async def _materialize(repo: KnowledgeRepository, ranked: list[tuple[str, tuple[float, dict[str, Any]]]], terms: list[str]) -> list[dict[str, Any]]:
max_score = ranked[0][1][0] if ranked else 1.0
results: list[dict[str, Any]] = []
for nid, (score, meta) in ranked:
note = meta if (meta and meta.get("content_md")) else await repo.get_note(nid, with_sources=False)
if note is None:
continue
snippet_src = note.get("content_md") or meta.get("_chunk") or note.get("summary") or ""
results.append(
{
"note_id": nid,
"chunk_id": None,
"title": note.get("title") or meta.get("title"),
"summary": note.get("summary"),
"snippet": _snippet(snippet_src, terms),
"score": round(score / max_score, 4) if max_score else 0.0,
"tags": note.get("tags") or [],
"source_type": note.get("source_type"),
"updated_at": note.get("updated_at"),
"sources": [],
}
)
return results
@@ -0,0 +1,54 @@
"""Sensitive-information redaction for knowledge content (phase 3).
Best-effort masking of secrets / PII before a note is persisted: API keys,
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
ID-card numbers and email addresses. Conservative by design — it favours not
mangling legitimate prose over catching every possible secret.
"""
from __future__ import annotations
import re
_MASK = "[REDACTED]"
# Order matters: more specific patterns first.
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
# key/secret/password/token assignment: keep the field name, mask the value.
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
]
def redact(text: str | None) -> tuple[str, list[str]]:
"""Return ``(redacted_text, hit_kinds)``.
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
is falsy it is returned unchanged with no hits.
"""
if not text:
return text or "", []
hits: list[str] = []
out = text
for kind, pattern in _PATTERNS:
def _sub(m: re.Match[str], _kind: str = kind) -> str:
# For assignments keep the field name + delimiter, mask only the value.
if _kind == "assignment":
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
sep = ":" if ":" in m.group(0) else "="
return f"{head}{sep} {_MASK}"
if _kind == "bearer":
return f"Bearer {_MASK}"
return _MASK
new_out, n = pattern.subn(_sub, out)
if n:
hits.append(kind)
out = new_out
return out, hits
@@ -0,0 +1,689 @@
"""KnowledgeService — high-level orchestration for the knowledge base.
Ties together the repository (DB mirror), the :class:`ObsidianWikiAdapter`
(Markdown vault + index/log/hot/manifest + graph export) and the extractor
(thread/search → :class:`NoteDraft`). The Gateway router only talks to this
service; everything obsidian-wiki specific stays inside the ``knowledge``
module.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.runtime_paths import runtime_home
from deerflow.knowledge import extractor, llm_extractor
from deerflow.knowledge.chunking import chunk_markdown
from deerflow.knowledge.embeddings import EmbeddingClient
from deerflow.knowledge.markdown_vault import MarkdownVault, VaultPathError, extract_wikilinks, normalize_wikilink_target
from deerflow.knowledge.obsidian_adapter import ObsidianWikiAdapter
from deerflow.knowledge.repository import KnowledgeRepository
from deerflow.knowledge.schemas import NoteDraft, SourceDraft, normalize_source_type, normalize_status
from deerflow.knowledge.search import search_notes
from deerflow.knowledge.sources import from_reference_batches, from_search_results
logger = logging.getLogger(__name__)
class EmptyThreadError(ValueError):
"""Raised when a thread has no sediment-able content."""
def _resolve_vault_dir(vault_dir: str) -> Path:
path = Path(vault_dir)
if path.is_absolute():
return path.resolve()
return (runtime_home() / vault_dir).resolve()
def _inject_related_notes(body: str, links: list[str]) -> str:
"""Insert real related-note ``[[wikilinks]]`` into the Related block.
Notes carry only content-specific related notes now (the structural meta-links
were dropped), so this is the sole writer of ``## Related`` links. When the
body has no ``## Related`` section yet, one is appended.
"""
if not links:
return body
bullets = "\n".join(f"- {link}" for link in links)
marker = "## Related\n"
idx = body.find(marker)
if idx == -1:
return body.rstrip() + "\n\n## Related\n\n" + bullets + "\n"
insert_at = idx + len(marker)
return body[:insert_at] + "\n" + bullets + "\n" + body[insert_at:]
class KnowledgeService:
"""Application-facing knowledge base API."""
def __init__(self, repo: KnowledgeRepository, config: KnowledgeConfig) -> None:
self.repo = repo
self.config = config
self.project_name = config.project_name
vault_root = _resolve_vault_dir(config.vault_dir)
self.adapter = ObsidianWikiAdapter(
vault_root,
project_name=config.project_name,
export_html=config.export.graph_html_enabled,
export_json=config.export.graph_json_enabled,
)
self.embedding_client = EmbeddingClient(config.embedding)
try:
self.adapter.ensure_ready()
except Exception: # pragma: no cover - vault init best-effort
logger.exception("Failed to initialize knowledge vault at %s", vault_root)
# -- internal helpers -------------------------------------------------
def _redact_draft(self, draft: NoteDraft) -> NoteDraft:
"""Mask secrets/PII in a draft before persistence (phase 3)."""
if not self.config.sensitive_redaction_enabled:
return draft
from deerflow.knowledge.sensitive import redact
draft.content_md, body_hits = redact(draft.content_md)
draft.summary, _ = redact(draft.summary)
for src in draft.sources:
if src.snippet:
src.snippet, _ = redact(src.snippet)
if body_hits:
logger.info("Knowledge redaction applied to note '%s': %s", draft.title, ",".join(sorted(set(body_hits))))
return draft
async def _find_related_notes(self, draft: NoteDraft) -> list[dict[str, Any]]:
"""Find existing notes to cross-link under ## Related — by **shared tags**.
Matching is strictly tag-based: a candidate must share at least one of the
draft's *distinctive* tags (the per-note project / ``knowledge-base``
defaults are dropped, else everything matches everything). Free-text
title/summary matching was removed because CJK tokenization made generic
words (助手 / 知识图谱 / 页面) link unrelated notes. No distinctive tags →
no related notes (better empty than noisy). Best-effort.
"""
if not self.config.related_notes_enabled:
return []
default_tags = {t.lower() for t in (self.project_name or "", "knowledge-base", "zncm") if t}
distinctive = [t for t in (draft.tags or []) if t and t.lower() not in default_tags]
if not distinctive:
return []
own_title = (draft.title or "").strip().lower()
# Rank candidates by how many distinctive tags they share with the draft.
shared: dict[str, dict[str, Any]] = {}
for tag in distinctive:
try:
items, _total = await self.repo.list_notes(tag=tag, limit=50, offset=0)
except Exception:
logger.exception("Related-notes tag lookup failed for tag '%s'", tag)
continue
for it in items:
nid = it.get("id")
title = (it.get("title") or "").strip()
if not nid or not title or title.lower() == own_title:
continue
entry = shared.setdefault(nid, {"id": nid, "title": title, "count": 0})
entry["count"] += 1
ranked = sorted(shared.values(), key=lambda e: e["count"], reverse=True)
limit = max(1, self.config.related_notes_limit)
return [{"id": e["id"], "title": e["title"]} for e in ranked[:limit]]
async def _persist_draft(self, draft: NoteDraft, *, created_by: str | None) -> dict[str, Any]:
"""Create a DB row, write the vault file, attach sources, return note."""
draft = self._redact_draft(draft)
related_notes = await self._find_related_notes(draft)
if related_notes:
real_links = [f"[[{n['title']}]]" for n in related_notes]
draft.content_md = _inject_related_notes(draft.content_md, real_links)
draft.related = list(dict.fromkeys([*real_links, *draft.related]))
note = await self.repo.create_note(
title=draft.title,
summary=draft.summary,
content_md=draft.content_md,
source_type=draft.source_type,
source_id=draft.source_id,
status=draft.status,
confidence=draft.confidence,
tags=draft.tags,
created_by=created_by,
)
note_id = note["id"]
try:
rel = self.adapter.write_note(note_id, draft, created=note["created_at"], updated=note["updated_at"], status=draft.status)
await self.repo.set_vault_path(note_id, rel)
note["vault_path"] = rel
except Exception:
logger.exception("Failed to write knowledge note %s to vault", note_id)
if draft.sources:
await self.repo.add_sources(note_id, draft.sources, created_by=created_by)
await self._reindex_embeddings(note_id, draft.content_md)
await self._store_entities(note_id, draft)
# Record note→note "related" edges (after _store_entities, which clears
# this note's relations) so the graph reflects the cross-links too.
for rel_note in related_notes:
try:
await self.repo.add_relation(from_note_id=note_id, to_note_id=rel_note["id"], relation_type="related")
except Exception:
logger.exception("Failed to link related note %s -> %s", note_id, rel_note["id"])
return await self.repo.get_note(note_id) or note
async def _store_entities(self, note_id: str, draft: NoteDraft) -> None:
"""Persist extracted entities/relations and link them to the note (phase 4)."""
if not self.config.entity_extraction_enabled or not draft.entities:
return
try:
await self.repo.clear_note_relations(note_id)
name_to_id: dict[str, str] = {}
for ent in draft.entities:
name = str(ent.get("name") or "").strip()
if not name:
continue
row = await self.repo.get_or_create_entity(name, ent.get("type"))
name_to_id[name.lower()] = row["id"]
# note -> entity "mentions"
await self.repo.add_relation(from_note_id=note_id, to_entity_id=row["id"], relation_type="mentions")
for rel in draft.relations:
a = name_to_id.get(str(rel.get("from") or "").strip().lower())
b = name_to_id.get(str(rel.get("to") or "").strip().lower())
if a and b and a != b:
await self.repo.add_relation(from_entity_id=a, to_entity_id=b, relation_type=str(rel.get("type") or "relates_to"))
except Exception:
logger.exception("Failed to store entities/relations for note %s", note_id)
async def _reindex_embeddings(self, note_id: str, content_md: str) -> None:
"""Best-effort: (re)build the per-chunk embedding index for a note."""
if not self.embedding_client.is_enabled():
return
try:
chunks = chunk_markdown(content_md)
if not chunks:
await self.repo.delete_embeddings(note_id)
return
vectors = await self.embedding_client.embed_texts(chunks)
if not vectors or len(vectors) != len(chunks):
return
await self.repo.replace_embeddings(note_id, [(i, c, v) for i, (c, v) in enumerate(zip(chunks, vectors, strict=False))])
except Exception:
logger.exception("Failed to (re)index embeddings for note %s", note_id)
# -- capture ----------------------------------------------------------
async def capture_thread(
self,
*,
thread_id: str,
messages: list[dict[str, Any]],
thread_title: str | None = None,
mode: str = "summary",
title: str | None = None,
status: str = "approved",
tags: list[str] | None = None,
include_sources: bool = True,
reference_batches: list[dict[str, Any]] | None = None,
created_by: str | None = None,
skip_existing: bool = False,
model_name: str | None = None,
) -> dict[str, Any]:
"""Sediment a thread into a knowledge note.
``model_name`` optionally overrides the configured extraction model for
this one capture (falls back to ``config.extract_model_name``).
Returns ``{"note": <note>, "created": bool}``. Raises
:class:`EmptyThreadError` when there is nothing worth keeping.
"""
# Always-on entry log: proves a capture was triggered for this thread,
# regardless of how it ends (success / skip / fallback). Lets you tell
# "capture never ran" apart from "logs not visible" via one grep.
logger.info(
"[knowledge] capturing thread %s (msgs=%d, llm_extract=%s, skip_existing=%s)",
thread_id,
len(messages or []),
self.config.llm_extract_enabled,
skip_existing,
)
if skip_existing:
existing = await self.repo.find_by_source("thread", thread_id)
if existing is None and self.adapter.manifest.has_source(f"thread:{thread_id}"):
existing = await self.repo.find_by_source("thread", thread_id)
if existing:
logger.info("[knowledge] thread %s already captured (note %s); skipping", thread_id, existing.get("id"))
return {"note": existing, "created": False}
sources = []
if include_sources and reference_batches:
sources = from_reference_batches(reference_batches, thread_id=thread_id, max_items=self.config.max_source_items)
drafts: list[NoteDraft] = []
# Prefer LLM distillation (wiki-capture style) so notes are declarative
# knowledge, not raw chat. A single conversation may split into several
# cross-linked notes. Falls back to the rule-based extractor (always one
# note) when the LLM is disabled/unavailable or returns nothing usable.
llm_attempted = self.config.llm_extract_enabled
llm_fallback = False
if llm_attempted:
try:
drafts = await llm_extractor.extract_thread_llm_multi(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
model_name=model_name or self.config.extract_model_name,
max_input_chars=self.config.extract_max_input_chars,
allow_multi=self.config.multi_note_enabled,
)
except llm_extractor.LLMRejectedIngest as exc:
# The model judged this conversation worthless (greeting / meta
# chatter). Skip ingestion entirely — do NOT dump the raw chat.
raise EmptyThreadError(f"Thread {thread_id} judged not worth ingesting: {exc}") from exc
if not drafts:
# The LLM path was supposed to run but produced nothing usable
# (API error / non-JSON). We will fall back to the rule-based
# extractor, which yields a low-quality verbatim dump with no
# entities/tags — flag it so it lands as a draft.
llm_fallback = True
if not drafts:
single = extractor.extract_thread(
messages,
thread_id=thread_id,
thread_title=thread_title,
mode=mode,
title=title,
tags=tags,
project_name=self.project_name,
sources=sources,
max_messages=self.config.max_thread_messages,
)
drafts = [single] if single else []
if not drafts:
logger.info("[knowledge] thread %s has no sediment-able content; skipping", thread_id)
raise EmptyThreadError(f"Thread {thread_id} has no sediment-able content")
# When the LLM was enabled but silently fell back to rule-based extraction,
# the note is a verbatim dump (no distillation / entities / tags). Land it
# as a *draft* for review instead of silently approving it, and log loudly
# so the extraction failure is diagnosable (see also the WARNING/EXCEPTION
# logged inside llm_extractor.extract_thread_llm_multi).
effective_status = status
if llm_fallback:
logger.warning(
"[knowledge] thread %s — capture fell back to rule-based extraction "
"(LLM distillation produced nothing usable); landing as DRAFT. "
"Check extraction model '%s' / earlier [knowledge] logs for the cause.",
thread_id,
self.config.extract_model_name or "models[0]",
)
if normalize_status(status) == "approved":
effective_status = "draft"
normalized = normalize_status(effective_status)
for draft in drafts:
draft.status = normalized
# Cross-link sibling notes from the same conversation by title, so the
# split pages reference each other (resolved once all are persisted).
if len(drafts) > 1:
for i, draft in enumerate(drafts):
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
if siblings:
draft.content_md = _inject_related_notes(draft.content_md, siblings)
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
logger.info(
"[knowledge] thread %s captured → %d note(s) [%s], status=%s, llm_fallback=%s",
thread_id,
len(notes),
", ".join(n.get("title", "") for n in notes),
normalized,
llm_fallback,
)
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
async def capture_search(
self,
*,
query: str,
results: list[dict[str, Any]] | None,
thread_id: str | None = None,
message_id: str | None = None,
title: str | None = None,
tags: list[str] | None = None,
status: str = "approved",
created_by: str | None = None,
) -> dict[str, Any]:
"""Sediment search/tool results into a reference note."""
sources = from_search_results(query, results, thread_id=thread_id, message_id=message_id, max_items=self.config.max_source_items)
draft = extractor.extract_search(query, sources, thread_id=thread_id, project_name=self.project_name, title=title, tags=tags)
draft.status = normalize_status(status)
return await self._persist_draft(draft, created_by=created_by)
async def create_manual_note(
self,
*,
title: str,
content_md: str,
tags: list[str] | None = None,
status: str = "approved",
summary: str | None = None,
created_by: str | None = None,
) -> dict[str, Any]:
"""Create a note directly from product-edited Markdown."""
# Strip any frontmatter the user may have pasted; vault writer adds its own.
_, body = MarkdownVault.parse(content_md)
body = body or content_md
base_tags = list(tags or [])
for default in (self.project_name, "knowledge-base"):
if default and default not in base_tags:
base_tags.append(default)
draft = NoteDraft(
title=title.strip() or "知识笔记",
summary=(summary or body.strip().splitlines()[0] if body.strip() else "")[:280],
content_md=body,
category="synthesis",
source_type="manual",
source_id=None,
status=normalize_status(status),
tags=base_tags,
confidence=0.5,
related=extractor.related_links(self.project_name),
)
return await self._persist_draft(draft, created_by=created_by)
# -- CRUD -------------------------------------------------------------
async def list_notes(
self,
*,
keyword: str | None = None,
tag: str | None = None,
source_type: str | None = None,
status: str | None = None,
limit: int = 20,
offset: int = 0,
) -> dict[str, Any]:
items, total = await self.repo.list_notes(
keyword=keyword,
tag=tag,
source_type=normalize_source_type(source_type) if source_type else None,
status=status,
limit=limit,
offset=offset,
)
return {"items": items, "total": total}
async def get_note(self, note_id: str) -> dict[str, Any] | None:
note = await self.repo.get_note(note_id)
if note is not None:
note["entities"] = await self.repo.list_note_entities(note_id)
return note
async def update_note(
self,
note_id: str,
*,
title: str | None = None,
content_md: str | None = None,
tags: list[str] | None = None,
status: str | None = None,
summary: str | None = None,
updated_by: str | None = None,
) -> dict[str, Any] | None:
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return None
# Snapshot the pre-edit state for version history (phase 3).
try:
await self.repo.add_version(current, edited_by=updated_by)
except Exception:
logger.exception("Failed to snapshot version for note %s", note_id)
body = content_md
if body is not None:
_, parsed = MarkdownVault.parse(body)
body = parsed or body
note = await self.repo.update_note(
note_id,
title=title,
summary=summary,
content_md=body,
status=normalize_status(status) if status else None,
tags=tags,
updated_by=updated_by,
)
if note is None:
return None
try:
rel = self.adapter.update_note_file(
note_id,
current.get("vault_path"),
title=note["title"],
content_md=note["content_md"],
tags=note.get("tags") or [],
status=note["status"],
summary=note.get("summary") or "",
category="synthesis",
created=note["created_at"],
updated=note["updated_at"],
)
if rel and rel != current.get("vault_path"):
await self.repo.set_vault_path(note_id, rel)
except Exception:
logger.exception("Failed to update knowledge note %s in vault", note_id)
if note.get("content_md") is not None:
await self._reindex_embeddings(note_id, note["content_md"])
return await self.repo.get_note(note_id)
async def list_versions(self, note_id: str) -> list[dict[str, Any]]:
return await self.repo.list_versions(note_id)
async def delete_note(self, note_id: str, *, updated_by: str | None = None, hard: bool = False) -> bool:
"""Soft delete (archive) by default; hard delete removes the vault file."""
current = await self.repo.get_note(note_id, with_sources=False)
if current is None:
return False
note = await self.repo.set_status(note_id, "archived", updated_by=updated_by)
try:
self.adapter.archive_note(note_id, current.get("vault_path"), updated=(note or current)["updated_at"])
if hard and current.get("vault_path"):
self.adapter.vault.delete_note(current["vault_path"])
self.adapter.manifest.remove_page(note_id)
await self.repo.delete_embeddings(note_id)
except Exception:
logger.exception("Failed to archive knowledge note %s in vault", note_id)
# Drop this note's entity mentions + any now-orphaned entities so the graph
# follows the deletion (no lingering nodes from removed notes).
try:
await self.repo.clear_note_relations(note_id)
pruned = await self.repo.prune_orphan_entities()
if pruned:
logger.info("[knowledge] deleted note %s → pruned %d orphan entities", note_id, pruned)
except Exception:
logger.exception("Failed to prune relations/entities for deleted note %s", note_id)
return True
# -- wikilinks --------------------------------------------------------
async def resolve_wikilink(self, target: str) -> dict[str, Any]:
"""Resolve a single ``[[wikilink]]`` target to a navigable destination.
Returns one of:
- ``{"kind": "note", "note_id", "title", "exists": True}`` — a knowledge note
- ``{"kind": "vault", "vault_path", "title", "exists": True}`` — a vault meta/hub page
- ``{"kind": "missing", "exists": False}`` — nothing matches
Resolution order: note id → note vault_path → note title → vault file.
"""
norm = normalize_wikilink_target(target)
base = {"target": target, "normalized": norm}
if not norm:
return {**base, "kind": "missing", "exists": False}
# 1) Direct note id (e.g. "kb_abc...").
if norm.startswith("kb_"):
note = await self.repo.get_note(norm, with_sources=False)
if note and note.get("status") != "archived":
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 2) Note by vault path (links from index.md / hot.md / structural hub).
note = await self.repo.find_by_vault_path(norm)
if note:
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 3) Note by title (the real related-note cross-links use [[Title]]).
note = await self.repo.find_by_title(norm) or await self.repo.find_by_title(norm.split("/")[-1])
if note:
return {**base, "kind": "note", "note_id": note["id"], "title": note["title"], "exists": True}
# 4) A vault meta/hub page that has no DB note (index, _meta/taxonomy, hub…).
page = self.read_vault_page(norm)
if page is not None:
return {**base, "kind": "vault", "vault_path": page["path"], "title": page["title"], "exists": True}
return {**base, "kind": "missing", "exists": False}
async def resolve_wikilinks(self, targets: list[str]) -> dict[str, dict[str, Any]]:
"""Resolve many targets at once; returns ``{original_target: resolution}``."""
out: dict[str, dict[str, Any]] = {}
for target in dict.fromkeys(t for t in targets if t and t.strip()):
out[target] = await self.resolve_wikilink(target)
return out
async def resolve_note_wikilinks(self, note_id: str) -> dict[str, dict[str, Any]] | None:
"""Extract every ``[[wikilink]]`` in a note's body and resolve them.
Returns ``{target: resolution}`` or ``None`` when the note is missing.
Numeric ``[[123]]`` citation badges are skipped (handled by the renderer).
"""
note = await self.repo.get_note(note_id, with_sources=False)
if note is None:
return None
targets = [t for t in extract_wikilinks(note.get("content_md") or "") if not t.isdigit()]
return await self.resolve_wikilinks(targets)
def read_vault_page(self, path: str) -> dict[str, Any] | None:
"""Read a raw vault Markdown page (meta/hub pages without a DB note).
Returns ``{"path", "title", "content_md"}`` or ``None`` if the page does
not exist or the path escapes the vault root.
"""
norm = normalize_wikilink_target(path)
if not norm:
return None
rel = f"{norm}.md"
try:
parsed = self.adapter.vault.read_note(rel)
except VaultPathError:
return None
if parsed is None:
return None
fm, body = parsed
title = str(fm.get("title") or "").strip() or norm.split("/")[-1]
return {"path": norm, "title": title, "content_md": body}
# -- search & graph ---------------------------------------------------
async def search(self, query: str, *, mode: str = "keyword", limit: int = 8) -> list[dict[str, Any]]:
return await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
async def build_rag_context(self, query: str, *, limit: int = 5) -> dict[str, Any]:
"""Retrieve top knowledge for ``query`` and format an injectable block.
Returns ``{"text": <system context or "">, "items": [...]}``. Uses hybrid
search when embeddings are available, else keyword (wiki-query style).
"""
mode = "hybrid" if self.embedding_client.is_enabled() else "keyword"
items = await search_notes(self.repo, query, mode=mode, limit=limit, embedding_client=self.embedding_client)
if not items:
return {"text": "", "items": []}
lines = [
"以下是产品知识库中与本问题相关的资料。回答时优先参考;若使用其中信息,请在回答中标注来源(note_id)。",
"",
]
for i, it in enumerate(items, 1):
lines.append(f"[知识 {i}] {it.get('title') or ''}")
if it.get("snippet"):
lines.append(it["snippet"])
lines.append(f"来源:{it['note_id']}")
lines.append("")
return {"text": "\n".join(lines).strip(), "items": items}
# -- dedup & merge (phase 3) -----------------------------------------
async def find_duplicates(self) -> list[dict[str, Any]]:
from deerflow.knowledge.dedup import find_duplicate_groups
notes = await self.repo.list_all()
return find_duplicate_groups(notes, threshold=self.config.dedup_similarity_threshold)
async def merge_notes(self, primary_id: str, duplicate_ids: list[str], *, updated_by: str | None = None) -> dict[str, Any] | None:
"""Merge ``duplicate_ids`` into ``primary_id``.
Sources from each duplicate are re-attached to the primary, a wikilink
+ "Merged from" note is appended to the primary body, the duplicates are
archived (soft delete), and a ``merged_into`` relation is recorded.
"""
primary = await self.repo.get_note(primary_id)
if primary is None:
return None
merged_titles: list[str] = []
for dup_id in duplicate_ids:
if dup_id == primary_id:
continue
dup = await self.repo.get_note(dup_id)
if dup is None:
continue
# Move sources.
for src in dup.get("sources") or []:
await self.repo.add_sources(
primary_id,
[
SourceDraft(
source_type=src.get("source_type") or "tool",
thread_id=src.get("thread_id"),
message_id=src.get("message_id"),
tool_name=src.get("tool_name"),
title=src.get("title"),
url=src.get("url"),
snippet=src.get("snippet"),
)
],
created_by=updated_by,
)
await self.repo.add_relation(from_note_id=dup_id, to_note_id=primary_id, relation_type="merged_into")
await self.delete_note(dup_id, updated_by=updated_by)
merged_titles.append(dup.get("title") or dup_id)
if merged_titles:
appended = primary["content_md"].rstrip() + "\n\n## Merged\n\n" + "\n".join(f"- {t}" for t in merged_titles) + "\n"
await self.update_note(primary_id, content_md=appended, updated_by=updated_by)
return await self.repo.get_note(primary_id)
async def reindex_all(self) -> dict[str, int]:
"""Rebuild embeddings for every non-archived note (admin/maintenance)."""
notes = await self.repo.list_all()
count = 0
for note in notes:
await self._reindex_embeddings(note["id"], note.get("content_md") or "")
count += 1
return {"reindexed": count, "embedding_enabled": int(self.embedding_client.is_enabled())}
async def export_graph(self) -> dict[str, Any]:
from deerflow.utils.time import now_iso
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
return {"files": result["files"], "stats": result["stats"]}
async def get_graph(self) -> dict[str, Any]:
"""Return product graph data ``{nodes, edges}`` (notes + entities)."""
from deerflow.knowledge import exporter
notes = await self.repo.list_all()
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
return exporter.build_graph(notes, entities, relations)
def export_file_path(self, filename: str) -> Path | None:
return self.adapter.export_file_path(filename)
@@ -0,0 +1,51 @@
"""Built-in obsidian-wiki skill templates.
Mirrors the key intent of ``ar9av/obsidian-wiki`` skills so the backend can
write vault content that stays compatible with the original tooling without
shelling out to its CLI:
- ``wiki-capture`` — how a conversation becomes a *declarative* knowledge note
(frontmatter schema + Context/Finding/Reasoning/Implications/Related body).
- ``wiki-query`` — layered retrieval (hot.md → index.md → frontmatter → body).
- ``wiki-export`` — graph artifacts (graph.json / graph.html / graphml / cypher).
These are intentionally lightweight: the structures here are what
:mod:`deerflow.knowledge.markdown_vault` and :mod:`deerflow.knowledge.extractor`
produce, and what the first-release LLM prompt (future enhancement) should
target.
"""
from __future__ import annotations
# Category → vault sub-directory (wiki-capture page taxonomy, §8 of the dev doc).
CATEGORY_DIRS = {
"synthesis": "synthesis",
"concepts": "concepts",
"concept": "concepts",
"references": "references",
"reference": "references",
"source": "references",
"decision": "synthesis",
"journal": "journal",
"session": "journal",
}
# Body section headings (wiki-capture). Declarative knowledge, never a chat log.
BODY_SECTIONS = ("Context", "Finding / Decision", "Reasoning", "Implications", "Related")
# Inline provenance markers reused from wiki-capture.
INFERRED_MARKER = "^[inferred]"
AMBIGUOUS_MARKER = "^[ambiguous]"
WIKI_CAPTURE_GUIDANCE = """\
Capture *declarative* knowledge, not a chat transcript. Classify the note as one
of: synthesis, concept, source, decision, session. Prefer at least two
[[wikilinks]]. Mark inferred content with ^[inferred] and uncertain/conflicting
content with ^[ambiguous]. Project-specific knowledge belongs under
projects/<project-name>/...
"""
def category_to_dir(category: str | None) -> str:
"""Map a wiki-capture category to its vault sub-directory."""
return CATEGORY_DIRS.get((category or "").strip().lower(), "synthesis")
@@ -0,0 +1,82 @@
"""Normalize knowledge sources from threads / searches / tool results.
Source items come from a few shapes: backend reference-batches (built by
``deerflow.runtime.references``), raw search-result payloads, and tool-call
messages. They are all normalized into :class:`SourceDraft` so the repository
and frontend see one consistent structure.
"""
from __future__ import annotations
from typing import Any
from deerflow.knowledge.schemas import SourceDraft
def _clip(value: Any, limit: int = 2000) -> str | None:
if value is None:
return None
text = str(value).strip()
if not text:
return None
return text[:limit]
def from_reference_batches(batches: list[dict[str, Any]] | None, *, thread_id: str | None = None, max_items: int = 20) -> list[SourceDraft]:
"""Convert backend reference-batches into :class:`SourceDraft`."""
out: list[SourceDraft] = []
seen: set[str] = set()
for batch in batches or []:
msg_id = batch.get("assistant_message_id") or batch.get("id")
for src in batch.get("sources", []) or []:
if not isinstance(src, dict):
continue
url = _clip(src.get("url"), 1024)
title = _clip(src.get("title"), 512) or _clip(src.get("source"), 512) or "来源"
dedup_key = url or f"{title}:{_clip(src.get('snippet'), 80)}"
if dedup_key in seen:
continue
seen.add(dedup_key)
out.append(
SourceDraft(
source_type="search" if url else "tool",
thread_id=thread_id,
message_id=_clip(msg_id, 128),
tool_name=_clip(src.get("skillName"), 128),
title=title,
url=url,
snippet=_clip(src.get("snippet")),
raw=None,
)
)
if len(out) >= max_items:
return out
return out
def from_search_results(
query: str,
results: list[dict[str, Any]] | None,
*,
thread_id: str | None = None,
message_id: str | None = None,
max_items: int = 20,
) -> list[SourceDraft]:
"""Convert raw search results (title/url/snippet) into :class:`SourceDraft`."""
out: list[SourceDraft] = []
for item in (results or [])[:max_items]:
if not isinstance(item, dict):
continue
out.append(
SourceDraft(
source_type="search",
thread_id=thread_id,
message_id=message_id,
tool_name=_clip(item.get("tool_name") or item.get("tool"), 128),
title=_clip(item.get("title"), 512) or query[:200],
url=_clip(item.get("url"), 1024),
snippet=_clip(item.get("snippet") or item.get("content")),
raw={k: item.get(k) for k in ("score",) if item.get(k) is not None} or None,
)
)
return out
@@ -0,0 +1,47 @@
"""Pure scheduling helpers for the admin leaderboard daily snapshots.
Kept import-light on purpose (only stdlib ``datetime``) so the decision logic
is unit-testable without standing up the FastAPI app, the DB engine, or the
ORM. The background scheduler in ``app/gateway/app.py`` wires this to the real
snapshot rows.
"""
from __future__ import annotations
from datetime import datetime
def should_finalize_previous_day(
*,
status: str | None,
generated_at: datetime | None,
now: datetime,
finalize_hour: int,
) -> bool:
"""Decide whether *yesterday's* snapshot should be force-recomputed once.
The previous day's snapshot is normally built before midnight, so it can
miss runs / tool calls whose rows only land in the database shortly before
or after the day boundary. Once per day, on/after ``finalize_hour`` (local
Beijing time), we recompute it a single time so the full day is captured.
``now`` and ``generated_at`` are expected to be timezone-aware Beijing
datetimes (``generated_at`` may be ``None`` for legacy rows).
Returns ``True`` only when all hold:
- it is already on/after ``finalize_hour`` today;
- a snapshot row exists (``status`` is not ``None`` — a missing day is
left to the normal prewarm/queue path);
- that snapshot is not already ``queued``/``running`` (a finalize already
in flight, or normal processing);
- it was generated before today (i.e. not yet finalized for this day).
"""
if status is None:
return False
if now.hour < finalize_hour:
return False
if status in ("queued", "running"):
return False
if generated_at is not None and generated_at.date() >= now.date():
return False
return True
@@ -119,18 +119,19 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
``scope``:
@@ -142,13 +143,15 @@ class AgentStore(ABC):
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
``search`` matches the raw name OR its desensitized display form.
``tag_ids`` keeps agents carrying ANY of the tags (OR). Both filters,
the COUNT, and LIMIT/OFFSET all execute in SQL.
``tag_ids`` keeps agents carrying ANY of the tags (OR). ``owner`` keeps
only agents published by that user (``user_id`` equality) — built-in
agents (no owner) never match. All filters, the COUNT, and LIMIT/OFFSET
all execute in SQL.
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
for the 1-based ``page``.
"""
raise NotImplementedError
for the 1-based ``page``.
"""
raise NotImplementedError
# --- Listing cache (skills / tool_groups / model) ---
@@ -199,28 +202,28 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
# --- Per-user pins ("置顶") ---
@abstractmethod
async def add_pin(self, user_id: str, agent_id: str) -> None:
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
"""Remove a per-user pin. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_pin_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids pinned by ``user_id``."""
raise NotImplementedError
@abstractmethod
async def delete_pins_for_agent(self, agent_id: str) -> None:
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
# --- Per-user pins ("置顶") ---
@abstractmethod
async def add_pin(self, user_id: str, agent_id: str) -> None:
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
"""Remove a per-user pin. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_pin_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids pinned by ``user_id``."""
raise NotImplementedError
@abstractmethod
async def delete_pins_for_agent(self, agent_id: str) -> None:
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
@@ -21,9 +21,9 @@ def _copy(data: dict[str, Any]) -> dict[str, Any]:
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@@ -186,51 +186,54 @@ class MemoryAgentStore(AgentStore):
return bool(agent.get("is_private"))
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
owner_key = owner.strip() if owner and owner.strip() else None
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
and (owner_key is None or a.get("user_id") == owner_key)
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
@@ -275,23 +278,23 @@ class MemoryAgentStore(AgentStore):
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
@@ -6,19 +6,19 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, case, delete, func, or_, select
from sqlalchemy import and_, case, delete, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.agent_name_desensitize import source_terms_for_needle
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.tags.model import TagAssignmentRow
@@ -278,18 +278,19 @@ class AgentRepository(AgentStore):
)
return AgentRow.id.in_(subq)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
owner: str | None = None,
) -> tuple[list[dict[str, Any]], int]:
filters = []
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
if scope_filter is not None:
@@ -298,30 +299,33 @@ class AgentRepository(AgentStore):
filters.append(self._search_clause(search))
if tag_ids:
filters.append(self._tag_clause(tag_ids))
if owner and owner.strip():
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
filters.append(AgentRow.user_id == owner.strip())
count_stmt = select(func.count()).select_from(AgentRow)
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
if sort == "time_asc":
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
elif sort == "updated_desc":
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
elif sort == "updated_asc":
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
elif sort == "name_asc":
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
elif sort == "name_desc":
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
else:
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
page_stmt = (
select(AgentRow)
.order_by(pin_rank.asc(), *order_by)
.limit(page_size)
.offset(max(page - 1, 0) * page_size)
)
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
if sort == "time_asc":
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
elif sort == "updated_desc":
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
elif sort == "updated_asc":
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
elif sort == "name_asc":
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
elif sort == "name_desc":
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
else:
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
page_stmt = (
select(AgentRow)
.order_by(pin_rank.asc(), *order_by)
.limit(page_size)
.offset(max(page - 1, 0) * page_size)
)
if filters:
where = and_(*filters)
count_stmt = count_stmt.where(where)
@@ -441,49 +445,49 @@ class AgentRepository(AgentStore):
)
return {row[0] for row in result.all()}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentPinRow.id)
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentPinRow).where(
AgentPinRow.user_id == user_id,
AgentPinRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_pin_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def delete_pins_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
await session.commit()
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentPinRow.id)
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentPinRow).where(
AgentPinRow.user_id == user_id,
AgentPinRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_pin_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def delete_pins_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
await session.commit()
@@ -0,0 +1,42 @@
"""Add name_zh column to skills table.
Revision ID: 20260609_01
Revises: 20260608_01
Create Date: 2026-06-09
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260609_01"
down_revision = "20260608_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
# ``skills`` may not exist yet on a brand-new DB — create_all handles that
# case with the column already present, so only patch existing tables.
if _has_table("skills") and not _has_column("skills", "name_zh"):
op.add_column("skills", sa.Column("name_zh", sa.String(length=128), nullable=True))
def downgrade() -> None:
if _has_column("skills", "name_zh"):
op.drop_column("skills", "name_zh")
@@ -0,0 +1,30 @@
"""Merge the skill_name_zh and chain-stages migration heads.
Two lineages both descend from ``20260608_01`` (admin_leaderboard_daily_stats):
``20260608_02`` … ``20260608_06`` (the chain-stages / parallel-agent branch) and
``20260609_01`` (skills.name_zh), which branched directly off ``20260608_01``
instead of off the latest head. That left the graph with two heads and made
``alembic upgrade head`` fail with "Multiple head revisions are present". This
no-op merge revision joins both lineages so ``upgrade head`` resolves to a single
head again.
Revision ID: 20260609_02
Revises: 20260608_06, 20260609_01
Create Date: 2026-06-10
"""
from __future__ import annotations
revision = "20260609_02"
down_revision = ("20260608_06", "20260609_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -25,6 +25,16 @@ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.knowledge.models import (
KnowledgeEmbeddingRow,
KnowledgeEntityRow,
KnowledgeNoteRow,
KnowledgeNoteTagRow,
KnowledgeNoteVersionRow,
KnowledgeRelationRow,
KnowledgeSourceRow,
KnowledgeTagRow,
)
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
@@ -63,6 +73,14 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"KnowledgeEmbeddingRow",
"KnowledgeEntityRow",
"KnowledgeNoteRow",
"KnowledgeNoteTagRow",
"KnowledgeNoteVersionRow",
"KnowledgeRelationRow",
"KnowledgeSourceRow",
"KnowledgeTagRow",
"LlmCallMetricRow",
"NotificationRow",
"ParallelAgentTaskRow",
@@ -70,6 +70,7 @@ class MemorySkillStore(SkillStore):
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", False)),
"detail": data.get("detail"),
"name_zh": data.get("name_zh"),
"created_at": _now(),
"updated_at": _now(),
}
@@ -85,6 +86,7 @@ class MemorySkillStore(SkillStore):
"owner_user_id": data.get("owner_user_id"),
"published": bool(data.get("published", True)),
"detail": data.get("detail"),
"name_zh": data.get("name_zh"),
"created_at": _now(),
"updated_at": _now(),
}
@@ -102,6 +104,8 @@ class MemorySkillStore(SkillStore):
record["published"] = bool(data["published"])
if "detail" in data:
record["detail"] = data["detail"]
if "name_zh" in data:
record["name_zh"] = data["name_zh"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -115,6 +119,8 @@ class MemorySkillStore(SkillStore):
record["owner_user_id"] = data["owner_user_id"]
if "detail" in data:
record["detail"] = data["detail"]
if "name_zh" in data:
record["name_zh"] = data["name_zh"]
record["updated_at"] = _now()
return _with_derived(_copy(record)) # type: ignore[arg-type]
@@ -29,6 +29,10 @@ class SkillRow(Base):
# kept in the DB so admins can annotate built-in skills without rewriting
# the shared on-disk SKILL.md file.
detail: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
# User-configured or LLM-generated Chinese display name. Kept in the DB (not
# the on-disk SKILL.md) so it can be set for built-in skills too, and so the
# shared source file is never rewritten.
name_zh: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -64,6 +64,7 @@ class SkillRepository(SkillStore):
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", False)),
detail=data.get("detail"),
name_zh=data.get("name_zh"),
created_at=now,
updated_at=now,
)
@@ -83,6 +84,7 @@ class SkillRepository(SkillStore):
owner_user_id=data.get("owner_user_id"),
published=bool(data.get("published", True)),
detail=data.get("detail"),
name_zh=data.get("name_zh"),
created_at=now,
updated_at=now,
)
@@ -105,6 +107,8 @@ class SkillRepository(SkillStore):
row.published = bool(data["published"])
if "detail" in data:
row.detail = data["detail"]
if "name_zh" in data:
row.name_zh = data["name_zh"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -121,6 +125,8 @@ class SkillRepository(SkillStore):
row.owner_user_id = data["owner_user_id"]
if "detail" in data:
row.detail = data["detail"]
if "name_zh" in data:
row.name_zh = data["name_zh"]
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -1,295 +1,302 @@
"""Async checkpointer factory.
Provides an **async context manager** for long-running async servers that need
proper resource cleanup.
Supported backends: memory, sqlite, postgres.
Usage (e.g. FastAPI lifespan)::
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
async with make_checkpointer() as checkpointer:
app.state.checkpointer = checkpointer # InMemorySaver if not configured
For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator
import os
from pathlib import Path
from langgraph.types import Checkpointer
from deerflow.config.app_config import AppConfig, get_app_config
from deerflow.runtime.checkpointer.provider import (
POSTGRES_CONN_REQUIRED,
POSTGRES_INSTALL,
SQLITE_INSTALL,
)
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
logger = logging.getLogger(__name__)
def _csv_env(name: str) -> list[str]:
raw = os.getenv(name, "")
return [part.strip() for part in raw.split(",") if part.strip()]
def _dual_read_enabled(config) -> bool:
if getattr(config, "dual_read_enabled", False):
return True
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
def _use_postgres_enabled(config) -> bool:
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
if raw:
return raw.strip().lower() not in {"0", "false", "no", "off"}
return bool(getattr(config, "use_postgres", True))
def _dual_read_paths(config) -> list[str]:
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
resolved: list[str] = []
seen: set[str] = set()
for path in paths:
conn_str = resolve_sqlite_conn_str(path)
if conn_str == ":memory:" or conn_str.startswith("file:"):
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
continue
if conn_str not in seen:
resolved.append(conn_str)
seen.add(conn_str)
return resolved
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
configured = getattr(config, "sqlite_connection_string", None)
if configured:
return configured
paths = _dual_read_paths(config)
return paths[0] if paths else "store.db"
@contextlib.asynccontextmanager
async def _pg_pool_saver(conn_str: str) -> AsyncIterator[Checkpointer]:
"""Pool-backed ``AsyncPostgresSaver``, resilient to idle-closed connections.
``AsyncPostgresSaver.from_conn_string`` opens **one** long-lived connection.
对远程 Postgres(本项目 47.88.25.99 这类),空闲几分钟后连接会被服务端 / NAT /
连接池中间件断开,而 psycopg **不会自动重连** —— 下一次 checkpoint 操作就抛
``OperationalError: the connection is closed``,整个 run 直接失败(实测:圆桌第二步
完成后闲置 ~6 分钟,用户再人工干预 → leader run 读 checkpoint 时撞死连接 → 前端
"(无文本输出)")。
用带健康检查的连接池根治:``check=AsyncConnectionPool.check_connection`` 在把连接交给
调用方前先验活,坏连接直接丢弃重建;``max_idle`` 主动回收久闲连接,赶在远程 idle 超时
之前。这样上层永远拿不到失效连接。``prepare_threshold=0`` 关预备语句,兼容 pgbouncer
一类连接池(8432 非标准端口很可能就是)。
"""
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
from psycopg_pool import AsyncConnectionPool
try:
max_size = int(os.getenv("DEER_FLOW_PG_POOL_MAX", "20"))
except ValueError:
max_size = 20
pool = AsyncConnectionPool(
conninfo=conn_str,
min_size=1,
max_size=max_size,
open=False,
kwargs={"autocommit": True, "prepare_threshold": 0},
check=AsyncConnectionPool.check_connection,
max_idle=120.0,
max_lifetime=1800.0,
)
await pool.open(wait=True)
try:
saver = AsyncPostgresSaver(pool)
await saver.setup()
logger.info("Checkpointer: Postgres connection pool ready (max_size=%d, health-checked)", max_size)
yield saver
finally:
await pool.close()
@contextlib.asynccontextmanager
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(raw)
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
# ---------------------------------------------------------------------------
# Async factory
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
"""Async context manager that constructs and tears down a checkpointer."""
if config.type == "memory":
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
if config.type == "sqlite":
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
yield saver
return
if config.type == "postgres":
if not _use_postgres_enabled(config):
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
yield saver
return
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
# 连接池 + 健康检查,根治远程 Postgres 单连接空闲被关导致的 run 崩溃(见 _pg_pool_saver)。
async with _pg_pool_saver(config.connection_string) as saver:
if not _dual_read_enabled(config):
yield saver
return
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.checkpointer.checkpoint_migration import (
DualReadCheckpointer,
SQLiteCheckpointSource,
)
sources: list[SQLiteCheckpointSource] = []
for sqlite_path in _dual_read_paths(config):
if not Path(sqlite_path).exists():
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
continue
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
source_saver = await ctx.__aenter__()
await source_saver.setup()
sources.append(
SQLiteCheckpointSource(
path=sqlite_path,
saver=source_saver,
context=ctx,
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
)
)
if not sources:
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
yield saver
return
proxy = DualReadCheckpointer(
saver,
sources,
)
try:
yield proxy
finally:
await proxy.aclose_sources()
return
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
# ---------------------------------------------------------------------------
# Public async context manager
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
"""Async context manager that constructs a checkpointer from unified DatabaseConfig."""
if db_config.backend == "memory":
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
if db_config.backend == "sqlite":
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = db_config.checkpointer_sqlite_path
ensure_sqlite_parent_dir(conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
return
if db_config.backend == "postgres":
if not db_config.postgres_url:
raise ValueError("database.postgres_url is required for the postgres backend")
# 连接池 + 健康检查(见 _pg_pool_saver),避免单连接空闲被关。
async with _pg_pool_saver(db_config.postgres_url) as saver:
yield saver
return
raise ValueError(f"Unknown database backend: {db_config.backend!r}")
@contextlib.asynccontextmanager
async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
"""Async context manager that yields a checkpointer for the caller's lifetime.
Resources are opened on enter and closed on exit -- no global state::
async with make_checkpointer(app_config) as checkpointer:
app.state.checkpointer = checkpointer
Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
Priority:
1. Legacy ``checkpointer:`` config section (backward compatible)
2. Unified ``database:`` config section
3. Default InMemorySaver
"""
if app_config is None:
app_config = get_app_config()
# Legacy: standalone checkpointer config takes precedence
if app_config.checkpointer is not None:
async with _async_checkpointer(app_config.checkpointer) as saver:
yield saver
return
# Unified database config
db_config = getattr(app_config, "database", None)
if db_config is not None and db_config.backend != "memory":
async with _async_checkpointer_from_database(db_config) as saver:
yield saver
return
# Default: in-memory
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
"""Async checkpointer factory.
Provides an **async context manager** for long-running async servers that need
proper resource cleanup.
Supported backends: memory, sqlite, postgres.
Usage (e.g. FastAPI lifespan)::
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
async with make_checkpointer() as checkpointer:
app.state.checkpointer = checkpointer # InMemorySaver if not configured
For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator
import os
from pathlib import Path
from langgraph.types import Checkpointer
from deerflow.config.app_config import AppConfig, get_app_config
from deerflow.runtime.checkpointer.provider import (
POSTGRES_CONN_REQUIRED,
POSTGRES_INSTALL,
SQLITE_INSTALL,
)
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
logger = logging.getLogger(__name__)
def _csv_env(name: str) -> list[str]:
raw = os.getenv(name, "")
return [part.strip() for part in raw.split(",") if part.strip()]
def _dual_read_enabled(config) -> bool:
if getattr(config, "dual_read_enabled", False):
return True
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
def _use_postgres_enabled(config) -> bool:
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
if raw:
return raw.strip().lower() not in {"0", "false", "no", "off"}
return bool(getattr(config, "use_postgres", True))
def _dual_read_paths(config) -> list[str]:
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
resolved: list[str] = []
seen: set[str] = set()
for path in paths:
conn_str = resolve_sqlite_conn_str(path)
if conn_str == ":memory:" or conn_str.startswith("file:"):
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
continue
if conn_str not in seen:
resolved.append(conn_str)
seen.add(conn_str)
return resolved
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
configured = getattr(config, "sqlite_connection_string", None)
if configured:
return configured
paths = _dual_read_paths(config)
return paths[0] if paths else "store.db"
def _pool_size_env(name: str, default: int) -> int:
raw = os.getenv(name, "").strip()
if not raw:
return default
try:
return max(0, int(raw))
except ValueError:
logger.warning("Invalid %s=%r; using default %d", name, raw, default)
return default
@contextlib.asynccontextmanager
async def _postgres_saver_from_pool(conn_string: str) -> AsyncIterator[Checkpointer]:
"""Yield an ``AsyncPostgresSaver`` backed by a self-healing connection pool.
``AsyncPostgresSaver.from_conn_string`` opens a *single* long-lived
connection. If that one connection dies (Postgres restart, idle timeout,
network blip, or a prior broken pipeline), every later checkpointer call
fails permanently with "the connection is closed" until the process is
restarted. A pool with ``check=check_connection`` validates each connection
on checkout and transparently replaces dead ones, so the runtime recovers
on its own.
"""
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
pool = AsyncConnectionPool(
conninfo=conn_string,
min_size=_pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
max_size=_pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
max_idle=300.0,
max_lifetime=3600.0,
check=AsyncConnectionPool.check_connection,
kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row},
open=False,
)
await pool.open(wait=True)
try:
saver = AsyncPostgresSaver(pool)
await saver.setup()
yield saver
finally:
await pool.close()
@contextlib.asynccontextmanager
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(raw)
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
# ---------------------------------------------------------------------------
# Async factory
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
"""Async context manager that constructs and tears down a checkpointer."""
if config.type == "memory":
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
if config.type == "sqlite":
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
yield saver
return
if config.type == "postgres":
if not _use_postgres_enabled(config):
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
yield saver
return
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
async with _postgres_saver_from_pool(config.connection_string) as saver:
if not _dual_read_enabled(config):
yield saver
return
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.checkpointer.checkpoint_migration import (
DualReadCheckpointer,
SQLiteCheckpointSource,
)
sources: list[SQLiteCheckpointSource] = []
for sqlite_path in _dual_read_paths(config):
if not Path(sqlite_path).exists():
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
continue
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
source_saver = await ctx.__aenter__()
await source_saver.setup()
sources.append(
SQLiteCheckpointSource(
path=sqlite_path,
saver=source_saver,
context=ctx,
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
)
)
if not sources:
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
yield saver
return
proxy = DualReadCheckpointer(
saver,
sources,
)
try:
yield proxy
finally:
await proxy.aclose_sources()
return
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
# ---------------------------------------------------------------------------
# Public async context manager
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
"""Async context manager that constructs a checkpointer from unified DatabaseConfig."""
if db_config.backend == "memory":
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
if db_config.backend == "sqlite":
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = db_config.checkpointer_sqlite_path
ensure_sqlite_parent_dir(conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
return
if db_config.backend == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not db_config.postgres_url:
raise ValueError("database.postgres_url is required for the postgres backend")
async with _postgres_saver_from_pool(db_config.postgres_url) as saver:
yield saver
return
raise ValueError(f"Unknown database backend: {db_config.backend!r}")
@contextlib.asynccontextmanager
async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
"""Async context manager that yields a checkpointer for the caller's lifetime.
Resources are opened on enter and closed on exit -- no global state::
async with make_checkpointer(app_config) as checkpointer:
app.state.checkpointer = checkpointer
Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
Priority:
1. Legacy ``checkpointer:`` config section (backward compatible)
2. Unified ``database:`` config section
3. Default InMemorySaver
"""
if app_config is None:
app_config = get_app_config()
# Legacy: standalone checkpointer config takes precedence
if app_config.checkpointer is not None:
async with _async_checkpointer(app_config.checkpointer) as saver:
yield saver
return
# Unified database config
db_config = getattr(app_config, "database", None)
if db_config is not None and db_config.backend != "memory":
async with _async_checkpointer_from_database(db_config) as saver:
yield saver
return
# Default: in-memory
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
@@ -28,6 +28,42 @@ from deerflow.runtime.store.provider import POSTGRES_CONN_REQUIRED, POSTGRES_STO
logger = logging.getLogger(__name__)
def _pool_size_env(name: str, default: int) -> int:
import os
raw = os.getenv(name, "").strip()
if not raw:
return default
try:
return max(0, int(raw))
except ValueError:
logger.warning("Invalid %s=%r; using default %d", name, raw, default)
return default
def _postgres_pool_config() -> dict:
"""Pool config for the Postgres store.
A single long-lived connection (the default ``from_conn_string`` behavior)
dies permanently on a server restart / idle timeout / network blip — every
later store call then fails with "server closed the connection
unexpectedly". A pool with ``check=check_connection`` validates each
connection on checkout and replaces dead ones, so the runtime self-heals.
Extra keys (``check``/``max_idle``/``max_lifetime``) are passed straight
through to ``AsyncConnectionPool`` by ``from_conn_string``.
"""
from psycopg_pool import AsyncConnectionPool
return {
"min_size": _pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
"max_size": _pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
"check": AsyncConnectionPool.check_connection,
"max_idle": 300.0,
"max_lifetime": 3600.0,
}
# ---------------------------------------------------------------------------
# Internal backend factory
# ---------------------------------------------------------------------------
@@ -71,9 +107,9 @@ async def _async_store(config) -> AsyncIterator[BaseStore]:
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
async with AsyncPostgresStore.from_conn_string(config.connection_string) as store:
async with AsyncPostgresStore.from_conn_string(config.connection_string, pool_config=_postgres_pool_config()) as store:
await store.setup()
logger.info("Store: using AsyncPostgresStore")
logger.info("Store: using AsyncPostgresStore (pooled, self-healing)")
yield store
return
@@ -0,0 +1,225 @@
"""Interactive browser action tool backed by the user's Chrome extension.
Unlike ``browser_fetch_page`` (a one-shot read), this tool drives an
*observe -> act* loop on a live page the extension keeps open:
1. The model calls ``browser_act`` with an action (navigate/click/type/...).
2. The extension performs the action on the session tab, then re-scans the page
and returns an *observation*: the final URL/title, an indexed list of
interactive elements, and the visible page text.
3. The model reads the observation, picks the next element by its index, and
calls ``browser_act`` again with the same ``session_id``.
The element ``index`` values are only valid for the most recent observation of a
session; always act on indices from the latest result.
"""
from __future__ import annotations
from typing import Any
from urllib.parse import urlparse
from uuid import uuid4
from langchain.tools import tool
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
from deerflow.config import get_app_config
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime.user_context import get_effective_user_id
_ACTIONS = {"navigate", "click", "type", "scroll", "wait", "extract", "observe", "close"}
_NEEDS_INDEX = {"click", "type"}
def _settings() -> BrowserContextConfig:
value = getattr(get_app_config(), "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _validate_url(raw_url: str) -> str | None:
url = (raw_url or "").strip()
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
return url
def _format_elements(elements: list[dict[str, Any]] | None) -> str:
if not elements:
return "(no interactive elements detected)"
lines: list[str] = []
for el in elements:
idx = el.get("index")
tag = str(el.get("tag") or "?")
el_type = str(el.get("type") or "").strip()
label = str(el.get("label") or "").strip().replace("\n", " ")
if len(label) > 120:
label = label[:120] + "…"
descriptor = f"{tag}" + (f"[{el_type}]" if el_type else "")
extra = ""
if el.get("value"):
value = str(el["value"]).strip().replace("\n", " ")
extra = f' value="{value[:60]}"'
lines.append(f"[{idx}] {descriptor} \"{label}\"{extra}".rstrip())
return "\n".join(lines)
def _format_observation(action: str, observation: dict[str, Any] | None, session_id: str, settings: BrowserContextConfig) -> str:
observation = observation or {}
status = str(observation.get("action_status") or "ok")
message = str(observation.get("message") or "").strip()
url = str(observation.get("url") or "").strip() or "(unknown)"
title = str(observation.get("title") or "").strip() or "(untitled)"
elements = observation.get("elements") if isinstance(observation.get("elements"), list) else []
text = str(observation.get("text") or "").strip()
text_limit = max(500, int(settings.observation_text_max_chars))
text_truncated = len(text) > text_limit
if text_truncated:
text = text[:text_limit]
parts = [
f"Browser session: {session_id}",
f"Last action: {action} -> {status}",
]
if message:
parts.append(f"Note: {message}")
parts.extend(
[
f"URL: {url}",
f"Title: {title}",
"",
"Interactive elements (use the [index] with click/type):",
_format_elements(elements),
"",
"Page text:",
text or "(empty page text)",
]
)
if text_truncated:
parts.append("\n[Page text truncated by browser_context.observation_text_max_chars]")
parts.append(
"\nNext step: call browser_act again with this session_id and the index of the element to act on. "
"When the task is finished, call browser_act with action=\"close\" to release the tab."
)
return "\n".join(parts)
@tool("browser_act", parse_docstring=True)
async def browser_act_tool(
action: str,
session_id: str = "",
url: str = "",
index: int = -1,
text: str = "",
submit: bool = False,
wait_for: str = "",
wait_seconds: int = 0,
) -> str:
"""Drive an interactive observe/act loop on a page in the user's Chrome browser.
Use this to operate a real web page on the user's behalf — log in, fill forms,
click through navigation, and read pages that need the user's session or local
network. Start every task with ``action="navigate"`` and a URL to open a tab
and get the first observation, then keep calling this tool with the same
``session_id``, choosing elements by the ``index`` shown in the observation.
Call ``action="close"`` when done to free the tab.
Args:
action: One of navigate, click, type, scroll, wait, extract, observe, close.
session_id: Session id from a previous observation. Leave empty to start a new session (requires navigate + url).
url: Target http(s) URL. Required for navigate (and to start a new session).
index: Index of the interactive element to act on, taken from the latest observation. Required for click and type.
text: Text to enter for the type action.
submit: For type, press Enter / submit the form after typing.
wait_for: For wait, a CSS selector to wait for before observing.
wait_seconds: Optional extra seconds to wait for this step's result. Zero uses the configured default.
"""
settings = _settings()
if not settings.enabled:
return "browser_act is disabled by config.yaml: browser_context.enabled is false."
if not settings.actions_enabled:
return "browser_act is disabled by config.yaml: browser_context.actions_enabled is false."
action = (action or "").strip().lower()
if action not in _ACTIONS:
return f"browser_act: unknown action {action!r}. Use one of: {', '.join(sorted(_ACTIONS))}."
session_id = (session_id or "").strip()
safe_url = _validate_url(url) if url else None
if action == "navigate" and safe_url is None:
return "browser_act navigate requires an exact http(s) URL, for example https://example.com/page."
if not session_id:
if action != "navigate":
return "browser_act: no session_id yet. Start with action=\"navigate\" and a url to open the page first."
session_id = uuid4().hex
if action in _NEEDS_INDEX and index < 0:
return f"browser_act {action} requires a non-negative element index from the latest observation."
if action == "type" and not text:
return "browser_act type requires the text to enter."
action_payload: dict[str, Any] = {"type": action}
if safe_url is not None:
action_payload["url"] = safe_url
if index >= 0:
action_payload["index"] = int(index)
if text:
action_payload["text"] = text
if submit:
action_payload["submit"] = True
if wait_for.strip():
action_payload["wait_for"] = wait_for.strip()[:300]
configured_timeout = max(5, int(settings.action_result_timeout_seconds))
requested_timeout = int(wait_seconds or configured_timeout)
timeout = min(max(5, requested_timeout), configured_timeout)
user_id = get_effective_user_id()
task = create_browser_task(
user_id=user_id,
url=safe_url or "",
instruction="",
kind="act",
session_id=session_id,
action=action_payload,
timeout_seconds=timeout + int(settings.task_lease_seconds),
max_content_chars=settings.max_content_chars,
metadata={"source": "browser_act_tool"},
)
completed = await wait_for_browser_task_result(
task_id=task["id"],
user_id=user_id,
timeout_seconds=timeout,
poll_interval_seconds=settings.task_poll_interval_seconds,
)
if completed is None:
return f"Browser task {task['id']} was not found after it was created."
status = completed.get("status")
if status == "error":
return f"browser_act session {session_id}: step failed: {completed.get('error') or 'unknown error'}"
if status == "expired":
return (
f"browser_act session {session_id}: the Chrome extension did not respond in time. "
"Check that Deer Web Clipper is installed, the same user is logged in, and task polling is enabled."
)
if status != "done":
return (
f"browser_act session {session_id}: step is still {status or 'pending'} after {timeout} seconds. "
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
)
if action == "close":
return f"browser_act session {session_id}: tab closed."
result = completed.get("result") or {}
observation = result.get("observation") if isinstance(result.get("observation"), dict) else None
return _format_observation(action, observation, session_id, settings)
@@ -0,0 +1,117 @@
"""Browser extension backed page fetch tool."""
from __future__ import annotations
from urllib.parse import urlparse
from langchain.tools import tool
from deerflow.browser_context.tasks import create_browser_task, wait_for_browser_task_result
from deerflow.config import get_app_config
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.runtime.user_context import get_effective_user_id
def _settings() -> BrowserContextConfig:
value = getattr(get_app_config(), "browser_context", None)
if isinstance(value, BrowserContextConfig):
return value
if isinstance(value, dict):
return BrowserContextConfig.model_validate(value)
return BrowserContextConfig()
def _validate_url(raw_url: str) -> str | None:
url = (raw_url or "").strip()
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
return url
@tool("browser_fetch_page", parse_docstring=True)
async def browser_fetch_page_tool(url: str, question: str = "", wait_seconds: int = 0) -> str:
"""Ask the user's paired Chrome extension to fetch a page and return visible page text.
Use this when the user explicitly asks to use the browser extension or asks
for data from a page that may require their browser session, login state, or
local network access. The URL must be an exact http(s) URL provided by the
user. If the user did not provide a URL, ask for the URL instead of guessing.
Args:
url: Exact http(s) URL to open in the user's Chrome browser.
question: Optional instruction describing what data the user wants from the page.
wait_seconds: Optional seconds to wait for the extension result. Zero uses the configured default.
"""
settings = _settings()
if not settings.enabled:
return "browser_fetch_page is disabled by config.yaml: browser_context.enabled is false."
safe_url = _validate_url(url)
if safe_url is None:
return "browser_fetch_page requires an exact http(s) URL, for example https://example.com/page."
configured_timeout = max(5, int(settings.task_result_timeout_seconds))
requested_timeout = int(wait_seconds or configured_timeout)
timeout = min(max(5, requested_timeout), configured_timeout)
user_id = get_effective_user_id()
task = create_browser_task(
user_id=user_id,
url=safe_url,
instruction=(question or "").strip(),
timeout_seconds=timeout + int(settings.task_lease_seconds),
max_content_chars=settings.max_content_chars,
metadata={"source": "browser_fetch_page_tool"},
)
completed = await wait_for_browser_task_result(
task_id=task["id"],
user_id=user_id,
timeout_seconds=timeout,
poll_interval_seconds=settings.task_poll_interval_seconds,
)
if completed is None:
return f"Browser task {task['id']} was not found after it was created."
status = completed.get("status")
if status == "error":
return f"Browser task {task['id']} failed: {completed.get('error') or 'unknown error'}"
if status == "expired":
return (
f"Browser task {task['id']} expired before the Chrome extension returned a result. "
"Check that Deer Web Clipper is installed, configured with the same access token, and task polling is enabled."
)
if status != "done":
return (
f"Browser task {task['id']} is still {status or 'pending'} after {timeout} seconds. "
"Keep the Deer Web Clipper extension running with task polling enabled, then try again."
)
result = completed.get("result") or {}
content = str(result.get("content") or "").strip()
tool_limit = max(1_000, int(settings.tool_result_max_chars))
tool_truncated = len(content) > tool_limit
if tool_truncated:
content = content[:tool_limit]
notes: list[str] = []
if result.get("truncated"):
notes.append("The browser capture was truncated by browser_context.max_content_chars.")
if tool_truncated:
notes.append("The tool response was truncated by browser_context.tool_result_max_chars.")
parts = [
"Browser page fetched through the user's Chrome extension.",
f"Task ID: {task['id']}",
f"Title: {result.get('title') or '(untitled)'}",
f"URL: {result.get('url') or safe_url}",
]
if question:
parts.extend(["", "Requested extraction:", question.strip()])
if notes:
parts.extend(["", "Notes:", "\n".join(f"- {note}" for note in notes)])
parts.extend(["", "Page content:", content or "(empty page text)"])
return "\n".join(parts)
@@ -11,6 +11,11 @@ from deerflow.tools.builtins.tool_search import reset_deferred_registry
logger = logging.getLogger(__name__)
BROWSER_CONTEXT_TOOL_USE = "deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool"
BROWSER_ACT_TOOL_USE = "deerflow.tools.builtins.browser_act_tool:browser_act_tool"
BROWSER_CONTEXT_TOOL_USES = {BROWSER_CONTEXT_TOOL_USE, BROWSER_ACT_TOOL_USE}
BROWSER_CONTEXT_TOOL_NAMES = {"browser_fetch_page", "browser_act"}
BUILTIN_TOOLS = [
present_file_tool,
ask_clarification_tool,
@@ -34,6 +39,14 @@ def _is_host_bash_tool(tool: object) -> bool:
return False
def _is_browser_context_tool(tool: object) -> bool:
"""Return True if the tool config represents the browser extension bridge."""
use = getattr(tool, "use", None)
name = getattr(tool, "name", None)
return use in BROWSER_CONTEXT_TOOL_USES or name in BROWSER_CONTEXT_TOOL_NAMES
def get_available_tools(
groups: list[str] | None = None,
include_mcp: bool = True,
@@ -64,6 +77,18 @@ def get_available_tools(
config = app_config or get_app_config()
tool_configs = [tool for tool in config.tools if groups is None or tool.group in groups]
browser_context_config = getattr(config, "browser_context", None)
if not getattr(browser_context_config, "enabled", False):
tool_configs = [tool for tool in tool_configs if not _is_browser_context_tool(tool)]
elif not getattr(browser_context_config, "actions_enabled", True):
# browser_context is on, but interactive page actions are disabled — keep
# the read-only browser_fetch_page tool, drop only browser_act.
tool_configs = [
tool
for tool in tool_configs
if getattr(tool, "use", None) != BROWSER_ACT_TOOL_USE and getattr(tool, "name", None) != "browser_act"
]
# Do not expose host bash by default when LocalSandboxProvider is active.
if not is_host_bash_allowed(config):
tool_configs = [tool for tool in tool_configs if not _is_host_bash_tool(tool)]