sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -17,6 +17,7 @@ from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.html_runtime_config import HtmlRuntimeConfig
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.knowledge_ingest_config import KnowledgeIngestConfig
from deerflow.config.llmwiki_config import LlmWikiConfig
@@ -43,6 +44,7 @@ from deerflow.config.token_usage_config import TokenUsageConfig
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
from deerflow.config.vision_config import VisionConfig
from deerflow.config.web_app_config import WebAppConfig
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.config.y_log_config import YLogConfig
@@ -151,7 +153,15 @@ class AppConfig(BaseModel):
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
y_log: YLogConfig = Field(default_factory=YLogConfig, description="操作日志上报配置(前端经 /api/public/config/y-log 读取)")
html_runtime: HtmlRuntimeConfig = Field(
default_factory=HtmlRuntimeConfig,
description="后台生成 HTML 的浏览器绘图库与地图瓦片配置",
)
workflows: WorkflowConfig = Field(default_factory=WorkflowConfig, description="Workflow Studio limits / iframe origins")
web_app: WebAppConfig = Field(
default_factory=WebAppConfig,
description="Web App 产物模式(多页面站点生成/构建/预览,默认关闭、可降级)",
)
report_collaboration: ReportCollaborationConfig = Field(
default_factory=ReportCollaborationConfig,
description="AgentScope report-collaboration workbench (default off)",
@@ -1,13 +1,25 @@
"""Unified extensions configuration for MCP servers and skills."""
import copy
import errno
import json
import logging
import os
import tempfile
import threading
import time
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.runtime_paths import existing_project_file
logger = logging.getLogger(__name__)
_EXTENSIONS_CONFIG_WRITE_LOCK = threading.Lock()
class McpOAuthConfig(BaseModel):
@@ -350,6 +362,142 @@ def reload_extensions_config(config_path: str | None = None) -> ExtensionsConfig
return _extensions_config
def _read_raw_extensions_config(config_path: Path) -> dict[str, Any]:
"""Read the raw JSON document without resolving env placeholders."""
if not config_path.exists():
return {}
with config_path.open(encoding="utf-8") as config_file:
document = json.load(config_file)
if not isinstance(document, dict):
raise ValueError("Extensions config root must be a JSON object")
return document
def read_extensions_config_document(config_path: str | Path) -> dict[str, Any]:
"""Return a fresh raw document while coordinating with all writers."""
path = Path(config_path)
with _EXTENSIONS_CONFIG_WRITE_LOCK:
with _extensions_config_process_lock(path):
return _read_raw_extensions_config(path)
@contextmanager
def _extensions_config_process_lock(config_path: Path) -> Iterator[None]:
"""Serialize writers across processes on Windows and POSIX.
The stable sidecar is intentional: deleting it after unlock can let two
processes lock different inodes. Keeping it beside the config also lets
multiple processes sharing the writable config directory coordinate.
"""
lock_path = config_path.with_name(f".{config_path.name}.lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
with lock_path.open("a+b") as lock_file:
if os.name == "nt":
import msvcrt
lock_file.seek(0, os.SEEK_END)
if lock_file.tell() == 0:
lock_file.write(b"\0")
lock_file.flush()
lock_file.seek(0)
while True:
try:
msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)
break
except OSError as exc:
if exc.errno not in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}:
raise
time.sleep(0.05)
try:
yield
finally:
lock_file.seek(0)
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
def _write_raw_extensions_config(config_path: Path, document: dict[str, Any]) -> None:
"""Durably replace raw JSON, with a bind-mounted-file EBUSY fallback."""
config_path.parent.mkdir(parents=True, exist_ok=True)
payload = json.dumps(document, indent=2, ensure_ascii=False) + "\n"
tmp_path: Path | None = None
try:
with tempfile.NamedTemporaryFile(
"w",
encoding="utf-8",
delete=False,
dir=str(config_path.parent),
) as tmp_file:
tmp_file.write(payload)
tmp_file.flush()
os.fsync(tmp_file.fileno())
tmp_path = Path(tmp_file.name)
try:
os.replace(tmp_path, config_path)
except OSError as exc:
if exc.errno != errno.EBUSY:
raise
logger.warning(
"Atomic replace is unavailable for bind-mounted extensions config %s; using in-place fallback",
config_path,
)
previous = config_path.read_bytes() if config_path.exists() else None
try:
mode = "r+" if config_path.exists() else "w"
with config_path.open(mode, encoding="utf-8") as config_file:
config_file.seek(0)
config_file.write(payload)
config_file.truncate()
config_file.flush()
os.fsync(config_file.fileno())
except Exception:
if previous is not None:
try:
with config_path.open("wb") as config_file:
config_file.write(previous)
config_file.flush()
os.fsync(config_file.fileno())
except Exception:
logger.exception("Failed to restore extensions config after in-place write failure")
raise
finally:
if tmp_path is not None and tmp_path.exists():
tmp_path.unlink()
def patch_extensions_config(
config_path: str | Path,
patch: Callable[[dict[str, Any]], None],
) -> ExtensionsConfig:
"""Apply a raw-document patch under thread and cross-process locks.
Callers should mutate only the fields they own. The document is read fresh
while locked, so concurrent MCP, skill-state, and display writes cannot
overwrite one another or resolve/persist secret ``$ENV`` placeholders.
"""
path = Path(config_path)
with _EXTENSIONS_CONFIG_WRITE_LOCK:
with _extensions_config_process_lock(path):
document = _read_raw_extensions_config(path)
patch(document)
if not isinstance(document, dict):
raise ValueError("Extensions config root must be a JSON object")
# Validate a deep copy before persisting. Environment expansion is
# intentionally confined to the copy so placeholders remain raw.
validation_document = copy.deepcopy(document)
ExtensionsConfig.resolve_env_variables(validation_document)
ExtensionsConfig.model_validate(validation_document)
_write_raw_extensions_config(path, document)
return reload_extensions_config(str(path))
def reset_extensions_config() -> None:
"""Reset the cached extensions config instance.
@@ -0,0 +1,13 @@
"""Browser runtime configuration for generated HTML artifacts."""
from pydantic import BaseModel, Field
class HtmlRuntimeConfig(BaseModel):
"""Resources used by standalone HTML pages generated in the sandbox."""
base_url: str = Field(
default="http://localhost:8001/api/public/html-runtime",
min_length=1,
description="Browser-accessible Gateway base URL for generated HTML runtime files",
)
@@ -77,6 +77,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
base_url: str = ""
api_key: str = Field(default="", repr=False)
local_model_path: str = ""
local_model_revision: str = ""
local_cache_dir: str = ""
local_process_isolation: bool = True
local_threads: int = Field(default=2, ge=1, le=32)
@@ -155,6 +156,10 @@ class LocalWikiIndexConfig(BaseModel):
min_similarity: float = Field(default=0.35, ge=-1, le=1)
keyword_boost_enabled: bool = False
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
# Lexical title matches receive a bounded score/RRF multiplier rather than
# an absolute priority. This is configurable without changing the index.
title_boost_enabled: bool = True
title_boost_weight: float = Field(default=0.25, ge=0, le=1)
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
database_batch_size: int = Field(default=50, ge=1, le=500)
@@ -12,8 +12,11 @@
injection: {...} # 系统提示词注入
security: {...} # 内容安全扫描
两层配置覆盖
============
运行时总开关与配置覆盖
======================
管理员维护的 ``system_settings.json -> memory.enabled`` 是权威总开关,默认
``false``。关闭时下面所有 provider 参数都不生效。
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
1. per-user 覆盖 数据库 ``user_memory_configs`` 表 (经进程内 cache 读取)
(有 per-user 记录时自动强制 enabled=True)
@@ -32,6 +35,8 @@ from typing import Any
from pydantic import BaseModel, Field
from deerflow.config.system_settings import get_memory_runtime_settings
logger = logging.getLogger(__name__)
@@ -126,7 +131,7 @@ class V1MemoryConfig(BaseModel):
class MemoryConfig(BaseModel):
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
enabled: bool = Field(default=True, description="记忆子系统总开关")
enabled: bool = Field(default=False, description="记忆子系统总开关")
version: str = Field(
default="v2",
description="记忆系统版本:v1(原版 DeerFlow,memory.json + LLM 自动提取) 或 v2(USER.md/MEMORY.md)",
@@ -253,10 +258,18 @@ def get_recall_budget_from_cache(user_id: str) -> str | None:
def get_effective_memory_config(user_id: str) -> MemoryConfig:
"""返回某用户的生效记忆配置 (同步,从进程内 cache 读取 per-user 覆盖)。
优先级 (高 -> 低):per-user DB 覆盖 (经 cache) > config.yaml > 默认值。
有 per-user 记录时自动强制 enabled=True。
管理员运行时总开关关闭时直接返回 ``enabled=False``;开启后再按
per-user DB 覆盖 (经 cache) > config.yaml > 默认值 合并参数。
"""
# The admin-managed runtime switch is authoritative and defaults to off.
# ``config.yaml`` supplies provider parameters only; it cannot silently
# turn memory on after a fresh deployment or settings-file reset.
runtime_enabled = get_memory_runtime_settings().enabled
if not runtime_enabled:
return _memory_config.model_copy(update={"enabled": False}, deep=True)
base = _memory_config.model_dump()
base["enabled"] = True
recall_budget = _recall_budget_cache.get(user_id)
if recall_budget is not None:
base.setdefault("hindsight", {})["recall_budget"] = recall_budget
@@ -10,9 +10,14 @@ class ModelConfig(BaseModel):
use: str = Field(
...,
description="Class path of the model provider(e.g. langchain_openai.ChatOpenAI)",
)
model: str = Field(..., description="Model name")
model_config = ConfigDict(extra="allow")
)
model: str = Field(..., description="Model name")
max_input_tokens: int | None = Field(
default=None,
ge=1,
description="Maximum input context tokens DeerFlow may send to the model",
)
model_config = ConfigDict(extra="allow")
use_responses_api: bool | None = Field(
default=None,
description="Whether to route OpenAI ChatOpenAI calls through the /v1/responses API",
@@ -18,7 +18,23 @@ class ReportCollaborationConfig(BaseModel):
default=False,
description="When true, a dispatcher/worker may claim run leases. Gateway-only processes keep this false.",
)
planner_llm_enabled: bool = Field(
default=False,
description="P1: LLM planner conversation (clarify + custom strategy). Off keeps the deterministic rule router and template plans.",
)
planner_model: str | None = Field(default=None, description="Model name from AppConfig.models for planning")
planner_timeout_seconds: int = Field(
default=120,
ge=30,
le=600,
description="Hard deadline for one streamed workflow-planning model call",
)
planner_max_tokens: int = Field(
default=1800,
ge=512,
le=8192,
description="Maximum output tokens for the workflow-planning model call",
)
coordinator_model: str | None = Field(default=None, description="Model name from AppConfig.models for the constrained coordinator")
research_model: str | None = Field(default=None, description="Model name from AppConfig.models for research members")
writer_model: str | None = Field(default=None, description="Model name from AppConfig.models for writing")
@@ -414,6 +414,22 @@ class AgentModuleSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class WxCommonAgentsSettings(BaseModel):
"""The gallery category shown in the wx chat route's common-agent panel.
``None`` keeps the original recommendation logic. ``mine`` and
``sentiment`` are built-in category values. Publish-square categories are
persisted as ``square:<square_id>`` so they cannot collide with the
built-ins.
"""
agent_type: str | None = Field(
default=None,
description="Selected gallery category; null retains the default recommendation logic",
)
model_config = ConfigDict(extra="ignore")
class PluginDomainAgentMapping(BaseModel):
"""One configured host -> agent mapping for iframe chat auto-selection."""
@@ -505,14 +521,41 @@ class ModelConcurrencySettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class MemoryRuntimeSettings(BaseModel):
"""Deployment-wide switch for Q&A memory.
Memory is deliberately opt-in. When this setting is false the lead agent
does not mount memory middleware or memory tools, does not inject stored
memory into prompts, and does not enqueue background memory extraction.
"""
enabled: bool = Field(
default=False,
description="Whether Q&A memory recall, injection and persistence are enabled.",
)
model_config = ConfigDict(extra="ignore")
class LocalWikiEncoderSettings(BaseModel):
"""Runtime override for the administrator-managed offline Wiki encoder."""
mode: Literal["configured", "local"] = Field(
default="configured",
description=(
"Use the embedding provider from config.yaml, or explicitly override it "
"with the administrator-managed offline BGE-M3 encoder."
),
)
model_path: str = Field(
default="",
max_length=4096,
description="Absolute or project-relative directory containing the offline BGE-M3 ONNX model.",
)
model_revision: str = Field(
default="",
max_length=64,
description="Opaque revision changed whenever an offline model directory is activated.",
)
model_config = ConfigDict(extra="ignore")
@@ -539,8 +582,10 @@ class SystemSettings(BaseModel):
skill_compression: SkillCompressionSettings = Field(default_factory=SkillCompressionSettings)
plugin_management: PluginManagementSettings = Field(default_factory=PluginManagementSettings)
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
wx_common_agents: WxCommonAgentsSettings = Field(default_factory=WxCommonAgentsSettings)
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
memory: MemoryRuntimeSettings = Field(default_factory=MemoryRuntimeSettings)
local_wiki_encoder: LocalWikiEncoderSettings = Field(default_factory=LocalWikiEncoderSettings)
strategy_projects: StrategyProjectSettings = Field(default_factory=StrategyProjectSettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
@@ -602,12 +647,26 @@ def get_model_concurrency_settings() -> ModelConcurrencySettings:
return ModelConcurrencySettings()
def get_memory_runtime_settings() -> MemoryRuntimeSettings:
"""Return the admin-controlled Q&A memory switch, failing closed."""
try:
return load_system_settings().memory
except Exception:
logger.debug("Failed to read memory runtime setting; keeping memory disabled", exc_info=True)
return MemoryRuntimeSettings()
def save_system_settings(settings: SystemSettings) -> SystemSettings:
"""Persist the settings document atomically, returning the saved value."""
path = _settings_path()
payload = settings.model_dump(mode="json")
with _lock:
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
tmp.replace(path)
# Multiple Gateway workers share this file. A unique temporary name
# prevents their atomic replaces from racing over one common `.tmp`.
tmp = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
try:
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
tmp.replace(path)
finally:
tmp.unlink(missing_ok=True)
return settings
@@ -0,0 +1,39 @@
"""Web App 产物模式配置(多页面站点生成/构建/预览)。"""
from __future__ import annotations
from pydantic import BaseModel, Field
class WebAppConfig(BaseModel):
"""站点产物(outputs/ 下的 site.json 站点)总开关与构建参数。
默认关闭:模板未预置 node_modules 或未启用时,所有站点 API 返回
结构化的"不可用"响应,智能体技能自动回退单页 HTML 模式,
不影响任何既有功能。
"""
enabled: bool = Field(
default=False,
description="总开关:关闭时 capability 返回不可用,入口按钮不渲染,技能回退单页 HTML",
)
template_dir: str = Field(
default="templates/web-app-template",
description="站点模板目录(相对 project root 或绝对路径;含预置 node_modules 与 scripts/build_site.py)",
)
build_timeout_seconds: int = Field(
default=180,
ge=10,
le=1800,
description="单次 vite build 超时(秒),超时强杀子进程",
)
preview_token_ttl_seconds: int = Field(
default=7200,
ge=60,
le=86400,
description="站点预览签名 URL 有效期(秒)",
)
node_path: str = Field(
default="",
description="node 可执行文件覆盖(空 = PATH 查找;一般无需配置)",
)