sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -17,6 +17,7 @@ from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.gateway_config import GatewayConfig
|
||||
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
|
||||
from deerflow.config.html_runtime_config import HtmlRuntimeConfig
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.knowledge_ingest_config import KnowledgeIngestConfig
|
||||
from deerflow.config.llmwiki_config import LlmWikiConfig
|
||||
@@ -43,6 +44,7 @@ from deerflow.config.token_usage_config import TokenUsageConfig
|
||||
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
|
||||
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
|
||||
from deerflow.config.vision_config import VisionConfig
|
||||
from deerflow.config.web_app_config import WebAppConfig
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.config.y_log_config import YLogConfig
|
||||
|
||||
@@ -151,7 +153,15 @@ class AppConfig(BaseModel):
|
||||
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
|
||||
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
|
||||
y_log: YLogConfig = Field(default_factory=YLogConfig, description="操作日志上报配置(前端经 /api/public/config/y-log 读取)")
|
||||
html_runtime: HtmlRuntimeConfig = Field(
|
||||
default_factory=HtmlRuntimeConfig,
|
||||
description="后台生成 HTML 的浏览器绘图库与地图瓦片配置",
|
||||
)
|
||||
workflows: WorkflowConfig = Field(default_factory=WorkflowConfig, description="Workflow Studio limits / iframe origins")
|
||||
web_app: WebAppConfig = Field(
|
||||
default_factory=WebAppConfig,
|
||||
description="Web App 产物模式(多页面站点生成/构建/预览,默认关闭、可降级)",
|
||||
)
|
||||
report_collaboration: ReportCollaborationConfig = Field(
|
||||
default_factory=ReportCollaborationConfig,
|
||||
description="AgentScope report-collaboration workbench (default off)",
|
||||
|
||||
+148
@@ -1,13 +1,25 @@
|
||||
"""Unified extensions configuration for MCP servers and skills."""
|
||||
|
||||
import copy
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.config.runtime_paths import existing_project_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EXTENSIONS_CONFIG_WRITE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
class McpOAuthConfig(BaseModel):
|
||||
@@ -350,6 +362,142 @@ def reload_extensions_config(config_path: str | None = None) -> ExtensionsConfig
|
||||
return _extensions_config
|
||||
|
||||
|
||||
def _read_raw_extensions_config(config_path: Path) -> dict[str, Any]:
|
||||
"""Read the raw JSON document without resolving env placeholders."""
|
||||
if not config_path.exists():
|
||||
return {}
|
||||
with config_path.open(encoding="utf-8") as config_file:
|
||||
document = json.load(config_file)
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("Extensions config root must be a JSON object")
|
||||
return document
|
||||
|
||||
|
||||
def read_extensions_config_document(config_path: str | Path) -> dict[str, Any]:
|
||||
"""Return a fresh raw document while coordinating with all writers."""
|
||||
path = Path(config_path)
|
||||
with _EXTENSIONS_CONFIG_WRITE_LOCK:
|
||||
with _extensions_config_process_lock(path):
|
||||
return _read_raw_extensions_config(path)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _extensions_config_process_lock(config_path: Path) -> Iterator[None]:
|
||||
"""Serialize writers across processes on Windows and POSIX.
|
||||
|
||||
The stable sidecar is intentional: deleting it after unlock can let two
|
||||
processes lock different inodes. Keeping it beside the config also lets
|
||||
multiple processes sharing the writable config directory coordinate.
|
||||
"""
|
||||
lock_path = config_path.with_name(f".{config_path.name}.lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with lock_path.open("a+b") as lock_file:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
lock_file.seek(0, os.SEEK_END)
|
||||
if lock_file.tell() == 0:
|
||||
lock_file.write(b"\0")
|
||||
lock_file.flush()
|
||||
lock_file.seek(0)
|
||||
while True:
|
||||
try:
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
break
|
||||
except OSError as exc:
|
||||
if exc.errno not in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}:
|
||||
raise
|
||||
time.sleep(0.05)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
lock_file.seek(0)
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _write_raw_extensions_config(config_path: Path, document: dict[str, Any]) -> None:
|
||||
"""Durably replace raw JSON, with a bind-mounted-file EBUSY fallback."""
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = json.dumps(document, indent=2, ensure_ascii=False) + "\n"
|
||||
tmp_path: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
delete=False,
|
||||
dir=str(config_path.parent),
|
||||
) as tmp_file:
|
||||
tmp_file.write(payload)
|
||||
tmp_file.flush()
|
||||
os.fsync(tmp_file.fileno())
|
||||
tmp_path = Path(tmp_file.name)
|
||||
try:
|
||||
os.replace(tmp_path, config_path)
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EBUSY:
|
||||
raise
|
||||
logger.warning(
|
||||
"Atomic replace is unavailable for bind-mounted extensions config %s; using in-place fallback",
|
||||
config_path,
|
||||
)
|
||||
previous = config_path.read_bytes() if config_path.exists() else None
|
||||
try:
|
||||
mode = "r+" if config_path.exists() else "w"
|
||||
with config_path.open(mode, encoding="utf-8") as config_file:
|
||||
config_file.seek(0)
|
||||
config_file.write(payload)
|
||||
config_file.truncate()
|
||||
config_file.flush()
|
||||
os.fsync(config_file.fileno())
|
||||
except Exception:
|
||||
if previous is not None:
|
||||
try:
|
||||
with config_path.open("wb") as config_file:
|
||||
config_file.write(previous)
|
||||
config_file.flush()
|
||||
os.fsync(config_file.fileno())
|
||||
except Exception:
|
||||
logger.exception("Failed to restore extensions config after in-place write failure")
|
||||
raise
|
||||
finally:
|
||||
if tmp_path is not None and tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
|
||||
|
||||
def patch_extensions_config(
|
||||
config_path: str | Path,
|
||||
patch: Callable[[dict[str, Any]], None],
|
||||
) -> ExtensionsConfig:
|
||||
"""Apply a raw-document patch under thread and cross-process locks.
|
||||
|
||||
Callers should mutate only the fields they own. The document is read fresh
|
||||
while locked, so concurrent MCP, skill-state, and display writes cannot
|
||||
overwrite one another or resolve/persist secret ``$ENV`` placeholders.
|
||||
"""
|
||||
path = Path(config_path)
|
||||
with _EXTENSIONS_CONFIG_WRITE_LOCK:
|
||||
with _extensions_config_process_lock(path):
|
||||
document = _read_raw_extensions_config(path)
|
||||
patch(document)
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("Extensions config root must be a JSON object")
|
||||
# Validate a deep copy before persisting. Environment expansion is
|
||||
# intentionally confined to the copy so placeholders remain raw.
|
||||
validation_document = copy.deepcopy(document)
|
||||
ExtensionsConfig.resolve_env_variables(validation_document)
|
||||
ExtensionsConfig.model_validate(validation_document)
|
||||
_write_raw_extensions_config(path, document)
|
||||
return reload_extensions_config(str(path))
|
||||
|
||||
|
||||
def reset_extensions_config() -> None:
|
||||
"""Reset the cached extensions config instance.
|
||||
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
"""Browser runtime configuration for generated HTML artifacts."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class HtmlRuntimeConfig(BaseModel):
|
||||
"""Resources used by standalone HTML pages generated in the sandbox."""
|
||||
|
||||
base_url: str = Field(
|
||||
default="http://localhost:8001/api/public/html-runtime",
|
||||
min_length=1,
|
||||
description="Browser-accessible Gateway base URL for generated HTML runtime files",
|
||||
)
|
||||
@@ -77,6 +77,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
|
||||
base_url: str = ""
|
||||
api_key: str = Field(default="", repr=False)
|
||||
local_model_path: str = ""
|
||||
local_model_revision: str = ""
|
||||
local_cache_dir: str = ""
|
||||
local_process_isolation: bool = True
|
||||
local_threads: int = Field(default=2, ge=1, le=32)
|
||||
@@ -155,6 +156,10 @@ class LocalWikiIndexConfig(BaseModel):
|
||||
min_similarity: float = Field(default=0.35, ge=-1, le=1)
|
||||
keyword_boost_enabled: bool = False
|
||||
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
|
||||
# Lexical title matches receive a bounded score/RRF multiplier rather than
|
||||
# an absolute priority. This is configurable without changing the index.
|
||||
title_boost_enabled: bool = True
|
||||
title_boost_weight: float = Field(default=0.25, ge=0, le=1)
|
||||
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
|
||||
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
|
||||
database_batch_size: int = Field(default=50, ge=1, le=500)
|
||||
|
||||
@@ -12,8 +12,11 @@
|
||||
injection: {...} # 系统提示词注入
|
||||
security: {...} # 内容安全扫描
|
||||
|
||||
两层配置覆盖
|
||||
============
|
||||
运行时总开关与配置覆盖
|
||||
======================
|
||||
管理员维护的 ``system_settings.json -> memory.enabled`` 是权威总开关,默认
|
||||
``false``。关闭时下面所有 provider 参数都不生效。
|
||||
|
||||
``get_effective_memory_config(user_id)`` 按优先级合并 (高 -> 低):
|
||||
1. per-user 覆盖 数据库 ``user_memory_configs`` 表 (经进程内 cache 读取)
|
||||
(有 per-user 记录时自动强制 enabled=True)
|
||||
@@ -32,6 +35,8 @@ from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.config.system_settings import get_memory_runtime_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -126,7 +131,7 @@ class V1MemoryConfig(BaseModel):
|
||||
class MemoryConfig(BaseModel):
|
||||
"""记忆子系统的全局配置 (config.yaml 的 ``memory:`` 段)。"""
|
||||
|
||||
enabled: bool = Field(default=True, description="记忆子系统总开关")
|
||||
enabled: bool = Field(default=False, description="记忆子系统总开关")
|
||||
version: str = Field(
|
||||
default="v2",
|
||||
description="记忆系统版本:v1(原版 DeerFlow,memory.json + LLM 自动提取) 或 v2(USER.md/MEMORY.md)",
|
||||
@@ -253,10 +258,18 @@ def get_recall_budget_from_cache(user_id: str) -> str | None:
|
||||
def get_effective_memory_config(user_id: str) -> MemoryConfig:
|
||||
"""返回某用户的生效记忆配置 (同步,从进程内 cache 读取 per-user 覆盖)。
|
||||
|
||||
优先级 (高 -> 低):per-user DB 覆盖 (经 cache) > config.yaml > 默认值。
|
||||
有 per-user 记录时自动强制 enabled=True。
|
||||
管理员运行时总开关关闭时直接返回 ``enabled=False``;开启后再按
|
||||
per-user DB 覆盖 (经 cache) > config.yaml > 默认值 合并参数。
|
||||
"""
|
||||
# The admin-managed runtime switch is authoritative and defaults to off.
|
||||
# ``config.yaml`` supplies provider parameters only; it cannot silently
|
||||
# turn memory on after a fresh deployment or settings-file reset.
|
||||
runtime_enabled = get_memory_runtime_settings().enabled
|
||||
if not runtime_enabled:
|
||||
return _memory_config.model_copy(update={"enabled": False}, deep=True)
|
||||
|
||||
base = _memory_config.model_dump()
|
||||
base["enabled"] = True
|
||||
recall_budget = _recall_budget_cache.get(user_id)
|
||||
if recall_budget is not None:
|
||||
base.setdefault("hindsight", {})["recall_budget"] = recall_budget
|
||||
|
||||
@@ -10,9 +10,14 @@ class ModelConfig(BaseModel):
|
||||
use: str = Field(
|
||||
...,
|
||||
description="Class path of the model provider(e.g. langchain_openai.ChatOpenAI)",
|
||||
)
|
||||
model: str = Field(..., description="Model name")
|
||||
model_config = ConfigDict(extra="allow")
|
||||
)
|
||||
model: str = Field(..., description="Model name")
|
||||
max_input_tokens: int | None = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
description="Maximum input context tokens DeerFlow may send to the model",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow")
|
||||
use_responses_api: bool | None = Field(
|
||||
default=None,
|
||||
description="Whether to route OpenAI ChatOpenAI calls through the /v1/responses API",
|
||||
|
||||
+16
@@ -18,7 +18,23 @@ class ReportCollaborationConfig(BaseModel):
|
||||
default=False,
|
||||
description="When true, a dispatcher/worker may claim run leases. Gateway-only processes keep this false.",
|
||||
)
|
||||
planner_llm_enabled: bool = Field(
|
||||
default=False,
|
||||
description="P1: LLM planner conversation (clarify + custom strategy). Off keeps the deterministic rule router and template plans.",
|
||||
)
|
||||
planner_model: str | None = Field(default=None, description="Model name from AppConfig.models for planning")
|
||||
planner_timeout_seconds: int = Field(
|
||||
default=120,
|
||||
ge=30,
|
||||
le=600,
|
||||
description="Hard deadline for one streamed workflow-planning model call",
|
||||
)
|
||||
planner_max_tokens: int = Field(
|
||||
default=1800,
|
||||
ge=512,
|
||||
le=8192,
|
||||
description="Maximum output tokens for the workflow-planning model call",
|
||||
)
|
||||
coordinator_model: str | None = Field(default=None, description="Model name from AppConfig.models for the constrained coordinator")
|
||||
research_model: str | None = Field(default=None, description="Model name from AppConfig.models for research members")
|
||||
writer_model: str | None = Field(default=None, description="Model name from AppConfig.models for writing")
|
||||
|
||||
+62
-3
@@ -414,6 +414,22 @@ class AgentModuleSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class WxCommonAgentsSettings(BaseModel):
|
||||
"""The gallery category shown in the wx chat route's common-agent panel.
|
||||
|
||||
``None`` keeps the original recommendation logic. ``mine`` and
|
||||
``sentiment`` are built-in category values. Publish-square categories are
|
||||
persisted as ``square:<square_id>`` so they cannot collide with the
|
||||
built-ins.
|
||||
"""
|
||||
|
||||
agent_type: str | None = Field(
|
||||
default=None,
|
||||
description="Selected gallery category; null retains the default recommendation logic",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class PluginDomainAgentMapping(BaseModel):
|
||||
"""One configured host -> agent mapping for iframe chat auto-selection."""
|
||||
|
||||
@@ -505,14 +521,41 @@ class ModelConcurrencySettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class MemoryRuntimeSettings(BaseModel):
|
||||
"""Deployment-wide switch for Q&A memory.
|
||||
|
||||
Memory is deliberately opt-in. When this setting is false the lead agent
|
||||
does not mount memory middleware or memory tools, does not inject stored
|
||||
memory into prompts, and does not enqueue background memory extraction.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether Q&A memory recall, injection and persistence are enabled.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class LocalWikiEncoderSettings(BaseModel):
|
||||
"""Runtime override for the administrator-managed offline Wiki encoder."""
|
||||
|
||||
mode: Literal["configured", "local"] = Field(
|
||||
default="configured",
|
||||
description=(
|
||||
"Use the embedding provider from config.yaml, or explicitly override it "
|
||||
"with the administrator-managed offline BGE-M3 encoder."
|
||||
),
|
||||
)
|
||||
model_path: str = Field(
|
||||
default="",
|
||||
max_length=4096,
|
||||
description="Absolute or project-relative directory containing the offline BGE-M3 ONNX model.",
|
||||
)
|
||||
model_revision: str = Field(
|
||||
default="",
|
||||
max_length=64,
|
||||
description="Opaque revision changed whenever an offline model directory is activated.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
@@ -539,8 +582,10 @@ class SystemSettings(BaseModel):
|
||||
skill_compression: SkillCompressionSettings = Field(default_factory=SkillCompressionSettings)
|
||||
plugin_management: PluginManagementSettings = Field(default_factory=PluginManagementSettings)
|
||||
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
|
||||
wx_common_agents: WxCommonAgentsSettings = Field(default_factory=WxCommonAgentsSettings)
|
||||
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
|
||||
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
|
||||
memory: MemoryRuntimeSettings = Field(default_factory=MemoryRuntimeSettings)
|
||||
local_wiki_encoder: LocalWikiEncoderSettings = Field(default_factory=LocalWikiEncoderSettings)
|
||||
strategy_projects: StrategyProjectSettings = Field(default_factory=StrategyProjectSettings)
|
||||
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
|
||||
@@ -602,12 +647,26 @@ def get_model_concurrency_settings() -> ModelConcurrencySettings:
|
||||
return ModelConcurrencySettings()
|
||||
|
||||
|
||||
def get_memory_runtime_settings() -> MemoryRuntimeSettings:
|
||||
"""Return the admin-controlled Q&A memory switch, failing closed."""
|
||||
try:
|
||||
return load_system_settings().memory
|
||||
except Exception:
|
||||
logger.debug("Failed to read memory runtime setting; keeping memory disabled", exc_info=True)
|
||||
return MemoryRuntimeSettings()
|
||||
|
||||
|
||||
def save_system_settings(settings: SystemSettings) -> SystemSettings:
|
||||
"""Persist the settings document atomically, returning the saved value."""
|
||||
path = _settings_path()
|
||||
payload = settings.model_dump(mode="json")
|
||||
with _lock:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
# Multiple Gateway workers share this file. A unique temporary name
|
||||
# prevents their atomic replaces from racing over one common `.tmp`.
|
||||
tmp = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
|
||||
try:
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
return settings
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Web App 产物模式配置(多页面站点生成/构建/预览)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class WebAppConfig(BaseModel):
|
||||
"""站点产物(outputs/ 下的 site.json 站点)总开关与构建参数。
|
||||
|
||||
默认关闭:模板未预置 node_modules 或未启用时,所有站点 API 返回
|
||||
结构化的"不可用"响应,智能体技能自动回退单页 HTML 模式,
|
||||
不影响任何既有功能。
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="总开关:关闭时 capability 返回不可用,入口按钮不渲染,技能回退单页 HTML",
|
||||
)
|
||||
template_dir: str = Field(
|
||||
default="templates/web-app-template",
|
||||
description="站点模板目录(相对 project root 或绝对路径;含预置 node_modules 与 scripts/build_site.py)",
|
||||
)
|
||||
build_timeout_seconds: int = Field(
|
||||
default=180,
|
||||
ge=10,
|
||||
le=1800,
|
||||
description="单次 vite build 超时(秒),超时强杀子进程",
|
||||
)
|
||||
preview_token_ttl_seconds: int = Field(
|
||||
default=7200,
|
||||
ge=60,
|
||||
le=86400,
|
||||
description="站点预览签名 URL 有效期(秒)",
|
||||
)
|
||||
node_path: str = Field(
|
||||
default="",
|
||||
description="node 可执行文件覆盖(空 = PATH 查找;一般无需配置)",
|
||||
)
|
||||
Reference in new issue
Block a user