Merge remote-tracking branch 'origin/master' into xuhd

# Conflicts:
#	offline-backend-20260512/backend/config.yaml
This commit is contained in:
xuhd committed 2026-06-09 15:43:40 +08:00
commit bd0eceda61
267 files changed
+41209 -14541

No files matched your search

@@ -0,0 +1,92 @@
"""Server-side mirror of the frontend agent-name desensitization map.
The web UI displays agent names through a static term→abbreviation map
(``frontend-web/src/lib/desensitize.ts``) — e.g. the stored name "态势分析" is
shown as "TS分析". The gallery search box therefore matches against the
*desensitized* display form, not the raw stored name.
When search moves server-side (paginated listing), the backend must match the
same way, so this module mirrors that static map verbatim. The frontend map is
never extended at runtime (``addDesensitizeEntries`` is defined but never
called), so a static copy stays in sync.
Used by the SQL listing query to match "raw name OR desensitized name".
"""
from __future__ import annotations
# Mirror of DESENSITIZE_MAP in frontend-web/src/lib/desensitize.ts.
# Keep entries identical to the frontend so server-side and client-side search
# behave the same. Order does not affect membership tests.
DESENSITIZE_MAP: dict[str, str] = {
"态势": "TS",
"你好": "nh",
"滨海作战体系": "第一张网",
"滨海作战部队": "第一张网",
"Taiwan Air Force Flight Units": "第二张网",
"台海空军飞行部队": "第二张网",
"空军": "KJ",
"海军": "HJ",
"陆军": "LJ",
"陆战队": "LZD",
"海巡": "HX",
"憲兵": "XB",
"資通電軍": "ZTDJ",
"情报": "QB",
"指挥": "ZH",
"信息准备": "XX准备",
"行为体": "XWT",
"驱动因素": "QDYS",
"叙事": "XS",
"前三定": "QSD",
"受众目标": "SZMB",
"作战": "ZZ",
"目的": "MD",
"关键传播节点": "GJCBJD",
"战法": "ZF",
"行动": "XD",
"打击": "DJ",
"目标": "MB",
"战力": "ZL",
"筹划": "COH",
"策划": "CEH",
"模拟": "MN",
"推演": "TY",
"认知": "RZ",
"触及": "CJ",
"军": "J",
"美军": "MJ",
"台军": "TJ",
}
def desensitize_name(name: str) -> str:
"""Return ``name`` with every mapped term replaced by its abbreviation.
Mirrors the frontend ``desensitize()`` — a sequence of literal replacements.
"""
out = name
for term, abbr in DESENSITIZE_MAP.items():
if term in out:
out = out.replace(term, abbr)
return out
def name_matches(name: str, needle: str) -> bool:
"""True when ``needle`` (case-insensitive) is in the raw OR desensitized name."""
n = needle.strip().lower()
if not n:
return True
return n in name.lower() or n in desensitize_name(name).lower()
def source_terms_for_needle(needle: str) -> list[str]:
"""Return the raw source terms whose abbreviation contains ``needle``.
Lets the SQL search add ``name LIKE %term%`` clauses so a user typing an
abbreviation (e.g. "TS") still matches the stored raw term (e.g. "态势").
"""
n = needle.strip().lower()
if not n:
return []
return [term for term, abbr in DESENSITIZE_MAP.items() if n in abbr.lower()]
@@ -15,6 +15,7 @@ from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
from deerflow.config.model_config import ModelConfig
@@ -108,6 +109,7 @@ class AppConfig(BaseModel):
models: list[ModelConfig] = Field(default_factory=list, description="Available models")
vision: VisionConfig = Field(default_factory=VisionConfig, description="Image recognition (识图) configuration")
auth_login: AuthLoginConfig = Field(default_factory=AuthLoginConfig, description="Custom login modes (username password gate, token exchange)")
log_system: LogSystemConfig = Field(default_factory=LogSystemConfig, description="Admin-only external log-system entry shown in the settings menu")
sandbox: SandboxConfig = Field(description="Sandbox configuration")
tools: list[ToolConfig] = Field(default_factory=list, description="Available tools")
tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups")
@@ -2,7 +2,7 @@
from typing import Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field
CheckpointerType = Literal["memory", "sqlite", "postgres"]
@@ -16,13 +16,41 @@ class CheckpointerConfig(BaseModel):
"'sqlite' persists to a local file (requires langgraph-checkpoint-sqlite). "
"'postgres' persists to PostgreSQL (requires langgraph-checkpoint-postgres)."
)
connection_string: str | None = Field(
default=None,
description="Connection string for sqlite (file path) or postgres (DSN). "
connection_string: str | None = Field(
default=None,
description="Connection string for sqlite (file path) or postgres (DSN). "
"Required for sqlite and postgres types. "
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
)
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
)
use_postgres: bool = Field(
default=True,
description=(
"When type='postgres', set false to keep the Postgres DSN in config "
"but run the checkpointer on SQLite instead."
),
)
sqlite_connection_string: str | None = Field(
default=None,
description=(
"SQLite file used when type='postgres' and use_postgres=false. "
"Defaults to the first dual_read_sqlite_paths entry or 'store.db'."
),
)
dual_read_enabled: bool = Field(
default=False,
description=(
"When type='postgres', read from Postgres first and fall back to legacy "
"SQLite checkpoint files when Postgres does not have the requested data."
),
)
dual_read_sqlite_paths: list[str] = Field(
default_factory=list,
description=(
"Legacy SQLite checkpoint database paths used for read fallback. "
"Plain paths are resolved the same way as SQLite connection_string."
),
)
# Global configuration instance — None means no checkpointer is configured.
@@ -11,13 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
No database is initialized.
@@ -26,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
to reference environment variables from .env:
database:
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
backend: postgres
postgres_url: $DATABASE_URL
The $VAR resolution is handled by AppConfig.resolve_env_variables()
before this config is instantiated -- DatabaseConfig itself does not
need to do any environment variable processing.
"""
from __future__ import annotations
@@ -42,40 +42,54 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
)
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
mysql_url: str = Field(
default="",
description=(
"MySQL connection URL for the application ORM. "
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
),
)
),
)
mysql_url: str = Field(
default="",
description=(
"MySQL connection URL for the application ORM. "
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
),
)
echo_sql: bool = Field(
default=False,
description="Echo all SQL statements to log (debug only).",
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres/mysql only).",
description="Connection pool size for the app ORM engine (postgres/mysql only).",
)
max_overflow: int = Field(
default=10,
description=(
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
),
)
pool_timeout: int = Field(
default=30,
description=(
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
"SQLAlchemy default is 30."
),
)
# -- Derived helpers (not user-configured) --
@@ -108,16 +122,16 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
if url.startswith("mysql://"):
url = url.replace("mysql://", "mysql+asyncmy://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
if url.startswith("mysql://"):
url = url.replace("mysql://", "mysql+asyncmy://", 1)
return url
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
@@ -0,0 +1,18 @@
"""Configuration for the admin-only external log-system entry.
A small read-only section in ``config.yaml`` that lets a deployment point the
"日志系统" entry (shown in the bottom-right settings menu for admins) at an
external log/observability system. The URL is opened in a new browser tab.
"""
from __future__ import annotations
from pydantic import BaseModel, Field
class LogSystemConfig(BaseModel):
"""External log-system link surfaced in the admin settings menu."""
enabled: bool = Field(default=False, description="Whether the log-system entry is shown in the settings menu")
url: str = Field(default="", description="External log-system URL opened in a new tab when the entry is clicked")
label: str = Field(default="日志系统", description="Menu label shown for the entry")
@@ -51,6 +51,14 @@ class AppearanceSettings(BaseModel):
default="light",
description="Theme applied for users who haven't picked their own (light / dark)",
)
default_skill_view: Literal["card", "list"] = Field(
default="list",
description="Default skill list display mode for users who haven't picked their own (card / list)",
)
default_agent_view: Literal["card", "list"] = Field(
default="card",
description="Default agent list display mode for users who haven't picked their own (card / list)",
)
model_config = ConfigDict(extra="ignore")
@@ -78,6 +86,17 @@ class PrivateSquareSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class LeaderboardSettings(BaseModel):
"""Admin-configurable defaults for the user leaderboard analytics."""
exclude_user_ids: list[str] = Field(default_factory=list, description="User ids excluded from leaderboard aggregation.")
include_scheduled: bool = Field(default=True, description="Whether scheduled task runs participate in leaderboard statistics.")
include_admins: bool = Field(default=True, description="Whether admin accounts participate in leaderboard statistics.")
include_failed: bool = Field(default=True, description="Whether failed runs and provider calls participate in leaderboard statistics.")
cleanup_system_questions: bool = Field(default=True, description="Hide attachment/system marker questions such as <uploaded_files>.")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -85,6 +104,7 @@ class SystemSettings(BaseModel):
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
model_config = ConfigDict(extra="ignore")