Merge branch 'ji' into master

This commit is contained in:
Administrator committed 2026-06-12 18:28:13 +08:00
commit e74ef4b7dc
81 files changed
+3712 -1625

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -100,7 +100,7 @@ async def _ensure_admin_user(app: FastAPI) -> None:
After admin creation, migrate orphan threads from the LangGraph
store (metadata.user_id unset) to the admin account. This is the
"no-auth → with-auth" upgrade path: users who ran DeerFlow without
"no-auth ? with-auth" upgrade path: users who ran DeerFlow without
authentication have existing LangGraph thread data that needs an
owner assigned.
First boot (no admin exists):
@@ -108,7 +108,7 @@ async def _ensure_admin_user(app: FastAPI) -> None:
- The operator must visit ``/setup`` to create the first admin.
Subsequent boots (admin already exists):
- Runs the one-time "no-auth → with-auth" orphan thread migration for
- Runs the one-time "no-auth ? with-auth" orphan thread migration for
existing LangGraph thread metadata that has no owner_id.
No SQL persistence migration is needed: the four user_id columns
@@ -138,12 +138,12 @@ async def _ensure_admin_user(app: FastAPI) -> None:
if admin_count == 0:
logger.info("=" * 60)
logger.info(" First boot detected — no admin account exists.")
logger.info(" First boot detected ? no admin account exists.")
logger.info(" Visit /setup to complete admin account creation.")
logger.info("=" * 60)
return
# Admin already exists — run orphan thread migration for any
# Admin already exists ? run orphan thread migration for any
# LangGraph thread metadata that pre-dates the auth module.
async with sf() as session:
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
@@ -154,8 +154,8 @@ async def _ensure_admin_user(app: FastAPI) -> None:
admin_id = str(row.id)
# LangGraph store orphan migration — non-fatal.
# This covers the "no-auth → with-auth" upgrade path for users
# LangGraph store orphan migration ? non-fatal.
# This covers the "no-auth ? with-auth" upgrade path for users
# whose existing LangGraph thread metadata has no user_id set.
store = getattr(app.state, "store", None)
if store is not None:
@@ -217,7 +217,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
# No admin yet — leave ownership null; first admin is responsible for
# No admin yet ? leave ownership null; first admin is responsible for
# claiming via the UI once they exist.
admin_id: str | None = None
else:
@@ -278,7 +278,7 @@ async def _run_db_migrations() -> None:
engine = get_engine()
if engine is None:
logger.info("Persistence backend=memory — skipping Alembic migrations")
logger.info("Persistence backend=memory ? skipping Alembic migrations")
return
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
@@ -327,28 +327,28 @@ async def _run_db_migrations() -> None:
raise result
logger.info("Database migrations applied (head)")
except Exception:
logger.exception("Database migration failed — schema may be out of date")
logger.exception("Database migration failed ? schema may be out of date")
# Skill curator 调度参数。
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
# Skill curator ?????
# - ????:?????????"??????";??????? should_run_now() ???
# - ????:?????????????,???????
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 ??
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 ??
def _acquire_singleton_lock(lock_path):
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
"""???????????,??? worker ?? leader ???
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
进程异常退出时 OS 会自动释放该锁。
?? -> ??????????(?????????,??????/????);
????(???? worker ??)????? -> ?? None?
??????? OS ????????
"""
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = open(lock_path, "a+")
except OSError:
logger.exception("Curator scheduler: 打开锁文件失败")
logger.exception("Curator scheduler: ???????")
return None
try:
if sys.platform == "win32":
@@ -367,7 +367,7 @@ def _acquire_singleton_lock(lock_path):
def _release_singleton_lock(handle) -> None:
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
"""????? _acquire_singleton_lock ?????????"""
try:
if sys.platform == "win32":
import msvcrt
@@ -388,24 +388,24 @@ def _release_singleton_lock(handle) -> None:
async def _curator_scheduler_loop() -> None:
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
"""??????:?? _CURATOR_CHECK_INTERVAL_SECONDS ???????????
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
? worker ???????? leader ?? ?? ???????? worker ????
????,?? worker ????,?? curator ?????????? LLM?
? cancel ? CancelledError ??? ``except Exception``(?? BaseException
??,?????),?????????? finally ?????
"""
from deerflow.config.runtime_paths import runtime_home
from deerflow.skills.curator import maybe_run_curator
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
if lock_handle is None:
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
logger.info("Curator scheduler: ??? worker ??????,? worker ??")
return
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
logger.info("Curator scheduler: ?????,? worker ?? curator ??")
try:
# 启动后延迟一会儿再首次检查,避开启动高峰
# ?????????????,??????
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
while True:
try:
@@ -477,7 +477,7 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
)
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
# *previous* day's snapshot once, so records that landed near midnight are fully
# captured. Default 1 → "每天凌晨 1 点重新统计前一天".
# captured. Default 1 ? "???? 1 ????????".
_LEADERBOARD_FINALIZE_HOUR = min(
23,
_env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
@@ -648,7 +648,7 @@ async def _claim_leaderboard_snapshot(
if age < running_ttl_seconds:
return False
row.status = "running"
row.message = "排行榜后台统计任务已认领。"
row.message = "?????????????"
row.error = None
row.updated_at = datetime.now(UTC)
await _time_leaderboard_scheduler_db_operation(
@@ -701,7 +701,7 @@ async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_has
[yesterday],
settings_hash,
force=True,
message="每日凌晨重新统计前一天完整数据,确保全部记录纳入统计。",
message="???????????????????????????",
)
if queued:
logger.info(
@@ -734,10 +734,10 @@ async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
session_factory,
stat_dates,
settings_hash,
message="排行榜后台统计任务定期预热/补算队列。",
message="?????????????/?????",
)
# 每天凌晨(默认 1 点)后强制重算"前一天"的完整快照一次,覆盖跨午夜才落库的记录。
# ??????? 1 ???????"???"????????????????????
try:
await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
except Exception:
@@ -816,31 +816,31 @@ async def _leaderboard_snapshot_scheduler_loop() -> None:
def _warn_if_multi_worker() -> None:
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
"""?? uvicorn / gunicorn ? worker ???? ERROR ???
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
维护 run_id → SSE 订阅者的内存 map。多 worker 下,用户在 worker A 发起
``/runs/stream``、稍后在 worker B 调 join/resume,会因 in-process map 互相
看不见而 404 / 卡死。
? agent / canvas / AI ??????? Gateway ????? ``StreamBridge``
?? run_id ? SSE ?????? map?? worker ????? worker A ??
``/runs/stream``???? worker B ? join/resume??? in-process map ??
???? 404 / ???
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
????? startup ? ERROR ????**???? fail-fast** ?? ??
???????????????? sticky session??????????
"""
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
# / uvicorn 都认)。
# uvicorn / gunicorn ??????? worker ?????????????
# ?????? sibling ??????????? WEB_CONCURRENCY?gunicorn
# / uvicorn ????
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
logger.error(
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
"? ??? WEB_CONCURRENCY=%s?? worker ????AI ?? / ? agent "
"??????????????? worker ????????? / "
"SSE ?? / resume 400??????????? WEB_CONCURRENCY=1 "
"????????? sticky session??? AIWritingPage.tsx ????",
web_concurrency,
)
return
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
# ???? /proc ??? uvicorn ?????? Linux ??????
if not sys.platform.startswith("linux"):
return
try:
@@ -858,13 +858,13 @@ def _warn_if_multi_worker() -> None:
peers += 1
if peers > 1:
logger.error(
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
"启用 sticky session。",
"? ??? %d ??? uvicorn ???? worker ????? AI ?? "
"????? / SSE ??????????? worker ??????? "
"?? sticky session?",
peers,
)
except Exception:
# 检测本身失败不算事 —— 这只是个提醒
# ????????? ?? ??????
pass
@@ -872,8 +872,8 @@ def _warn_if_multi_worker() -> None:
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan handler."""
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
# ??????? worker ???????? ERROR ?????
_warn_if_multi_worker()
# Load config and check necessary environment variables at startup
@@ -895,7 +895,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# Run Alembic migrations before anything else touches the schema.
await _run_db_migrations()
# 预热 recall_budget cache(从 DB 批量加载,使 get_effective_memory_config 同步可读)
# ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
try:
from deerflow.config.memory_config import load_recall_budget_cache
@@ -911,18 +911,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# Must run AFTER langgraph_runtime so app.state.store is available for thread migration
await _ensure_admin_user(app)
# Migrate "ownerless" on-disk skills into the new SkillStore table —
# Migrate "ownerless" on-disk skills into the new SkillStore table ?
# makes the multitenancy switch transparent for existing deployments.
try:
await _sync_legacy_skills(app)
except Exception:
logger.exception("Skill ownership migration failed (non-fatal)")
# 圆桌功能型 agent(intent / recommender / coordinator)缺失自动补建。
# 必须在 _sync_legacy_agents **之前**——这样刚补建的目录能在同一次启动被
# 扫描并 upsert 进 agents 表,新机器一启动 /api/agents 就有这三个,
# 无需用户先打开圆桌页触发懒加载。.deer-flow/ 被 gitignore,新机器 clone
# 后这些目录不存在,seeder 从源码资源 _roundtable_seed_assets/ 复制验证版。
# ????? agent(intent / recommender / coordinator)???????
# ??? _sync_legacy_agents **??**??????????????????
# ??? upsert ? agents ?,?????? /api/agents ?????,
# ????????????????.deer-flow/ ? gitignore,??? clone
# ????????,seeder ????? _roundtable_seed_assets/ ??????
try:
from app.gateway.routers._roundtable_seed import (
ensure_roundtable_functional_agents,
@@ -934,9 +934,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"Roundtable functional agent seeding failed at startup (non-fatal)"
)
# AI 写作功能型 agent(researcher / outliner / writer / editor)缺失自动补建。
# 同圆桌:必须在 _sync_legacy_agents 之前,补建目录才能在同一次启动被
# upsert 进 agents 表,admin 启动后即可在智能体管理页编辑四个写作 agent。
# AI ????? agent(researcher / outliner / writer / editor)???????
# ???:??? _sync_legacy_agents ??,?????????????
# upsert ? agents ?,admin ?????????????????? agent?
try:
from app.gateway.routers._ai_writing_seed import (
ensure_ai_writing_functional_agents,
@@ -948,7 +948,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"AI-writing functional agent seeding failed at startup (non-fatal)"
)
# 业务链条管理:幂等补建固定业务行(3Q/6BF/7BF/8BF),agent/chain 初始为空。
# ?????????????????3Q/6BF/7BF/8BF??agent/chain ?????
try:
business_mapping_store = getattr(app.state, "business_mapping_store", None)
if business_mapping_store is not None:
@@ -958,10 +958,10 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Business mapping seeding failed at startup (non-fatal)")
# 一次性把 .deer-flow/agents/{id}/ 目录里历史遗留的 agent 导入 agents 表。
# 路由层原本每次请求都跑一遍这个 sync(扫文件系统 + N 个 agent 各一次 DB
# SELECT/UPDATE),把它挪到启动期 + 路由层只保留 TTL 兜底,显著降低
# /api/agents 列表延迟。
# ???? .deer-flow/agents/{id}/ ???????? agent ?? agents ??
# ??????????????? sync(????? + N ? agent ??? DB
# SELECT/UPDATE),??????? + ?????? TTL ??,????
# /api/agents ?????
try:
from app.gateway.routers.agents import (
_mark_legacy_sync_fresh,
@@ -974,7 +974,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
await _sync_legacy_agents(agent_store)
_mark_legacy_sync_fresh()
# ``services.py`` keeps an independent TTL gate (used by
# ``start_run`` → ``_enforce_agent_access``). Without this
# ``start_run`` ? ``_enforce_agent_access``). Without this
# ack, the first ``/runs/stream`` after startup would
# re-run the heavy sync even though we just did it.
_services_mark_sync_fresh()
@@ -982,17 +982,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Legacy agent sync failed at startup (non-fatal)")
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
# ?? skill curator ???:??????,?????????????
# (per-user interval_hours ??????????????)?
try:
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
except Exception:
logger.exception("Skill curator scheduler failed to start (non-fatal)")
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
# ?? AI ????????:? config.yaml ? ai_writing.cleanup ???,
# ?????? 3 ????,? retention_days(?? 7 ?)?????,??
# ?? MySQL ??? + SQLite checkpointer ?? thread ???? worker
# ?????? leader,?????? worker ??????
try:
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
@@ -1031,7 +1031,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
yield
# 停止 curator 调度器
# ?? curator ???
curator_task = getattr(app.state, "curator_scheduler_task", None)
if curator_task is not None:
curator_task.cancel()
@@ -1042,7 +1042,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Curator scheduler shutdown error")
# 停止 AI 写作清理调度器
# ?? AI ???????
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
if cleanup_task is not None:
cleanup_task.cancel()
@@ -1063,9 +1063,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Leaderboard snapshot scheduler shutdown error")
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
# ``langgraph_runtime()`` 的 contextmanager 统一关闭,无需在这里特殊处理。
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
# ``langgraph_runtime()`` ? contextmanager ???????????????
try:
scheduler_service = getattr(app.state, "scheduled_task_service", None)
@@ -1197,7 +1197,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# CORS
if is_auth_disabled():
logger.warning(
"DEER_FLOW_AUTH_DISABLED is set — API authentication is bypassed. "
"DEER_FLOW_AUTH_DISABLED is set ? API authentication is bypassed. "
"Do not use this in production."
)
@@ -1222,7 +1222,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
if _cors_allow_any_origin():
logger.warning(
"Permissive CORS enabled — any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
"Permissive CORS enabled ? any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
"DEER_FLOW_AUTH_DISABLED). Do not expose publicly."
)
app.add_middleware(
@@ -1238,7 +1238,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
elif cors_origins:
if "*" in cors_origins:
logger.error(
"GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True — invalid. "
"GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True ? invalid. "
"Set GATEWAY_CORS_ALLOW_ALL=1 instead, or list explicit origins."
)
cors_origins = [o for o in cors_origins if o != "*"]
@@ -1319,10 +1319,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Roundtable-planning business chains at /api/roundtable-chains
app.include_router(roundtable_chains.router)
# Global business → agent / chain mapping at /api/business-mapping
# Global business ? agent / chain mapping at /api/business-mapping
app.include_router(business_mapping.router)
# Light-app registry (轻应用中心) at /api/light-apps
# Light-app registry (?????) at /api/light-apps
app.include_router(light_apps.router)
# Standalone parallel multi-agent panel: fan-out streaming + saved tasks
@@ -1400,7 +1400,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Public, unauthenticated read-only share snapshots (/api/public/shares/*).
app.include_router(thread_shares.public_router)
# 圆桌会商草稿分享(import 复制副本 / view 只读结果报告页)
# ?????????import ???? / view ????????
app.include_router(roundtable_draft_shares.router)
# Public, unauthenticated read-only roundtable result (/api/public/roundtable-shares/*).
app.include_router(roundtable_draft_shares.public_router)
@@ -1408,17 +1408,17 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Stateless Runs API (stream/wait without a pre-existing thread)
app.include_router(runs.router)
# --- 圆桌规划三步流(前端 /page/workspace/roundtable/planning)---
# 实现见 app/gateway/routers/{multi_agent,intent,recommend}.py;
# 文档见 frontend-web/docs/multi-agent-backend-dev.md
app.include_router(multi_agent.router) # Step 2:init(SSE) + run/stream(leader|special)
app.include_router(intent.router) # Step 1:/api/intent/init + /stream
app.include_router(recommend.router) # Step 1→2:/api/recommend/stream(无 init)
# Step 3 产物列表/下载复用下方 artifacts.router(/api/threads/{id}/artifacts)
# --- ?????????? /page/workspace/roundtable/planning?---
# ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
# ??? frontend-web/docs/multi-agent-backend-dev.md
app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
app.include_router(intent.router) # Step 1?/api/intent/init + /stream
app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
# Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
# 探活端点:无需鉴权(见 auth_middleware._PUBLIC_PATH_PREFIXES)。
# 同时暴露裸 ``/health`` 与带 API 前缀的 ``/api/health``,方便监控按
# 任意 base URL 直接探活,不会被认证中间件拦成 401。
# ?????????(? auth_middleware._PUBLIC_PATH_PREFIXES)?
# ????? ``/health`` ?? API ??? ``/api/health``,?????
# ?? base URL ????,?????????? 401?
@app.get("/health", tags=["health"])
@app.get("/api/health", tags=["health"])
async def health_check() -> dict:
@@ -31,21 +31,43 @@ from __future__ import annotations
import functools
import inspect
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
import logging
from collections.abc import Callable
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from fastapi import HTTPException, Request
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
if TYPE_CHECKING:
from app.gateway.auth.models import User
logger = logging.getLogger(__name__)
P = ParamSpec("P")
T = TypeVar("T")
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
A checkpoint with no matching ``threads_meta`` row is the signature of a
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
survives in the checkpointer while the business row is gone. Normal thread
deletion removes the checkpoint too, so this can never resurrect a
deliberately-deleted thread.
"""
try:
from app.gateway.deps import get_checkpointer
checkpointer = get_checkpointer(request)
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
return tup is not None
except Exception:
return False
# Permission constants
class Permissions:
"""Permission constants for resource:action format."""
@@ -205,6 +227,7 @@ def require_permission(
action: str,
owner_check: bool = False,
require_existing: bool = False,
claim_orphan: bool = False,
) -> Callable[[Callable[P, T]], Callable[P, T]]:
"""Decorator that checks permission for resource:action.
@@ -221,6 +244,15 @@ def require_permission(
**destructive / mutating** routes (DELETE, PATCH,
state-update) so a deleted thread can't be re-targeted
by another user via the missing-row code path.
claim_orphan: Only meaningful with ``owner_check=True`` and
``require_existing=True``. When the ``threads_meta``
row is missing **and** a LangGraph checkpoint exists
for the thread (a DB-switch orphan), adopt it for the
current user instead of 404ing. Use on **run-create**
routes so a conversation whose business row was lost
(sqlite↔mysql swap) stays answerable. Safe: normal
deletion removes the checkpoint, so this never revives
a deliberately-deleted thread.
Usage:
# Read-style: legacy untracked threads are allowed
@@ -289,16 +321,44 @@ def require_permission(
from app.gateway.deps import get_thread_store
thread_store = get_thread_store(request)
user_id = str(auth.user.id)
allowed = await thread_store.check_access(
thread_id,
str(auth.user.id),
user_id,
require_existing=require_existing,
)
if not allowed:
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not found",
)
# Tell apart three denial causes so the client gets an
# accurate message and so DB-switch orphans can be adopted
# rather than hard-404'd:
# 1. row exists but owned by another user -> not accessible
# 2. no row, no checkpoint -> truly missing
# 3. no row + checkpoint + claim_orphan -> orphan, claim
existing = await thread_store.get(thread_id, user_id=None)
if existing is not None:
# Distinct detail string lets the frontend show "无权访问"
# instead of "不存在" for a cross-user / stale-session hit.
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not accessible",
)
claimed = False
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
try:
await thread_store.create(thread_id, user_id=user_id)
logger.info(
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
thread_id,
user_id,
)
claimed = True
except Exception:
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
if not claimed:
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not found",
)
return await func(*args, **kwargs)
@@ -59,6 +59,11 @@ class AgentResponse(BaseModel):
description="Recommended questions shown on the agent chat home / sidebar (常用问题). Each item is {title, prompt}; legacy string lists are coerced.",
)
template_name: str | None = Field(
default=None,
description="入库模板名 (external knowledge-ingest templateName). When set, markdown artifacts produced in this agent's chats can be published to the external knowledge-ingest API.",
)
tags: list[dict] = Field(default_factory=list, description="Tags assigned to this agent")
featured_order: int | None = Field(
@@ -109,6 +114,7 @@ class AgentCreateRequest(BaseModel):
model: str | None = Field(default=None, description="Optional model override persisted to config.yaml")
tool_groups: list[str] | None = Field(default=None, description="Optional tool group whitelist persisted to config.yaml")
common_questions: list[CommonQuestion] | None = Field(default=None, description="Recommended questions (常用问题) persisted to config.yaml. Each item is {title, prompt}; legacy strings are coerced.")
template_name: str | None = Field(default=None, description="入库模板名 persisted to config.yaml. Enables publishing markdown artifacts to the external knowledge-ingest API.")
class AgentUpdateRequest(BaseModel):
@@ -124,6 +130,7 @@ class AgentUpdateRequest(BaseModel):
model: str | None = Field(default=None, description="Updated model override (config.yaml)")
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
common_questions: list[CommonQuestion] | None = Field(default=None, description="Updated recommended questions (常用问题, config.yaml). Each item is {title, prompt}; legacy strings are coerced.")
template_name: str | None = Field(default=None, description="Updated 入库模板名 (config.yaml). Empty string clears it.")
class AgentSelectorResponse(BaseModel):
@@ -228,6 +235,7 @@ def _write_agent_config(
model: str | None = None,
tool_groups: list[str] | None = None,
common_questions: list[CommonQuestion] | None = None,
template_name: str | None = None,
) -> None:
agent_dir = get_paths().agent_dir(agent_id)
agent_dir.mkdir(parents=True, exist_ok=True)
@@ -254,13 +262,15 @@ def _write_agent_config(
config_data["common_questions"] = [
q.model_dump() for q in normalized if q.prompt.strip()
]
if template_name is not None and template_name.strip():
config_data["template_name"] = template_name.strip()
with open(config_file, "w", encoding="utf-8") as f:
yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)
def _load_config_extras(agent_id: str) -> dict[str, Any]:
"""Read skills/model/tool_groups/common_questions from the agent's config.yaml, tolerant of missing files."""
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None, "template_name": None}
try:
cfg = load_agent_config(agent_id)
except (FileNotFoundError, ValueError):
@@ -272,6 +282,7 @@ def _load_config_extras(agent_id: str) -> dict[str, Any]:
"model": cfg.model,
"tool_groups": cfg.tool_groups,
"common_questions": cfg.common_questions,
"template_name": cfg.template_name,
}
@@ -432,7 +443,7 @@ def _record_to_response(
can happen lazily when the agent is actually opened.
"""
soul: str | None = None
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None, "template_name": None}
if include_extras:
_ensure_agent_dir_from_record(record)
extras = _load_config_extras(record["id"])
@@ -455,6 +466,7 @@ def _record_to_response(
tool_groups=extras["tool_groups"],
skills=extras["skills"],
common_questions=extras.get("common_questions"),
template_name=extras.get("template_name"),
featured_order=record.get("featured_order"),
)
@@ -723,6 +735,10 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
await _maybe_sync_legacy_agents(store)
user_id = _current_user_id(request)
record = await store.get_visible(agent_id, user_id)
if record is None and _is_admin_user(request):
# The admin console lists every agent (?admin=true), so an admin
# must be able to open any of them — mirror that here.
record = await store.get_any(agent_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
@@ -773,6 +789,7 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
model=request.model,
tool_groups=request.tool_groups,
common_questions=request.common_questions,
template_name=request.template_name,
)
_write_agent_soul(agent_id, request.soul)
@@ -852,8 +869,9 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
next_model = request.model if "model" in fields_set else existing_extras["model"]
next_tool_groups = request.tool_groups if "tool_groups" in fields_set else existing_extras["tool_groups"]
next_common_questions = request.common_questions if "common_questions" in fields_set else existing_extras["common_questions"]
next_template_name = request.template_name if "template_name" in fields_set else existing_extras["template_name"]
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions"))
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions", "template_name"))
if config_dirty:
_write_agent_config(
agent_id,
@@ -863,6 +881,7 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
model=next_model,
tool_groups=next_tool_groups,
common_questions=next_common_questions,
template_name=next_template_name,
)
# Keep the listing cache in lock-step with config.yaml, but only
# when the cached fields were actually touched — a partial update
@@ -424,6 +424,7 @@ async def list_notes(
request: Request,
keyword: str | None = None,
tag: str | None = None,
entity: str | None = None,
source_type: str | None = None,
status: str | None = None,
limit: int = 20,
@@ -432,7 +433,7 @@ async def list_notes(
service = get_knowledge_service(request)
limit = max(1, min(limit, 100))
offset = max(0, offset)
return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
return await service.list_notes(keyword=keyword, tag=tag, entity=entity, source_type=source_type, status=status, limit=limit, offset=offset)
@router.get("/notes/{note_id}")
@@ -0,0 +1,206 @@
"""外部知识入库接口(发布 Markdown 产物到 knowes 知识库)。
智能体 config.yaml 配置了 ``template_name`` 后,该智能体会话中生成的 Markdown
产物可以通过 ``POST /api/knowledge-ingest/publish`` 一键入库。
入库目标接口的字段名不固定:标题字段可能叫「名称」也可能叫「标题」,因此先按
「名称」发送一次,若判定失败则用 **同一个 nodeName** 把标题字段换成「标题」再
发一次。nodeName 是入库记录的唯一标识,取「md 标题 + 时间戳」。
"""
from __future__ import annotations
import json
import logging
import os
import hashlib
from datetime import datetime
from typing import Any
import httpx
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, Field
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/knowledge-ingest", tags=["knowledge-ingest"])
# 入库接口地址,可用环境变量覆盖(如切换 UAT/生产)。
_REQUEST_TIMEOUT_SECONDS = 30.0
# 标题字段在目标模板里可能叫「名称」或「标题」,按此顺序逐个尝试。
_TITLE_FIELD_CANDIDATES = ("名称", "标题")
_CONTENT_FIELD = "内容"
class KnowledgeIngestPublishRequest(BaseModel):
"""Request body for publishing one markdown artifact."""
agent_id: str = Field(..., description="智能体 id,用于读取其 config.yaml 中的 template_name")
title: str = Field(..., description="md 标题(一级标题或文件名)")
content: str = Field(..., description="md 正文全文")
class KnowledgeIngestAttempt(BaseModel):
"""One delivery attempt against the ingest API."""
title_field: str = Field(..., description="本次尝试使用的标题字段名(名称/标题)")
status_code: int | None = Field(default=None, description="HTTP 状态码;请求异常时为 null")
response_text: str = Field(default="", description="入库接口原始响应(截断到 2000 字符)")
success: bool = False
error: str | None = Field(default=None, description="请求层异常信息(网络/超时等)")
class KnowledgeIngestPublishResponse(BaseModel):
success: bool
template_name: str
node_name: str = Field(..., description="入库唯一标识(md 标题 + 时间戳),两次尝试共用")
batch_number: str
attempts: list[KnowledgeIngestAttempt]
def _judge_success(status_code: int, response_text: str) -> bool:
"""判断入库接口响应是否表示成功。
接口未提供正式契约:优先看响应 JSON 中常见的 success/code/status/state
字段;JSON 不可解析或没有这类字段时,退化为按 HTTP 2xx 判定。
"""
if not (200 <= status_code < 300):
return False
try:
data = json.loads(response_text)
except (ValueError, TypeError):
return True
if not isinstance(data, dict):
return True
if isinstance(data.get("success"), bool):
return data["success"]
for key in ("code", "status", "state"):
if key in data:
return str(data[key]).lower() in {"0", "200", "success", "ok", "true"}
return True
def _build_payload(template_name: str, batch_number: str, node_name: str, title_field: str, title: str, content: str) -> dict[str, Any]:
return {
"templateName": template_name,
"batchNumber": batch_number,
"list": [
{
"rec": {"nodeName": node_name},
"data": [
{"fieldName": title_field, "value": title},
{"fieldName": _CONTENT_FIELD, "value": content},
],
}
],
}
def _build_node_name(title: str, timestamp: str) -> str:
"""Build a stable node name as md5(title) + timestamp + .md."""
title_md5 = hashlib.md5(title.encode("utf-8")).hexdigest()
return f"{title_md5}-{timestamp}.md"
def _resolve_ingest_url() -> str:
"""Resolve the external ingest URL from config.yaml, with env fallback."""
config_url = getattr(get_app_config().knowledge_ingest, "url", "").strip()
return os.environ.get(
"KNOWES_INGEST_URL",
config_url or "https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add",
)
@router.post("/publish", response_model=KnowledgeIngestPublishResponse, summary="Publish markdown artifact to external knowledge base")
async def publish_to_knowledge_base(request: KnowledgeIngestPublishRequest) -> KnowledgeIngestPublishResponse:
title = request.title.strip()
content = request.content.strip()
if not title:
raise HTTPException(status_code=422, detail="标题不能为空")
if not content:
raise HTTPException(status_code=422, detail="内容为空,无法入库")
try:
agent_cfg = load_agent_config(request.agent_id)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
template_name = (agent_cfg.template_name or "").strip() if agent_cfg else ""
if not template_name:
raise HTTPException(status_code=400, detail=f"智能体 '{request.agent_id}' 未配置入库模板名(template_name),无法发布")
now = datetime.now()
# nodeName 是入库记录的唯一标识:md 标题 + 时间戳。两次尝试必须共用同一个
# nodeName,避免第一次实际已入库时第二次产生重复记录。
timestamp = now.strftime("%Y%m%d%H%M%S")
node_name = _build_node_name(title, timestamp)
batch_number = f"md{now.strftime('%Y%m%d')}"
ingest_url = _resolve_ingest_url()
logger.info(
"[knowledge-ingest] 开始入库: agent=%s template=%s nodeName=%s batchNumber=%s title=%r content_len=%d url=%s",
request.agent_id,
template_name,
node_name,
batch_number,
title,
len(content),
ingest_url,
)
attempts: list[KnowledgeIngestAttempt] = []
headers = {"Content-Type": "application/json", "User-Agent": "DeerFlow/1.0"}
async with httpx.AsyncClient(timeout=_REQUEST_TIMEOUT_SECONDS) as client:
for title_field in _TITLE_FIELD_CANDIDATES:
payload = _build_payload(template_name, batch_number, node_name, title_field, title, content)
logger.info(
"[knowledge-ingest] 第 %d 次发送 (标题字段=%s): %s",
len(attempts) + 1,
title_field,
json.dumps({**payload, "list": [{**payload["list"][0], "data": [{"fieldName": d["fieldName"], "value": d["value"][:200]} for d in payload["list"][0]["data"]]}]}, ensure_ascii=False),
)
try:
response = await client.post(ingest_url, json=payload, headers=headers)
except httpx.HTTPError as exc:
logger.error("[knowledge-ingest] 第 %d 次发送请求异常 (标题字段=%s): %s", len(attempts) + 1, title_field, exc)
attempts.append(KnowledgeIngestAttempt(title_field=title_field, error=str(exc)))
continue
text = response.text or ""
success = _judge_success(response.status_code, text)
logger.info(
"[knowledge-ingest] 第 %d 次响应 (标题字段=%s): status=%d success=%s body=%s",
len(attempts) + 1,
title_field,
response.status_code,
success,
text[:2000],
)
attempts.append(
KnowledgeIngestAttempt(
title_field=title_field,
status_code=response.status_code,
response_text=text[:2000],
success=success,
)
)
if success:
break
logger.warning("[knowledge-ingest] 第 %d 次入库判定失败 (标题字段=%s),准备换字段重试", len(attempts), title_field)
overall = any(a.success for a in attempts)
if overall:
logger.info("[knowledge-ingest] 入库成功: nodeName=%s 使用标题字段=%s", node_name, next(a.title_field for a in attempts if a.success))
else:
logger.error("[knowledge-ingest] 入库失败(全部 %d 次尝试均未成功): nodeName=%s", len(attempts), node_name)
return KnowledgeIngestPublishResponse(
success=overall,
template_name=template_name,
node_name=node_name,
batch_number=batch_number,
attempts=attempts,
)
@@ -21,7 +21,7 @@ from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_current_user, get_feedback_repo, get_run_event_store, get_run_manager, get_run_store, get_stream_bridge
from app.gateway.services import sse_consumer, start_run
from app.gateway.services import immediate_end_sse, sse_consumer, start_run
from deerflow.runtime import RunRecord, serialize_channel_values
logger = logging.getLogger(__name__)
@@ -96,7 +96,7 @@ def _record_to_response(record: RunRecord) -> RunResponse:
@router.post("/{thread_id}/runs", response_model=RunResponse)
@require_permission("runs", "create", owner_check=True, require_existing=True)
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
async def create_run(thread_id: str, body: RunCreateRequest, request: Request) -> RunResponse:
"""Create a background run (returns immediately)."""
record = await start_run(body, thread_id, request)
@@ -104,7 +104,7 @@ async def create_run(thread_id: str, body: RunCreateRequest, request: Request) -
@router.post("/{thread_id}/runs/stream")
@require_permission("runs", "create", owner_check=True, require_existing=True)
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
async def stream_run(thread_id: str, body: RunCreateRequest, request: Request) -> StreamingResponse:
"""Create a run and stream events via SSE.
@@ -132,7 +132,7 @@ async def stream_run(thread_id: str, body: RunCreateRequest, request: Request) -
@router.post("/{thread_id}/runs/wait", response_model=dict)
@require_permission("runs", "create", owner_check=True, require_existing=True)
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
async def wait_run(thread_id: str, body: RunCreateRequest, request: Request) -> dict:
"""Create a run and block until it completes, returning the final state."""
record = await start_run(body, thread_id, request)
@@ -173,7 +173,11 @@ async def get_run(thread_id: str, run_id: str, request: Request) -> RunResponse:
run_mgr = get_run_manager(request)
record = run_mgr.get(run_id)
if record is None or record.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
# In-memory miss: fall back to the persistent store so a run still
# resolves after a Gateway restart instead of 404ing.
record = await run_mgr.get_persisted(run_id)
if record is None or record.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return _record_to_response(record)
@@ -222,11 +226,27 @@ async def join_run(thread_id: str, run_id: str, request: Request) -> StreamingRe
bridge = get_stream_bridge(request)
run_mgr = get_run_manager(request)
record = run_mgr.get(run_id)
if record is None or record.thread_id != thread_id:
if record is not None and record.thread_id == thread_id:
return StreamingResponse(
sse_consumer(bridge, record, request, run_mgr),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
# Not in memory: a run that still exists in the store but has no live task
# means the Gateway restarted (or the finished run was cleaned up). There
# is nothing to reconnect to, so stream a clean terminal end instead of a
# 404 — the SDK then clears its reconnect key and renders history.
persisted = await run_mgr.get_persisted(run_id)
if persisted is None or persisted.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return StreamingResponse(
sse_consumer(bridge, record, request, run_mgr),
immediate_end_sse(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
@@ -255,7 +275,22 @@ async def stream_existing_run(
run_mgr = get_run_manager(request)
record = run_mgr.get(run_id)
if record is None or record.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
# In-memory miss. If the run survives in the store it's a dead record
# from before a restart (or already finished) — nothing to join or
# cancel, so emit a clean terminal end so the SDK stops reconnecting
# instead of 404ing. A truly unknown run still 404s.
persisted = await run_mgr.get_persisted(run_id)
if persisted is None or persisted.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return StreamingResponse(
immediate_end_sse(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
# Cancel if an action was requested (stop-button / interrupt flow)
if action is not None:
@@ -42,6 +42,20 @@ logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
async def immediate_end_sse():
"""Yield a single terminal ``end`` SSE frame and stop.
Used when a run exists in the persistent store but not in the in-memory
registry — typically after a Gateway restart, or once a finished run's
record has been cleaned up. There is no live stream to join, so instead of
returning 404 (which the SDK's ``joinStream`` surfaces as a hard error and
never clears its reconnect key for) we stream a clean end. The SDK treats
this as success, clears ``lg:stream:{threadId}``, and the conversation
falls back to history rendering — killing the repeating switch-back 404.
"""
yield format_sse("end", None)
def format_sse(event: str, data: Any, *, event_id: str | None = None) -> str:
"""Format a single SSE frame.
@@ -477,6 +491,9 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
store = get_agent_store(request)
await _maybe_sync_legacy_agent_records(store)
user_id = get_effective_user_id()
# Admins see every agent in the admin console listing (?admin=true), so
# they may also start a run against any of them.
is_admin = getattr(getattr(request.state, "user", None), "system_role", None) == "admin"
for agent_id in agent_ids:
try:
normalized = validate_agent_id(agent_id)
@@ -484,8 +501,11 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
raise HTTPException(status_code=422, detail=str(exc)) from exc
if normalized is None:
continue
if await store.get_visible(normalized, user_id) is None:
raise HTTPException(status_code=404, detail=f"Agent {normalized} not found")
if await store.get_visible(normalized, user_id) is not None:
continue
if is_admin and await store.get_any(normalized) is not None:
continue
raise HTTPException(status_code=404, detail=f"Agent {normalized} not found")
# ---------------------------------------------------------------------------
@@ -8,6 +8,7 @@
"workspace-search": { "enabled": true },
"policy-knowledge": { "enabled": false },
"html-page-builder": { "enabled": true },
"html-charts": { "enabled": true }
"html-charts": { "enabled": true },
"html-multipage": { "enabled": true }
}
}
@@ -100,6 +100,9 @@ class AgentConfig(BaseModel):
# Accepts both the legacy ``list[str]`` shape and the new ``{title, prompt}``
# objects (see CommonQuestion).
common_questions: list[CommonQuestion] | None = None
# 入库模板名(外部知识入库接口的 templateName)。配置后该智能体会话中的
# Markdown 产物可一键发布到外部入库接口;None/空 = 不支持入库。
template_name: str | None = None
def load_agent_config(agent_id: str | None) -> AgentConfig | None:
@@ -15,9 +15,10 @@ from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpo
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.knowledge_config import KnowledgeConfig
from deerflow.config.knowledge_ingest_config import KnowledgeIngestConfig
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
from deerflow.config.model_config import ModelConfig
@@ -135,8 +136,9 @@ class AppConfig(BaseModel):
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
@classmethod
def resolve_config_path(cls, config_path: str | None = None) -> Path:
@@ -0,0 +1,12 @@
"""Configuration for the external knowledge-ingest endpoint."""
from pydantic import BaseModel, Field
class KnowledgeIngestConfig(BaseModel):
"""External knowledge-ingest configuration (`config.yaml` -> `knowledge_ingest`)."""
url: str = Field(
default="https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add",
description="External knowledge-ingest endpoint URL.",
)
@@ -13,25 +13,35 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
# with error 1071 ("Specified key was too long").
class KnowledgeNoteRow(Base):
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
__tablename__ = "knowledge_notes"
__table_args__ = (
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
vault_path: Mapped[str | None] = mapped_column(String(1024))
source_type: Mapped[str] = mapped_column(String(64), index=True)
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
source_id: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
confidence: Mapped[float] = mapped_column(Float, default=0.0)
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
@@ -118,10 +128,15 @@ class KnowledgeEntityRow(Base):
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
__tablename__ = "knowledge_entities"
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
__table_args__ = (
Index("ix_knowledge_entities_name", "name", mysql_length=191),
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
# realistically far shorter than 120 chars, so the prefix is lossless.
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), index=True)
name: Mapped[str] = mapped_column(String(256))
entity_type: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -134,9 +149,10 @@ class KnowledgeGraphBlacklistRow(Base):
"""
__tablename__ = "knowledge_graph_blacklist"
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
label: Mapped[str] = mapped_column(String(512), unique=True, index=True)
label: Mapped[str] = mapped_column(String(512))
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -276,6 +276,7 @@ class KnowledgeRepository:
*,
keyword: str | None = None,
tag: str | None = None,
entity: str | None = None,
source_type: str | None = None,
status: str | None = None,
include_archived: bool = False,
@@ -303,6 +304,16 @@ class KnowledgeRepository:
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
)
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
if entity:
# Exact entity match via the note→entity relation graph — precise
# and index-backed, unlike a content LIKE scan on large vaults.
entity_subq = (
select(KnowledgeRelationRow.from_note_id)
.join(KnowledgeEntityRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
.where(func.lower(KnowledgeEntityRow.name) == entity.lower())
.where(KnowledgeRelationRow.from_note_id.is_not(None))
)
conditions.append(KnowledgeNoteRow.id.in_(entity_subq))
for cond in conditions:
stmt = stmt.where(cond)
@@ -523,6 +523,7 @@ class KnowledgeService:
*,
keyword: str | None = None,
tag: str | None = None,
entity: str | None = None,
source_type: str | None = None,
status: str | None = None,
limit: int = 20,
@@ -531,6 +532,7 @@ class KnowledgeService:
items, total = await self.repo.list_notes(
keyword=keyword,
tag=tag,
entity=entity,
source_type=normalize_source_type(source_type) if source_type else None,
status=status,
limit=limit,
@@ -32,6 +32,15 @@ class MemoryAgentStore(AgentStore):
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
"""Single-record visibility (``get_visible``) — the union of every
listing surface, so a list-visible agent always opens. Mirrors
``AgentRepository._visible_filter``: baseline visibility OR
private-square (password is the discovery gate, reads are not
re-gated) OR favorited (the "我的" listing keeps favorites even after
the owner unpublishes)."""
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
rows.sort(key=lambda agent: agent.get("created_at") or "")
@@ -39,7 +48,7 @@ class MemoryAgentStore(AgentStore):
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
agent = self._agents.get(agent_id)
if agent is None or not self._is_visible(agent, user_id):
if agent is None or not self._is_fetchable(agent, user_id):
return None
return _copy(agent)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -32,7 +32,7 @@ class AgentRow(Base):
# ``is_private``. Empty string = "not assigned" → the listing layer treats
# it as the admin-configured default square. Each agent belongs to at most
# one square (single column), so it can never be published to two squares.
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", server_default="", index=True)
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
# featured rank (lower sorts first). Only ``published=True`` agents are
# eligible; the router enforces that invariant on writes. BIGINT so the
@@ -86,7 +86,11 @@ class AgentSkillRow(Base):
"""
__tablename__ = "agent_skills"
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -40,12 +40,27 @@ class AgentRepository(AgentStore):
def _visible_filter(user_id: str):
"""Visibility for single-record fetches (``get_visible``).
Lenient: a caller who already knows the agent id can fetch it even
when it's flagged ``is_private``. The private-square password is the
*discovery* gate (listings); individual reads are not re-gated, so a
bookmarked / linked private agent stays openable.
Lenient — must cover the union of every listing surface, so any agent
a user can see in a list always opens (detail + chat) instead of
404ing:
- built-in / owned / published — the baseline gallery visibility;
- ``is_private`` — the private-square password is the *discovery*
gate (listings); individual reads are not re-gated, so a private
agent opened from the square (or a bookmark/link) stays openable.
Note moving an agent into the private square forces
``published=false``, so this clause cannot lean on ``published``;
- favorited — the "我的" listing keeps favorited agents even after
the owner unpublishes them, so reads keep honoring the favorite.
"""
return or_(AgentRow.user_id.is_(None), AgentRow.user_id == user_id, AgentRow.published.is_(True))
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
return or_(
AgentRow.user_id.is_(None),
AgentRow.user_id == user_id,
AgentRow.published.is_(True),
AgentRow.is_private.is_(True),
AgentRow.id.in_(fav_subq),
)
@staticmethod
def _listing_filter(user_id: str):
@@ -194,6 +194,79 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
``_run_alembic_upgrade`` call) — so on an existing database a new
``mapped_column`` on an ORM model would otherwise error at first query
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Indexes covering a newly
added column are created as well. Works on sqlite/mysql/postgres — the
column DDL is compiled through the active dialect, so portable types
(``PortableJSON``/``PortableLongText``/``BeijingDateTime``) resolve to the
right backend type. Everything is per-column best-effort: a failure is
logged and startup continues.
"""
from sqlalchemy import inspect as sa_inspect
from sqlalchemy.schema import CreateColumn
from deerflow.persistence.base import Base
try:
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
except ImportError:
pass
inspector = sa_inspect(sync_conn)
existing_tables = set(inspector.get_table_names())
preparer = sync_conn.dialect.identifier_preparer
for table in Base.metadata.sorted_tables:
if table.name not in existing_tables:
continue
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
missing = [c for c in table.columns if c.name not in existing_cols]
if not missing:
continue
added: set[str] = set()
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
table.name,
col.name,
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
added.add(col.name)
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
if not added:
continue
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
@@ -344,6 +417,15 @@ async def init_engine(
else:
raise
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
except Exception:
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, Text, UniqueConstraint
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -14,7 +14,12 @@ from deerflow.persistence.types import BeijingDateTime
class FeedbackRow(Base):
__tablename__ = "feedback"
__table_args__ = (UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),)
# Prefix-unique instead of UniqueConstraint: a plain unique over three
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
# mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -36,7 +36,7 @@ class SkillRow(Base):
# Publish square (发布广场) this skill is filed under once published. Pure
# organizational category, no access control. Empty string = unassigned →
# the listing layer resolves it to the default square. One square per skill.
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", server_default="", index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -112,6 +112,52 @@ class RunManager:
"""Return a run record by ID, or ``None``."""
return self._runs.get(run_id)
def _record_from_store(self, data: dict) -> RunRecord:
"""Rebuild a :class:`RunRecord` from a persisted store row.
The reconstructed record has no live ``task``/``abort_event`` wired to a
running coroutine — it represents *history*, not an in-flight run.
"""
try:
status = RunStatus(data.get("status") or RunStatus.success.value)
except ValueError:
status = RunStatus.success
return RunRecord(
run_id=data["run_id"],
thread_id=data["thread_id"],
assistant_id=data.get("assistant_id"),
status=status,
# Disconnect mode is irrelevant for a store-only record (no task to
# cancel); ``continue_`` keeps sse_consumer's finally block a no-op.
on_disconnect=DisconnectMode.continue_,
multitask_strategy=data.get("multitask_strategy", "reject"),
metadata=data.get("metadata") or {},
kwargs=data.get("kwargs") or {},
created_at=data.get("created_at") or "",
updated_at=data.get("updated_at") or "",
error=data.get("error"),
)
async def get_persisted(self, run_id: str) -> RunRecord | None:
"""Load a run from the backing store (process-restart fallback).
The in-memory registry is wiped on every Gateway restart, but the
``RunStore`` row survives. When :meth:`get` misses, callers fall back
here so a stale ``joinStream``/``GET run`` can report the run's final
status instead of 404ing. Returns ``None`` when there is no store, no
matching row, or the lookup fails (e.g. no user context).
"""
if self._store is None:
return None
try:
data = await self._store.get(run_id)
except Exception:
logger.warning("Failed to load run %s from store", run_id, exc_info=True)
return None
if not data:
return None
return self._record_from_store(data)
async def list_by_thread(self, thread_id: str) -> list[RunRecord]:
"""Return all runs for a given thread, newest first."""
async with self._lock:
@@ -39,10 +39,15 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
"""Return the explicit skill name set declared in an agent's config.yaml.
Semantics:
- ``None`` returned → no agent_id, or config missing — caller should
fall back to "all skills visible".
- ``set()`` (empty) → agent has ``skills: []`` declared — visibility
is intentionally empty.
- ``None`` returned → no agent_id, or the agent's on-disk config is
missing entirely — caller should fall back to "all skills visible"
(default lead agent behavior).
- ``set()`` (empty) → custom agent with ``skills: []`` declared **or**
with no ``skills`` key at all — visibility is intentionally empty.
This mirrors the prompt layer (``agents_config.AgentConfig.skills``:
``None``/``[]`` both mean "no skills" for custom agents), so an agent
whose config omits ``skills`` can no longer enumerate every skill
through this tool.
- Non-empty set → only these names are visible to this agent.
"""
if not agent_id:
@@ -51,9 +56,9 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
from deerflow.config.agents_config import load_agent_config
cfg = load_agent_config(agent_id)
if cfg is None or cfg.skills is None:
if cfg is None:
return None
return set(cfg.skills)
return set(cfg.skills or [])
except Exception:
logger.debug("Failed to load agent config for skill_list filter", exc_info=True)
return None
@@ -119,16 +124,21 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
return None
@tool("skill_view")
def skill_view_tool(name: str) -> str:
"""Read the full SKILL.md content of a skill by name.
def skill_view_impl(name: str, agent_id: str | None) -> str:
"""Read a skill's SKILL.md, honoring the agent's config.yaml whitelist.
Args:
name: The skill name (hyphen-case, e.g. 'my-skill').
Custom agents may only read skills on their whitelist — same filter as
``skill_list``, so a restricted agent cannot bypass its allowlist by
guessing skill names. Replies "not found" rather than "forbidden" to
avoid leaking which skills exist.
"""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import bump_view
allowed = _load_agent_skill_whitelist(agent_id)
if allowed is not None and name not in allowed:
return f"Skill '{name}' not found."
storage = get_or_new_skill_storage()
try:
if storage.custom_skill_exists(name):
@@ -156,6 +166,16 @@ def skill_view_tool(name: str) -> str:
return content
@tool("skill_view")
def skill_view_tool(name: str, runtime: ToolRuntime) -> str:
"""Read the full SKILL.md content of a skill by name.
Args:
name: The skill name (hyphen-case, e.g. 'my-skill').
"""
return skill_view_impl(name, _resolve_active_agent_id(runtime))
@tool("skill_list")
def skill_list_tool(runtime: ToolRuntime) -> str:
"""List skills available to this agent, with metadata and usage stats.
@@ -175,11 +195,15 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
state, use_count, view_count, patch_count, last_activity_at, pinned,
source.
"""
return skill_list_impl(_resolve_active_agent_id(runtime))
def skill_list_impl(agent_id: str | None) -> str:
"""List skills visible to ``agent_id`` (None = default lead agent)."""
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.types import SkillCategory
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
agent_id = _resolve_active_agent_id(runtime)
allowed = _load_agent_skill_whitelist(agent_id)
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
@@ -218,14 +218,16 @@ async def _skill_manage_impl(
"""
# Read-only aliases — delegate to the dedicated tools so a model that
# over-uses skill_manage doesn't trip on Unsupported action errors.
# Pass the active agent_id through so a custom agent's skill whitelist
# is enforced on these aliases just like on skill_list/skill_view.
if action == "list":
from deerflow.tools.builtins.skill_tools import skill_list_tool
from deerflow.tools.builtins.skill_tools import _resolve_active_agent_id, skill_list_impl
return await _to_thread(skill_list_tool.invoke, {})
return await _to_thread(skill_list_impl, _resolve_active_agent_id(runtime))
if action == "view":
from deerflow.tools.builtins.skill_tools import skill_view_tool
from deerflow.tools.builtins.skill_tools import _resolve_active_agent_id, skill_view_impl
return await _to_thread(skill_view_tool.invoke, {"name": name})
return await _to_thread(skill_view_impl, name, _resolve_active_agent_id(runtime))
name = SkillStorage.validate_skill_name(name)
lock = _get_lock(name)
@@ -0,0 +1,71 @@
"""Audit ORM index/unique-key byte sizes against MySQL InnoDB prefix limits.
MySQL with the legacy 767-byte InnoDB index-prefix limit (5.6, or 5.7 with
COMPACT row format) rejects any index whose key exceeds 767 bytes — and
utf8mb4 costs 4 bytes per char, so an indexed VARCHAR key may total at most
191 chars. Modern servers (DYNAMIC row format) allow 3072 bytes = 768 chars.
Run: PYTHONPATH=. .venv/Scripts/python scripts/audit_index_key_lengths.py
"""
from __future__ import annotations
import sys
from sqlalchemy import String, UniqueConstraint
import deerflow.persistence.models # noqa: F401 (registers all ORM models)
from deerflow.persistence.base import Base
LIMIT_OLD = 767
LIMIT_NEW = 3072
BYTES_PER_CHAR = 4 # utf8mb4
def _col_key_bytes(col) -> int | None:
t = col.type
# unwrap TypeDecorator
impl = getattr(t, "impl", None)
if impl is not None:
t = impl
if isinstance(t, String):
return (t.length or 255) * BYTES_PER_CHAR
return 8 # ints/datetimes/floats — small fixed size, close enough
def _key_bytes(col, mysql_length) -> int:
full = _col_key_bytes(col) or 0
if mysql_length is None:
return full
if isinstance(mysql_length, dict):
prefix = mysql_length.get(col.name)
else:
prefix = mysql_length
if prefix is None:
return full
return min(full, int(prefix) * BYTES_PER_CHAR)
def main() -> int:
bad = 0
for table in Base.metadata.sorted_tables:
keys: list[tuple[str, str, list, object]] = []
for idx in table.indexes:
keys.append(("INDEX", idx.name, list(idx.columns), idx.dialect_options["mysql"].get("length")))
for cons in table.constraints:
if isinstance(cons, UniqueConstraint):
keys.append(("UNIQUE", cons.name or "?", list(cons.columns), None))
for kind, name, cols, mysql_length in keys:
total = sum(_key_bytes(c, mysql_length) for c in cols)
if total > LIMIT_OLD:
bad += 1
tag = "EXCEEDS 3072 (fails on ALL MySQL)" if total > LIMIT_NEW else "exceeds 767 (fails on legacy MySQL)"
detail = ", ".join(f"{c.name}({getattr(getattr(c.type, 'impl', c.type), 'length', '?')})" for c in cols)
print(f"{table.name}: {kind} {name} = {total} bytes [{detail}] -- {tag}")
if not bad:
print("OK: every index key fits within 767 bytes (utf8mb4 worst case)")
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -108,6 +108,8 @@ vision:
model_name: deepseek-ai/DeepSeek-OCR
# prompt: 请详细描述这张图片的内容。
# max_tokens: 2048
knowledge_ingest:
url: "https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add"
# 上传文件识别配置。
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。