Merge branch 'ji' into master
This commit is contained in:
commit
e74ef4b7dc
81 files changed
+3712
-1625
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -100,7 +100,7 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
After admin creation, migrate orphan threads from the LangGraph
|
||||
store (metadata.user_id unset) to the admin account. This is the
|
||||
"no-auth → with-auth" upgrade path: users who ran DeerFlow without
|
||||
"no-auth ? with-auth" upgrade path: users who ran DeerFlow without
|
||||
authentication have existing LangGraph thread data that needs an
|
||||
owner assigned.
|
||||
First boot (no admin exists):
|
||||
@@ -108,7 +108,7 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
- The operator must visit ``/setup`` to create the first admin.
|
||||
|
||||
Subsequent boots (admin already exists):
|
||||
- Runs the one-time "no-auth → with-auth" orphan thread migration for
|
||||
- Runs the one-time "no-auth ? with-auth" orphan thread migration for
|
||||
existing LangGraph thread metadata that has no owner_id.
|
||||
|
||||
No SQL persistence migration is needed: the four user_id columns
|
||||
@@ -138,12 +138,12 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
if admin_count == 0:
|
||||
logger.info("=" * 60)
|
||||
logger.info(" First boot detected — no admin account exists.")
|
||||
logger.info(" First boot detected ? no admin account exists.")
|
||||
logger.info(" Visit /setup to complete admin account creation.")
|
||||
logger.info("=" * 60)
|
||||
return
|
||||
|
||||
# Admin already exists — run orphan thread migration for any
|
||||
# Admin already exists ? run orphan thread migration for any
|
||||
# LangGraph thread metadata that pre-dates the auth module.
|
||||
async with sf() as session:
|
||||
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
|
||||
@@ -154,8 +154,8 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
admin_id = str(row.id)
|
||||
|
||||
# LangGraph store orphan migration — non-fatal.
|
||||
# This covers the "no-auth → with-auth" upgrade path for users
|
||||
# LangGraph store orphan migration ? non-fatal.
|
||||
# This covers the "no-auth ? with-auth" upgrade path for users
|
||||
# whose existing LangGraph thread metadata has no user_id set.
|
||||
store = getattr(app.state, "store", None)
|
||||
if store is not None:
|
||||
@@ -217,7 +217,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
# No admin yet — leave ownership null; first admin is responsible for
|
||||
# No admin yet ? leave ownership null; first admin is responsible for
|
||||
# claiming via the UI once they exist.
|
||||
admin_id: str | None = None
|
||||
else:
|
||||
@@ -278,7 +278,7 @@ async def _run_db_migrations() -> None:
|
||||
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
logger.info("Persistence backend=memory — skipping Alembic migrations")
|
||||
logger.info("Persistence backend=memory ? skipping Alembic migrations")
|
||||
return
|
||||
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
|
||||
@@ -327,28 +327,28 @@ async def _run_db_migrations() -> None:
|
||||
raise result
|
||||
logger.info("Database migrations applied (head)")
|
||||
except Exception:
|
||||
logger.exception("Database migration failed — schema may be out of date")
|
||||
logger.exception("Database migration failed ? schema may be out of date")
|
||||
|
||||
|
||||
# Skill curator 调度参数。
|
||||
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
|
||||
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
|
||||
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
|
||||
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
|
||||
# Skill curator ?????
|
||||
# - ????:?????????"??????";??????? should_run_now() ???
|
||||
# - ????:?????????????,???????
|
||||
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 ??
|
||||
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 ??
|
||||
|
||||
|
||||
def _acquire_singleton_lock(lock_path):
|
||||
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
|
||||
"""???????????,??? worker ?? leader ???
|
||||
|
||||
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
|
||||
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
|
||||
进程异常退出时 OS 会自动释放该锁。
|
||||
?? -> ??????????(?????????,??????/????);
|
||||
????(???? worker ??)????? -> ?? None?
|
||||
??????? OS ????????
|
||||
"""
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(lock_path, "a+")
|
||||
except OSError:
|
||||
logger.exception("Curator scheduler: 打开锁文件失败")
|
||||
logger.exception("Curator scheduler: ???????")
|
||||
return None
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
@@ -367,7 +367,7 @@ def _acquire_singleton_lock(lock_path):
|
||||
|
||||
|
||||
def _release_singleton_lock(handle) -> None:
|
||||
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
|
||||
"""????? _acquire_singleton_lock ?????????"""
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
@@ -388,24 +388,24 @@ def _release_singleton_lock(handle) -> None:
|
||||
|
||||
|
||||
async def _curator_scheduler_loop() -> None:
|
||||
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
|
||||
"""??????:?? _CURATOR_CHECK_INTERVAL_SECONDS ???????????
|
||||
|
||||
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
|
||||
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
|
||||
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
|
||||
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
|
||||
? worker ???????? leader ?? ?? ???????? worker ????
|
||||
????,?? worker ????,?? curator ?????????? LLM?
|
||||
? cancel ? CancelledError ??? ``except Exception``(?? BaseException
|
||||
??,?????),?????????? finally ?????
|
||||
"""
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.skills.curator import maybe_run_curator
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
|
||||
logger.info("Curator scheduler: ??? worker ??????,? worker ??")
|
||||
return
|
||||
|
||||
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
|
||||
logger.info("Curator scheduler: ?????,? worker ?? curator ??")
|
||||
try:
|
||||
# 启动后延迟一会儿再首次检查,避开启动高峰
|
||||
# ?????????????,??????
|
||||
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
@@ -477,7 +477,7 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
|
||||
)
|
||||
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
|
||||
# *previous* day's snapshot once, so records that landed near midnight are fully
|
||||
# captured. Default 1 → "每天凌晨 1 点重新统计前一天".
|
||||
# captured. Default 1 ? "???? 1 ????????".
|
||||
_LEADERBOARD_FINALIZE_HOUR = min(
|
||||
23,
|
||||
_env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
|
||||
@@ -648,7 +648,7 @@ async def _claim_leaderboard_snapshot(
|
||||
if age < running_ttl_seconds:
|
||||
return False
|
||||
row.status = "running"
|
||||
row.message = "排行榜后台统计任务已认领。"
|
||||
row.message = "?????????????"
|
||||
row.error = None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await _time_leaderboard_scheduler_db_operation(
|
||||
@@ -701,7 +701,7 @@ async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_has
|
||||
[yesterday],
|
||||
settings_hash,
|
||||
force=True,
|
||||
message="每日凌晨重新统计前一天完整数据,确保全部记录纳入统计。",
|
||||
message="???????????????????????????",
|
||||
)
|
||||
if queued:
|
||||
logger.info(
|
||||
@@ -734,10 +734,10 @@ async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
|
||||
session_factory,
|
||||
stat_dates,
|
||||
settings_hash,
|
||||
message="排行榜后台统计任务定期预热/补算队列。",
|
||||
message="?????????????/?????",
|
||||
)
|
||||
|
||||
# 每天凌晨(默认 1 点)后强制重算"前一天"的完整快照一次,覆盖跨午夜才落库的记录。
|
||||
# ??????? 1 ???????"???"????????????????????
|
||||
try:
|
||||
await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
|
||||
except Exception:
|
||||
@@ -816,31 +816,31 @@ async def _leaderboard_snapshot_scheduler_loop() -> None:
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
|
||||
"""?? uvicorn / gunicorn ? worker ???? ERROR ???
|
||||
|
||||
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
|
||||
维护 run_id → SSE 订阅者的内存 map。多 worker 下,用户在 worker A 发起
|
||||
``/runs/stream``、稍后在 worker B 调 join/resume,会因 in-process map 互相
|
||||
看不见而 404 / 卡死。
|
||||
? agent / canvas / AI ??????? Gateway ????? ``StreamBridge``
|
||||
?? run_id ? SSE ?????? map?? worker ????? worker A ??
|
||||
``/runs/stream``???? worker B ? join/resume??? in-process map ??
|
||||
???? 404 / ???
|
||||
|
||||
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
|
||||
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
|
||||
????? startup ? ERROR ????**???? fail-fast** ?? ??
|
||||
???????????????? sticky session??????????
|
||||
"""
|
||||
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
|
||||
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
|
||||
# / uvicorn 都认)。
|
||||
# uvicorn / gunicorn ??????? worker ?????????????
|
||||
# ?????? sibling ??????????? WEB_CONCURRENCY?gunicorn
|
||||
# / uvicorn ????
|
||||
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
|
||||
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
|
||||
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
|
||||
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
|
||||
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
|
||||
"? ??? WEB_CONCURRENCY=%s?? worker ????AI ?? / ? agent "
|
||||
"??????????????? worker ????????? / "
|
||||
"SSE ?? / resume 400??????????? WEB_CONCURRENCY=1 "
|
||||
"????????? sticky session??? AIWritingPage.tsx ????",
|
||||
web_concurrency,
|
||||
)
|
||||
return
|
||||
|
||||
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
|
||||
# ???? /proc ??? uvicorn ?????? Linux ??????
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
@@ -858,13 +858,13 @@ def _warn_if_multi_worker() -> None:
|
||||
peers += 1
|
||||
if peers > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
|
||||
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
|
||||
"启用 sticky session。",
|
||||
"? ??? %d ??? uvicorn ???? worker ????? AI ?? "
|
||||
"????? / SSE ??????????? worker ??????? "
|
||||
"?? sticky session?",
|
||||
peers,
|
||||
)
|
||||
except Exception:
|
||||
# 检测本身失败不算事 —— 这只是个提醒
|
||||
# ????????? ?? ??????
|
||||
pass
|
||||
|
||||
|
||||
@@ -872,8 +872,8 @@ def _warn_if_multi_worker() -> None:
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
|
||||
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
|
||||
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
|
||||
# ??????? worker ???????? ERROR ?????
|
||||
_warn_if_multi_worker()
|
||||
|
||||
# Load config and check necessary environment variables at startup
|
||||
@@ -895,7 +895,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Run Alembic migrations before anything else touches the schema.
|
||||
await _run_db_migrations()
|
||||
|
||||
# 预热 recall_budget cache(从 DB 批量加载,使 get_effective_memory_config 同步可读)
|
||||
# ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
|
||||
try:
|
||||
from deerflow.config.memory_config import load_recall_budget_cache
|
||||
|
||||
@@ -911,18 +911,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Must run AFTER langgraph_runtime so app.state.store is available for thread migration
|
||||
await _ensure_admin_user(app)
|
||||
|
||||
# Migrate "ownerless" on-disk skills into the new SkillStore table —
|
||||
# Migrate "ownerless" on-disk skills into the new SkillStore table ?
|
||||
# makes the multitenancy switch transparent for existing deployments.
|
||||
try:
|
||||
await _sync_legacy_skills(app)
|
||||
except Exception:
|
||||
logger.exception("Skill ownership migration failed (non-fatal)")
|
||||
|
||||
# 圆桌功能型 agent(intent / recommender / coordinator)缺失自动补建。
|
||||
# 必须在 _sync_legacy_agents **之前**——这样刚补建的目录能在同一次启动被
|
||||
# 扫描并 upsert 进 agents 表,新机器一启动 /api/agents 就有这三个,
|
||||
# 无需用户先打开圆桌页触发懒加载。.deer-flow/ 被 gitignore,新机器 clone
|
||||
# 后这些目录不存在,seeder 从源码资源 _roundtable_seed_assets/ 复制验证版。
|
||||
# ????? agent(intent / recommender / coordinator)???????
|
||||
# ??? _sync_legacy_agents **??**??????????????????
|
||||
# ??? upsert ? agents ?,?????? /api/agents ?????,
|
||||
# ????????????????.deer-flow/ ? gitignore,??? clone
|
||||
# ????????,seeder ????? _roundtable_seed_assets/ ??????
|
||||
try:
|
||||
from app.gateway.routers._roundtable_seed import (
|
||||
ensure_roundtable_functional_agents,
|
||||
@@ -934,9 +934,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"Roundtable functional agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# AI 写作功能型 agent(researcher / outliner / writer / editor)缺失自动补建。
|
||||
# 同圆桌:必须在 _sync_legacy_agents 之前,补建目录才能在同一次启动被
|
||||
# upsert 进 agents 表,admin 启动后即可在智能体管理页编辑四个写作 agent。
|
||||
# AI ????? agent(researcher / outliner / writer / editor)???????
|
||||
# ???:??? _sync_legacy_agents ??,?????????????
|
||||
# upsert ? agents ?,admin ?????????????????? agent?
|
||||
try:
|
||||
from app.gateway.routers._ai_writing_seed import (
|
||||
ensure_ai_writing_functional_agents,
|
||||
@@ -948,7 +948,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"AI-writing functional agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 业务链条管理:幂等补建固定业务行(3Q/6BF/7BF/8BF),agent/chain 初始为空。
|
||||
# ?????????????????3Q/6BF/7BF/8BF??agent/chain ?????
|
||||
try:
|
||||
business_mapping_store = getattr(app.state, "business_mapping_store", None)
|
||||
if business_mapping_store is not None:
|
||||
@@ -958,10 +958,10 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Business mapping seeding failed at startup (non-fatal)")
|
||||
|
||||
# 一次性把 .deer-flow/agents/{id}/ 目录里历史遗留的 agent 导入 agents 表。
|
||||
# 路由层原本每次请求都跑一遍这个 sync(扫文件系统 + N 个 agent 各一次 DB
|
||||
# SELECT/UPDATE),把它挪到启动期 + 路由层只保留 TTL 兜底,显著降低
|
||||
# /api/agents 列表延迟。
|
||||
# ???? .deer-flow/agents/{id}/ ???????? agent ?? agents ??
|
||||
# ??????????????? sync(????? + N ? agent ??? DB
|
||||
# SELECT/UPDATE),??????? + ?????? TTL ??,????
|
||||
# /api/agents ?????
|
||||
try:
|
||||
from app.gateway.routers.agents import (
|
||||
_mark_legacy_sync_fresh,
|
||||
@@ -974,7 +974,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
await _sync_legacy_agents(agent_store)
|
||||
_mark_legacy_sync_fresh()
|
||||
# ``services.py`` keeps an independent TTL gate (used by
|
||||
# ``start_run`` → ``_enforce_agent_access``). Without this
|
||||
# ``start_run`` ? ``_enforce_agent_access``). Without this
|
||||
# ack, the first ``/runs/stream`` after startup would
|
||||
# re-run the heavy sync even though we just did it.
|
||||
_services_mark_sync_fresh()
|
||||
@@ -982,17 +982,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Legacy agent sync failed at startup (non-fatal)")
|
||||
|
||||
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
|
||||
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
|
||||
# ?? skill curator ???:??????,?????????????
|
||||
# (per-user interval_hours ??????????????)?
|
||||
try:
|
||||
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
|
||||
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
|
||||
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
|
||||
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
|
||||
# ?? AI ????????:? config.yaml ? ai_writing.cleanup ???,
|
||||
# ?????? 3 ????,? retention_days(?? 7 ?)?????,??
|
||||
# ?? MySQL ??? + SQLite checkpointer ?? thread ???? worker
|
||||
# ?????? leader,?????? worker ??????
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
@@ -1031,7 +1031,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
yield
|
||||
|
||||
# 停止 curator 调度器
|
||||
# ?? curator ???
|
||||
curator_task = getattr(app.state, "curator_scheduler_task", None)
|
||||
if curator_task is not None:
|
||||
curator_task.cancel()
|
||||
@@ -1042,7 +1042,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
# 停止 AI 写作清理调度器
|
||||
# ?? AI ???????
|
||||
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
|
||||
if cleanup_task is not None:
|
||||
cleanup_task.cancel()
|
||||
@@ -1063,9 +1063,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler shutdown error")
|
||||
|
||||
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
|
||||
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
|
||||
# ``langgraph_runtime()`` 的 contextmanager 统一关闭,无需在这里特殊处理。
|
||||
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
|
||||
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
|
||||
# ``langgraph_runtime()`` ? contextmanager ???????????????
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
@@ -1197,7 +1197,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# CORS
|
||||
if is_auth_disabled():
|
||||
logger.warning(
|
||||
"DEER_FLOW_AUTH_DISABLED is set — API authentication is bypassed. "
|
||||
"DEER_FLOW_AUTH_DISABLED is set ? API authentication is bypassed. "
|
||||
"Do not use this in production."
|
||||
)
|
||||
|
||||
@@ -1222,7 +1222,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
if _cors_allow_any_origin():
|
||||
logger.warning(
|
||||
"Permissive CORS enabled — any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
|
||||
"Permissive CORS enabled ? any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
|
||||
"DEER_FLOW_AUTH_DISABLED). Do not expose publicly."
|
||||
)
|
||||
app.add_middleware(
|
||||
@@ -1238,7 +1238,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
elif cors_origins:
|
||||
if "*" in cors_origins:
|
||||
logger.error(
|
||||
"GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True — invalid. "
|
||||
"GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True ? invalid. "
|
||||
"Set GATEWAY_CORS_ALLOW_ALL=1 instead, or list explicit origins."
|
||||
)
|
||||
cors_origins = [o for o in cors_origins if o != "*"]
|
||||
@@ -1319,10 +1319,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Roundtable-planning business chains at /api/roundtable-chains
|
||||
app.include_router(roundtable_chains.router)
|
||||
|
||||
# Global business → agent / chain mapping at /api/business-mapping
|
||||
# Global business ? agent / chain mapping at /api/business-mapping
|
||||
app.include_router(business_mapping.router)
|
||||
|
||||
# Light-app registry (轻应用中心) at /api/light-apps
|
||||
# Light-app registry (?????) at /api/light-apps
|
||||
app.include_router(light_apps.router)
|
||||
|
||||
# Standalone parallel multi-agent panel: fan-out streaming + saved tasks
|
||||
@@ -1400,7 +1400,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Public, unauthenticated read-only share snapshots (/api/public/shares/*).
|
||||
app.include_router(thread_shares.public_router)
|
||||
|
||||
# 圆桌会商草稿分享(import 复制副本 / view 只读结果报告页)
|
||||
# ?????????import ???? / view ????????
|
||||
app.include_router(roundtable_draft_shares.router)
|
||||
# Public, unauthenticated read-only roundtable result (/api/public/roundtable-shares/*).
|
||||
app.include_router(roundtable_draft_shares.public_router)
|
||||
@@ -1408,17 +1408,17 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Stateless Runs API (stream/wait without a pre-existing thread)
|
||||
app.include_router(runs.router)
|
||||
|
||||
# --- 圆桌规划三步流(前端 /page/workspace/roundtable/planning)---
|
||||
# 实现见 app/gateway/routers/{multi_agent,intent,recommend}.py;
|
||||
# 文档见 frontend-web/docs/multi-agent-backend-dev.md
|
||||
app.include_router(multi_agent.router) # Step 2:init(SSE) + run/stream(leader|special)
|
||||
app.include_router(intent.router) # Step 1:/api/intent/init + /stream
|
||||
app.include_router(recommend.router) # Step 1→2:/api/recommend/stream(无 init)
|
||||
# Step 3 产物列表/下载复用下方 artifacts.router(/api/threads/{id}/artifacts)
|
||||
# --- ?????????? /page/workspace/roundtable/planning?---
|
||||
# ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
|
||||
# ??? frontend-web/docs/multi-agent-backend-dev.md
|
||||
app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
|
||||
app.include_router(intent.router) # Step 1?/api/intent/init + /stream
|
||||
app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
|
||||
# Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
|
||||
|
||||
# 探活端点:无需鉴权(见 auth_middleware._PUBLIC_PATH_PREFIXES)。
|
||||
# 同时暴露裸 ``/health`` 与带 API 前缀的 ``/api/health``,方便监控按
|
||||
# 任意 base URL 直接探活,不会被认证中间件拦成 401。
|
||||
# ?????????(? auth_middleware._PUBLIC_PATH_PREFIXES)?
|
||||
# ????? ``/health`` ?? API ??? ``/api/health``,?????
|
||||
# ?? base URL ????,?????????? 401?
|
||||
@app.get("/health", tags=["health"])
|
||||
@app.get("/api/health", tags=["health"])
|
||||
async def health_check() -> dict:
|
||||
|
||||
@@ -31,21 +31,43 @@ from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
|
||||
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.gateway.auth.models import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P = ParamSpec("P")
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
|
||||
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
|
||||
|
||||
A checkpoint with no matching ``threads_meta`` row is the signature of a
|
||||
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
|
||||
survives in the checkpointer while the business row is gone. Normal thread
|
||||
deletion removes the checkpoint too, so this can never resurrect a
|
||||
deliberately-deleted thread.
|
||||
"""
|
||||
try:
|
||||
from app.gateway.deps import get_checkpointer
|
||||
|
||||
checkpointer = get_checkpointer(request)
|
||||
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
return tup is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Permission constants
|
||||
class Permissions:
|
||||
"""Permission constants for resource:action format."""
|
||||
@@ -205,6 +227,7 @@ def require_permission(
|
||||
action: str,
|
||||
owner_check: bool = False,
|
||||
require_existing: bool = False,
|
||||
claim_orphan: bool = False,
|
||||
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
||||
"""Decorator that checks permission for resource:action.
|
||||
|
||||
@@ -221,6 +244,15 @@ def require_permission(
|
||||
**destructive / mutating** routes (DELETE, PATCH,
|
||||
state-update) so a deleted thread can't be re-targeted
|
||||
by another user via the missing-row code path.
|
||||
claim_orphan: Only meaningful with ``owner_check=True`` and
|
||||
``require_existing=True``. When the ``threads_meta``
|
||||
row is missing **and** a LangGraph checkpoint exists
|
||||
for the thread (a DB-switch orphan), adopt it for the
|
||||
current user instead of 404ing. Use on **run-create**
|
||||
routes so a conversation whose business row was lost
|
||||
(sqlite↔mysql swap) stays answerable. Safe: normal
|
||||
deletion removes the checkpoint, so this never revives
|
||||
a deliberately-deleted thread.
|
||||
|
||||
Usage:
|
||||
# Read-style: legacy untracked threads are allowed
|
||||
@@ -289,16 +321,44 @@ def require_permission(
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
thread_store = get_thread_store(request)
|
||||
user_id = str(auth.user.id)
|
||||
allowed = await thread_store.check_access(
|
||||
thread_id,
|
||||
str(auth.user.id),
|
||||
user_id,
|
||||
require_existing=require_existing,
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not found",
|
||||
)
|
||||
# Tell apart three denial causes so the client gets an
|
||||
# accurate message and so DB-switch orphans can be adopted
|
||||
# rather than hard-404'd:
|
||||
# 1. row exists but owned by another user -> not accessible
|
||||
# 2. no row, no checkpoint -> truly missing
|
||||
# 3. no row + checkpoint + claim_orphan -> orphan, claim
|
||||
existing = await thread_store.get(thread_id, user_id=None)
|
||||
if existing is not None:
|
||||
# Distinct detail string lets the frontend show "无权访问"
|
||||
# instead of "不存在" for a cross-user / stale-session hit.
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not accessible",
|
||||
)
|
||||
claimed = False
|
||||
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
|
||||
try:
|
||||
await thread_store.create(thread_id, user_id=user_id)
|
||||
logger.info(
|
||||
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
|
||||
thread_id,
|
||||
user_id,
|
||||
)
|
||||
claimed = True
|
||||
except Exception:
|
||||
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
|
||||
if not claimed:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not found",
|
||||
)
|
||||
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -59,6 +59,11 @@ class AgentResponse(BaseModel):
|
||||
description="Recommended questions shown on the agent chat home / sidebar (常用问题). Each item is {title, prompt}; legacy string lists are coerced.",
|
||||
)
|
||||
|
||||
template_name: str | None = Field(
|
||||
default=None,
|
||||
description="入库模板名 (external knowledge-ingest templateName). When set, markdown artifacts produced in this agent's chats can be published to the external knowledge-ingest API.",
|
||||
)
|
||||
|
||||
tags: list[dict] = Field(default_factory=list, description="Tags assigned to this agent")
|
||||
|
||||
featured_order: int | None = Field(
|
||||
@@ -109,6 +114,7 @@ class AgentCreateRequest(BaseModel):
|
||||
model: str | None = Field(default=None, description="Optional model override persisted to config.yaml")
|
||||
tool_groups: list[str] | None = Field(default=None, description="Optional tool group whitelist persisted to config.yaml")
|
||||
common_questions: list[CommonQuestion] | None = Field(default=None, description="Recommended questions (常用问题) persisted to config.yaml. Each item is {title, prompt}; legacy strings are coerced.")
|
||||
template_name: str | None = Field(default=None, description="入库模板名 persisted to config.yaml. Enables publishing markdown artifacts to the external knowledge-ingest API.")
|
||||
|
||||
|
||||
class AgentUpdateRequest(BaseModel):
|
||||
@@ -124,6 +130,7 @@ class AgentUpdateRequest(BaseModel):
|
||||
model: str | None = Field(default=None, description="Updated model override (config.yaml)")
|
||||
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
|
||||
common_questions: list[CommonQuestion] | None = Field(default=None, description="Updated recommended questions (常用问题, config.yaml). Each item is {title, prompt}; legacy strings are coerced.")
|
||||
template_name: str | None = Field(default=None, description="Updated 入库模板名 (config.yaml). Empty string clears it.")
|
||||
|
||||
|
||||
class AgentSelectorResponse(BaseModel):
|
||||
@@ -228,6 +235,7 @@ def _write_agent_config(
|
||||
model: str | None = None,
|
||||
tool_groups: list[str] | None = None,
|
||||
common_questions: list[CommonQuestion] | None = None,
|
||||
template_name: str | None = None,
|
||||
) -> None:
|
||||
agent_dir = get_paths().agent_dir(agent_id)
|
||||
agent_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -254,13 +262,15 @@ def _write_agent_config(
|
||||
config_data["common_questions"] = [
|
||||
q.model_dump() for q in normalized if q.prompt.strip()
|
||||
]
|
||||
if template_name is not None and template_name.strip():
|
||||
config_data["template_name"] = template_name.strip()
|
||||
with open(config_file, "w", encoding="utf-8") as f:
|
||||
yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
|
||||
def _load_config_extras(agent_id: str) -> dict[str, Any]:
|
||||
"""Read skills/model/tool_groups/common_questions from the agent's config.yaml, tolerant of missing files."""
|
||||
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None, "template_name": None}
|
||||
try:
|
||||
cfg = load_agent_config(agent_id)
|
||||
except (FileNotFoundError, ValueError):
|
||||
@@ -272,6 +282,7 @@ def _load_config_extras(agent_id: str) -> dict[str, Any]:
|
||||
"model": cfg.model,
|
||||
"tool_groups": cfg.tool_groups,
|
||||
"common_questions": cfg.common_questions,
|
||||
"template_name": cfg.template_name,
|
||||
}
|
||||
|
||||
|
||||
@@ -432,7 +443,7 @@ def _record_to_response(
|
||||
can happen lazily when the agent is actually opened.
|
||||
"""
|
||||
soul: str | None = None
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None, "template_name": None}
|
||||
if include_extras:
|
||||
_ensure_agent_dir_from_record(record)
|
||||
extras = _load_config_extras(record["id"])
|
||||
@@ -455,6 +466,7 @@ def _record_to_response(
|
||||
tool_groups=extras["tool_groups"],
|
||||
skills=extras["skills"],
|
||||
common_questions=extras.get("common_questions"),
|
||||
template_name=extras.get("template_name"),
|
||||
featured_order=record.get("featured_order"),
|
||||
)
|
||||
|
||||
@@ -723,6 +735,10 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
|
||||
await _maybe_sync_legacy_agents(store)
|
||||
user_id = _current_user_id(request)
|
||||
record = await store.get_visible(agent_id, user_id)
|
||||
if record is None and _is_admin_user(request):
|
||||
# The admin console lists every agent (?admin=true), so an admin
|
||||
# must be able to open any of them — mirror that here.
|
||||
record = await store.get_any(agent_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
|
||||
@@ -773,6 +789,7 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
|
||||
model=request.model,
|
||||
tool_groups=request.tool_groups,
|
||||
common_questions=request.common_questions,
|
||||
template_name=request.template_name,
|
||||
)
|
||||
_write_agent_soul(agent_id, request.soul)
|
||||
|
||||
@@ -852,8 +869,9 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
next_model = request.model if "model" in fields_set else existing_extras["model"]
|
||||
next_tool_groups = request.tool_groups if "tool_groups" in fields_set else existing_extras["tool_groups"]
|
||||
next_common_questions = request.common_questions if "common_questions" in fields_set else existing_extras["common_questions"]
|
||||
next_template_name = request.template_name if "template_name" in fields_set else existing_extras["template_name"]
|
||||
|
||||
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions"))
|
||||
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions", "template_name"))
|
||||
if config_dirty:
|
||||
_write_agent_config(
|
||||
agent_id,
|
||||
@@ -863,6 +881,7 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
model=next_model,
|
||||
tool_groups=next_tool_groups,
|
||||
common_questions=next_common_questions,
|
||||
template_name=next_template_name,
|
||||
)
|
||||
# Keep the listing cache in lock-step with config.yaml, but only
|
||||
# when the cached fields were actually touched — a partial update
|
||||
|
||||
@@ -424,6 +424,7 @@ async def list_notes(
|
||||
request: Request,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
entity: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
@@ -432,7 +433,7 @@ async def list_notes(
|
||||
service = get_knowledge_service(request)
|
||||
limit = max(1, min(limit, 100))
|
||||
offset = max(0, offset)
|
||||
return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
|
||||
return await service.list_notes(keyword=keyword, tag=tag, entity=entity, source_type=source_type, status=status, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@router.get("/notes/{note_id}")
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""外部知识入库接口(发布 Markdown 产物到 knowes 知识库)。
|
||||
|
||||
智能体 config.yaml 配置了 ``template_name`` 后,该智能体会话中生成的 Markdown
|
||||
产物可以通过 ``POST /api/knowledge-ingest/publish`` 一键入库。
|
||||
|
||||
入库目标接口的字段名不固定:标题字段可能叫「名称」也可能叫「标题」,因此先按
|
||||
「名称」发送一次,若判定失败则用 **同一个 nodeName** 把标题字段换成「标题」再
|
||||
发一次。nodeName 是入库记录的唯一标识,取「md 标题 + 时间戳」。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/knowledge-ingest", tags=["knowledge-ingest"])
|
||||
|
||||
# 入库接口地址,可用环境变量覆盖(如切换 UAT/生产)。
|
||||
_REQUEST_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
# 标题字段在目标模板里可能叫「名称」或「标题」,按此顺序逐个尝试。
|
||||
_TITLE_FIELD_CANDIDATES = ("名称", "标题")
|
||||
_CONTENT_FIELD = "内容"
|
||||
|
||||
|
||||
class KnowledgeIngestPublishRequest(BaseModel):
|
||||
"""Request body for publishing one markdown artifact."""
|
||||
|
||||
agent_id: str = Field(..., description="智能体 id,用于读取其 config.yaml 中的 template_name")
|
||||
title: str = Field(..., description="md 标题(一级标题或文件名)")
|
||||
content: str = Field(..., description="md 正文全文")
|
||||
|
||||
|
||||
class KnowledgeIngestAttempt(BaseModel):
|
||||
"""One delivery attempt against the ingest API."""
|
||||
|
||||
title_field: str = Field(..., description="本次尝试使用的标题字段名(名称/标题)")
|
||||
status_code: int | None = Field(default=None, description="HTTP 状态码;请求异常时为 null")
|
||||
response_text: str = Field(default="", description="入库接口原始响应(截断到 2000 字符)")
|
||||
success: bool = False
|
||||
error: str | None = Field(default=None, description="请求层异常信息(网络/超时等)")
|
||||
|
||||
|
||||
class KnowledgeIngestPublishResponse(BaseModel):
|
||||
success: bool
|
||||
template_name: str
|
||||
node_name: str = Field(..., description="入库唯一标识(md 标题 + 时间戳),两次尝试共用")
|
||||
batch_number: str
|
||||
attempts: list[KnowledgeIngestAttempt]
|
||||
|
||||
|
||||
def _judge_success(status_code: int, response_text: str) -> bool:
|
||||
"""判断入库接口响应是否表示成功。
|
||||
|
||||
接口未提供正式契约:优先看响应 JSON 中常见的 success/code/status/state
|
||||
字段;JSON 不可解析或没有这类字段时,退化为按 HTTP 2xx 判定。
|
||||
"""
|
||||
if not (200 <= status_code < 300):
|
||||
return False
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except (ValueError, TypeError):
|
||||
return True
|
||||
if not isinstance(data, dict):
|
||||
return True
|
||||
if isinstance(data.get("success"), bool):
|
||||
return data["success"]
|
||||
for key in ("code", "status", "state"):
|
||||
if key in data:
|
||||
return str(data[key]).lower() in {"0", "200", "success", "ok", "true"}
|
||||
return True
|
||||
|
||||
|
||||
def _build_payload(template_name: str, batch_number: str, node_name: str, title_field: str, title: str, content: str) -> dict[str, Any]:
|
||||
return {
|
||||
"templateName": template_name,
|
||||
"batchNumber": batch_number,
|
||||
"list": [
|
||||
{
|
||||
"rec": {"nodeName": node_name},
|
||||
"data": [
|
||||
{"fieldName": title_field, "value": title},
|
||||
{"fieldName": _CONTENT_FIELD, "value": content},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_node_name(title: str, timestamp: str) -> str:
|
||||
"""Build a stable node name as md5(title) + timestamp + .md."""
|
||||
title_md5 = hashlib.md5(title.encode("utf-8")).hexdigest()
|
||||
return f"{title_md5}-{timestamp}.md"
|
||||
|
||||
|
||||
def _resolve_ingest_url() -> str:
|
||||
"""Resolve the external ingest URL from config.yaml, with env fallback."""
|
||||
config_url = getattr(get_app_config().knowledge_ingest, "url", "").strip()
|
||||
return os.environ.get(
|
||||
"KNOWES_INGEST_URL",
|
||||
config_url or "https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/publish", response_model=KnowledgeIngestPublishResponse, summary="Publish markdown artifact to external knowledge base")
|
||||
async def publish_to_knowledge_base(request: KnowledgeIngestPublishRequest) -> KnowledgeIngestPublishResponse:
|
||||
title = request.title.strip()
|
||||
content = request.content.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=422, detail="标题不能为空")
|
||||
if not content:
|
||||
raise HTTPException(status_code=422, detail="内容为空,无法入库")
|
||||
|
||||
try:
|
||||
agent_cfg = load_agent_config(request.agent_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
template_name = (agent_cfg.template_name or "").strip() if agent_cfg else ""
|
||||
if not template_name:
|
||||
raise HTTPException(status_code=400, detail=f"智能体 '{request.agent_id}' 未配置入库模板名(template_name),无法发布")
|
||||
|
||||
now = datetime.now()
|
||||
# nodeName 是入库记录的唯一标识:md 标题 + 时间戳。两次尝试必须共用同一个
|
||||
# nodeName,避免第一次实际已入库时第二次产生重复记录。
|
||||
timestamp = now.strftime("%Y%m%d%H%M%S")
|
||||
node_name = _build_node_name(title, timestamp)
|
||||
batch_number = f"md{now.strftime('%Y%m%d')}"
|
||||
ingest_url = _resolve_ingest_url()
|
||||
|
||||
logger.info(
|
||||
"[knowledge-ingest] 开始入库: agent=%s template=%s nodeName=%s batchNumber=%s title=%r content_len=%d url=%s",
|
||||
request.agent_id,
|
||||
template_name,
|
||||
node_name,
|
||||
batch_number,
|
||||
title,
|
||||
len(content),
|
||||
ingest_url,
|
||||
)
|
||||
|
||||
attempts: list[KnowledgeIngestAttempt] = []
|
||||
headers = {"Content-Type": "application/json", "User-Agent": "DeerFlow/1.0"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=_REQUEST_TIMEOUT_SECONDS) as client:
|
||||
for title_field in _TITLE_FIELD_CANDIDATES:
|
||||
payload = _build_payload(template_name, batch_number, node_name, title_field, title, content)
|
||||
logger.info(
|
||||
"[knowledge-ingest] 第 %d 次发送 (标题字段=%s): %s",
|
||||
len(attempts) + 1,
|
||||
title_field,
|
||||
json.dumps({**payload, "list": [{**payload["list"][0], "data": [{"fieldName": d["fieldName"], "value": d["value"][:200]} for d in payload["list"][0]["data"]]}]}, ensure_ascii=False),
|
||||
)
|
||||
try:
|
||||
response = await client.post(ingest_url, json=payload, headers=headers)
|
||||
except httpx.HTTPError as exc:
|
||||
logger.error("[knowledge-ingest] 第 %d 次发送请求异常 (标题字段=%s): %s", len(attempts) + 1, title_field, exc)
|
||||
attempts.append(KnowledgeIngestAttempt(title_field=title_field, error=str(exc)))
|
||||
continue
|
||||
|
||||
text = response.text or ""
|
||||
success = _judge_success(response.status_code, text)
|
||||
logger.info(
|
||||
"[knowledge-ingest] 第 %d 次响应 (标题字段=%s): status=%d success=%s body=%s",
|
||||
len(attempts) + 1,
|
||||
title_field,
|
||||
response.status_code,
|
||||
success,
|
||||
text[:2000],
|
||||
)
|
||||
attempts.append(
|
||||
KnowledgeIngestAttempt(
|
||||
title_field=title_field,
|
||||
status_code=response.status_code,
|
||||
response_text=text[:2000],
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
if success:
|
||||
break
|
||||
logger.warning("[knowledge-ingest] 第 %d 次入库判定失败 (标题字段=%s),准备换字段重试", len(attempts), title_field)
|
||||
|
||||
overall = any(a.success for a in attempts)
|
||||
if overall:
|
||||
logger.info("[knowledge-ingest] 入库成功: nodeName=%s 使用标题字段=%s", node_name, next(a.title_field for a in attempts if a.success))
|
||||
else:
|
||||
logger.error("[knowledge-ingest] 入库失败(全部 %d 次尝试均未成功): nodeName=%s", len(attempts), node_name)
|
||||
|
||||
return KnowledgeIngestPublishResponse(
|
||||
success=overall,
|
||||
template_name=template_name,
|
||||
node_name=node_name,
|
||||
batch_number=batch_number,
|
||||
attempts=attempts,
|
||||
)
|
||||
@@ -21,7 +21,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer, get_current_user, get_feedback_repo, get_run_event_store, get_run_manager, get_run_store, get_stream_bridge
|
||||
from app.gateway.services import sse_consumer, start_run
|
||||
from app.gateway.services import immediate_end_sse, sse_consumer, start_run
|
||||
from deerflow.runtime import RunRecord, serialize_channel_values
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -96,7 +96,7 @@ def _record_to_response(record: RunRecord) -> RunResponse:
|
||||
|
||||
|
||||
@router.post("/{thread_id}/runs", response_model=RunResponse)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
|
||||
async def create_run(thread_id: str, body: RunCreateRequest, request: Request) -> RunResponse:
|
||||
"""Create a background run (returns immediately)."""
|
||||
record = await start_run(body, thread_id, request)
|
||||
@@ -104,7 +104,7 @@ async def create_run(thread_id: str, body: RunCreateRequest, request: Request) -
|
||||
|
||||
|
||||
@router.post("/{thread_id}/runs/stream")
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
|
||||
async def stream_run(thread_id: str, body: RunCreateRequest, request: Request) -> StreamingResponse:
|
||||
"""Create a run and stream events via SSE.
|
||||
|
||||
@@ -132,7 +132,7 @@ async def stream_run(thread_id: str, body: RunCreateRequest, request: Request) -
|
||||
|
||||
|
||||
@router.post("/{thread_id}/runs/wait", response_model=dict)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
|
||||
async def wait_run(thread_id: str, body: RunCreateRequest, request: Request) -> dict:
|
||||
"""Create a run and block until it completes, returning the final state."""
|
||||
record = await start_run(body, thread_id, request)
|
||||
@@ -173,7 +173,11 @@ async def get_run(thread_id: str, run_id: str, request: Request) -> RunResponse:
|
||||
run_mgr = get_run_manager(request)
|
||||
record = run_mgr.get(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
# In-memory miss: fall back to the persistent store so a run still
|
||||
# resolves after a Gateway restart instead of 404ing.
|
||||
record = await run_mgr.get_persisted(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
return _record_to_response(record)
|
||||
|
||||
|
||||
@@ -222,11 +226,27 @@ async def join_run(thread_id: str, run_id: str, request: Request) -> StreamingRe
|
||||
bridge = get_stream_bridge(request)
|
||||
run_mgr = get_run_manager(request)
|
||||
record = run_mgr.get(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
if record is not None and record.thread_id == thread_id:
|
||||
return StreamingResponse(
|
||||
sse_consumer(bridge, record, request, run_mgr),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
# Not in memory: a run that still exists in the store but has no live task
|
||||
# means the Gateway restarted (or the finished run was cleaned up). There
|
||||
# is nothing to reconnect to, so stream a clean terminal end instead of a
|
||||
# 404 — the SDK then clears its reconnect key and renders history.
|
||||
persisted = await run_mgr.get_persisted(run_id)
|
||||
if persisted is None or persisted.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
|
||||
return StreamingResponse(
|
||||
sse_consumer(bridge, record, request, run_mgr),
|
||||
immediate_end_sse(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
@@ -255,7 +275,22 @@ async def stream_existing_run(
|
||||
run_mgr = get_run_manager(request)
|
||||
record = run_mgr.get(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
# In-memory miss. If the run survives in the store it's a dead record
|
||||
# from before a restart (or already finished) — nothing to join or
|
||||
# cancel, so emit a clean terminal end so the SDK stops reconnecting
|
||||
# instead of 404ing. A truly unknown run still 404s.
|
||||
persisted = await run_mgr.get_persisted(run_id)
|
||||
if persisted is None or persisted.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
return StreamingResponse(
|
||||
immediate_end_sse(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
# Cancel if an action was requested (stop-button / interrupt flow)
|
||||
if action is not None:
|
||||
|
||||
@@ -42,6 +42,20 @@ logger = logging.getLogger(__name__)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def immediate_end_sse():
|
||||
"""Yield a single terminal ``end`` SSE frame and stop.
|
||||
|
||||
Used when a run exists in the persistent store but not in the in-memory
|
||||
registry — typically after a Gateway restart, or once a finished run's
|
||||
record has been cleaned up. There is no live stream to join, so instead of
|
||||
returning 404 (which the SDK's ``joinStream`` surfaces as a hard error and
|
||||
never clears its reconnect key for) we stream a clean end. The SDK treats
|
||||
this as success, clears ``lg:stream:{threadId}``, and the conversation
|
||||
falls back to history rendering — killing the repeating switch-back 404.
|
||||
"""
|
||||
yield format_sse("end", None)
|
||||
|
||||
|
||||
def format_sse(event: str, data: Any, *, event_id: str | None = None) -> str:
|
||||
"""Format a single SSE frame.
|
||||
|
||||
@@ -477,6 +491,9 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
|
||||
store = get_agent_store(request)
|
||||
await _maybe_sync_legacy_agent_records(store)
|
||||
user_id = get_effective_user_id()
|
||||
# Admins see every agent in the admin console listing (?admin=true), so
|
||||
# they may also start a run against any of them.
|
||||
is_admin = getattr(getattr(request.state, "user", None), "system_role", None) == "admin"
|
||||
for agent_id in agent_ids:
|
||||
try:
|
||||
normalized = validate_agent_id(agent_id)
|
||||
@@ -484,8 +501,11 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
if normalized is None:
|
||||
continue
|
||||
if await store.get_visible(normalized, user_id) is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent {normalized} not found")
|
||||
if await store.get_visible(normalized, user_id) is not None:
|
||||
continue
|
||||
if is_admin and await store.get_any(normalized) is not None:
|
||||
continue
|
||||
raise HTTPException(status_code=404, detail=f"Agent {normalized} not found")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"workspace-search": { "enabled": true },
|
||||
"policy-knowledge": { "enabled": false },
|
||||
"html-page-builder": { "enabled": true },
|
||||
"html-charts": { "enabled": true }
|
||||
"html-charts": { "enabled": true },
|
||||
"html-multipage": { "enabled": true }
|
||||
}
|
||||
}
|
||||
@@ -100,6 +100,9 @@ class AgentConfig(BaseModel):
|
||||
# Accepts both the legacy ``list[str]`` shape and the new ``{title, prompt}``
|
||||
# objects (see CommonQuestion).
|
||||
common_questions: list[CommonQuestion] | None = None
|
||||
# 入库模板名(外部知识入库接口的 templateName)。配置后该智能体会话中的
|
||||
# Markdown 产物可一键发布到外部入库接口;None/空 = 不支持入库。
|
||||
template_name: str | None = None
|
||||
|
||||
|
||||
def load_agent_config(agent_id: str | None) -> AgentConfig | None:
|
||||
|
||||
@@ -15,9 +15,10 @@ from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpo
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.gateway_config import GatewayConfig
|
||||
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.log_system_config import LogSystemConfig
|
||||
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
|
||||
from deerflow.config.knowledge_config import KnowledgeConfig
|
||||
from deerflow.config.knowledge_ingest_config import KnowledgeIngestConfig
|
||||
from deerflow.config.log_system_config import LogSystemConfig
|
||||
from deerflow.config.login_config import AuthLoginConfig
|
||||
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
|
||||
from deerflow.config.model_config import ModelConfig
|
||||
@@ -135,8 +136,9 @@ class AppConfig(BaseModel):
|
||||
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
|
||||
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
|
||||
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
|
||||
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
|
||||
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
|
||||
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
|
||||
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
|
||||
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
|
||||
|
||||
@classmethod
|
||||
def resolve_config_path(cls, config_path: str | None = None) -> Path:
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
"""Configuration for the external knowledge-ingest endpoint."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class KnowledgeIngestConfig(BaseModel):
|
||||
"""External knowledge-ingest configuration (`config.yaml` -> `knowledge_ingest`)."""
|
||||
|
||||
url: str = Field(
|
||||
default="https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add",
|
||||
description="External knowledge-ingest endpoint URL.",
|
||||
)
|
||||
@@ -13,25 +13,35 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
|
||||
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
|
||||
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
|
||||
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
|
||||
# with error 1071 ("Specified key was too long").
|
||||
|
||||
|
||||
class KnowledgeNoteRow(Base):
|
||||
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
|
||||
|
||||
__tablename__ = "knowledge_notes"
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
|
||||
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024))
|
||||
source_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=0.0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
@@ -118,10 +128,15 @@ class KnowledgeEntityRow(Base):
|
||||
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
|
||||
|
||||
__tablename__ = "knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_entities_name", "name", mysql_length=191),
|
||||
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
|
||||
# realistically far shorter than 120 chars, so the prefix is lossless.
|
||||
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), index=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
entity_type: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
@@ -134,9 +149,10 @@ class KnowledgeGraphBlacklistRow(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_graph_blacklist"
|
||||
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
label: Mapped[str] = mapped_column(String(512), unique=True, index=True)
|
||||
label: Mapped[str] = mapped_column(String(512))
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
@@ -276,6 +276,7 @@ class KnowledgeRepository:
|
||||
*,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
entity: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
@@ -303,6 +304,16 @@ class KnowledgeRepository:
|
||||
.where(func.lower(KnowledgeTagRow.name) == tag.lower())
|
||||
)
|
||||
conditions.append(KnowledgeNoteRow.id.in_(tag_subq))
|
||||
if entity:
|
||||
# Exact entity match via the note→entity relation graph — precise
|
||||
# and index-backed, unlike a content LIKE scan on large vaults.
|
||||
entity_subq = (
|
||||
select(KnowledgeRelationRow.from_note_id)
|
||||
.join(KnowledgeEntityRow, KnowledgeRelationRow.to_entity_id == KnowledgeEntityRow.id)
|
||||
.where(func.lower(KnowledgeEntityRow.name) == entity.lower())
|
||||
.where(KnowledgeRelationRow.from_note_id.is_not(None))
|
||||
)
|
||||
conditions.append(KnowledgeNoteRow.id.in_(entity_subq))
|
||||
|
||||
for cond in conditions:
|
||||
stmt = stmt.where(cond)
|
||||
|
||||
@@ -523,6 +523,7 @@ class KnowledgeService:
|
||||
*,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
entity: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
@@ -531,6 +532,7 @@ class KnowledgeService:
|
||||
items, total = await self.repo.list_notes(
|
||||
keyword=keyword,
|
||||
tag=tag,
|
||||
entity=entity,
|
||||
source_type=normalize_source_type(source_type) if source_type else None,
|
||||
status=status,
|
||||
limit=limit,
|
||||
|
||||
+10
-1
@@ -32,6 +32,15 @@ class MemoryAgentStore(AgentStore):
|
||||
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
|
||||
return agent.get("user_id") is None or agent.get("user_id") == user_id or bool(agent.get("published"))
|
||||
|
||||
def _is_fetchable(self, agent: dict[str, Any], user_id: str) -> bool:
|
||||
"""Single-record visibility (``get_visible``) — the union of every
|
||||
listing surface, so a list-visible agent always opens. Mirrors
|
||||
``AgentRepository._visible_filter``: baseline visibility OR
|
||||
private-square (password is the discovery gate, reads are not
|
||||
re-gated) OR favorited (the "我的" listing keeps favorites even after
|
||||
the owner unpublishes)."""
|
||||
return self._is_visible(agent, user_id) or bool(agent.get("is_private")) or (user_id, agent["id"]) in self._favorites
|
||||
|
||||
async def list_visible(self, user_id: str) -> list[dict[str, Any]]:
|
||||
rows = [_copy(agent) for agent in self._agents.values() if self._is_visible(agent, user_id)]
|
||||
rows.sort(key=lambda agent: agent.get("created_at") or "")
|
||||
@@ -39,7 +48,7 @@ class MemoryAgentStore(AgentStore):
|
||||
|
||||
async def get_visible(self, agent_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
agent = self._agents.get(agent_id)
|
||||
if agent is None or not self._is_visible(agent, user_id):
|
||||
if agent is None or not self._is_fetchable(agent, user_id):
|
||||
return None
|
||||
return _copy(agent)
|
||||
|
||||
|
||||
+7
-3
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -32,7 +32,7 @@ class AgentRow(Base):
|
||||
# ``is_private``. Empty string = "not assigned" → the listing layer treats
|
||||
# it as the admin-configured default square. Each agent belongs to at most
|
||||
# one square (single column), so it can never be published to two squares.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", server_default="", index=True)
|
||||
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
|
||||
# featured rank (lower sorts first). Only ``published=True`` agents are
|
||||
# eligible; the router enforces that invariant on writes. BIGINT so the
|
||||
@@ -86,7 +86,11 @@ class AgentSkillRow(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "agent_skills"
|
||||
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
|
||||
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
|
||||
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
|
||||
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
|
||||
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
+20
-5
@@ -40,12 +40,27 @@ class AgentRepository(AgentStore):
|
||||
def _visible_filter(user_id: str):
|
||||
"""Visibility for single-record fetches (``get_visible``).
|
||||
|
||||
Lenient: a caller who already knows the agent id can fetch it even
|
||||
when it's flagged ``is_private``. The private-square password is the
|
||||
*discovery* gate (listings); individual reads are not re-gated, so a
|
||||
bookmarked / linked private agent stays openable.
|
||||
Lenient — must cover the union of every listing surface, so any agent
|
||||
a user can see in a list always opens (detail + chat) instead of
|
||||
404ing:
|
||||
|
||||
- built-in / owned / published — the baseline gallery visibility;
|
||||
- ``is_private`` — the private-square password is the *discovery*
|
||||
gate (listings); individual reads are not re-gated, so a private
|
||||
agent opened from the square (or a bookmark/link) stays openable.
|
||||
Note moving an agent into the private square forces
|
||||
``published=false``, so this clause cannot lean on ``published``;
|
||||
- favorited — the "我的" listing keeps favorited agents even after
|
||||
the owner unpublishes them, so reads keep honoring the favorite.
|
||||
"""
|
||||
return or_(AgentRow.user_id.is_(None), AgentRow.user_id == user_id, AgentRow.published.is_(True))
|
||||
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
|
||||
return or_(
|
||||
AgentRow.user_id.is_(None),
|
||||
AgentRow.user_id == user_id,
|
||||
AgentRow.published.is_(True),
|
||||
AgentRow.is_private.is_(True),
|
||||
AgentRow.id.in_(fav_subq),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _listing_filter(user_id: str):
|
||||
|
||||
@@ -194,6 +194,79 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
|
||||
``_run_alembic_upgrade`` call) — so on an existing database a new
|
||||
``mapped_column`` on an ORM model would otherwise error at first query
|
||||
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Indexes covering a newly
|
||||
added column are created as well. Works on sqlite/mysql/postgres — the
|
||||
column DDL is compiled through the active dialect, so portable types
|
||||
(``PortableJSON``/``PortableLongText``/``BeijingDateTime``) resolve to the
|
||||
right backend type. Everything is per-column best-effort: a failure is
|
||||
logged and startup continues.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy.schema import CreateColumn
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
try:
|
||||
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
inspector = sa_inspect(sync_conn)
|
||||
existing_tables = set(inspector.get_table_names())
|
||||
preparer = sync_conn.dialect.identifier_preparer
|
||||
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if table.name not in existing_tables:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
added: set[str] = set()
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
table.name,
|
||||
col.name,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
added.add(col.name)
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
if not added:
|
||||
continue
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
@@ -344,6 +417,15 @@ async def init_engine(
|
||||
else:
|
||||
raise
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
+7
-2
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -14,7 +14,12 @@ from deerflow.persistence.types import BeijingDateTime
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
__table_args__ = (UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),)
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ class SkillRow(Base):
|
||||
# Publish square (发布广场) this skill is filed under once published. Pure
|
||||
# organizational category, no access control. Empty string = unassigned →
|
||||
# the listing layer resolves it to the default square. One square per skill.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", server_default="", index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
@@ -112,6 +112,52 @@ class RunManager:
|
||||
"""Return a run record by ID, or ``None``."""
|
||||
return self._runs.get(run_id)
|
||||
|
||||
def _record_from_store(self, data: dict) -> RunRecord:
|
||||
"""Rebuild a :class:`RunRecord` from a persisted store row.
|
||||
|
||||
The reconstructed record has no live ``task``/``abort_event`` wired to a
|
||||
running coroutine — it represents *history*, not an in-flight run.
|
||||
"""
|
||||
try:
|
||||
status = RunStatus(data.get("status") or RunStatus.success.value)
|
||||
except ValueError:
|
||||
status = RunStatus.success
|
||||
return RunRecord(
|
||||
run_id=data["run_id"],
|
||||
thread_id=data["thread_id"],
|
||||
assistant_id=data.get("assistant_id"),
|
||||
status=status,
|
||||
# Disconnect mode is irrelevant for a store-only record (no task to
|
||||
# cancel); ``continue_`` keeps sse_consumer's finally block a no-op.
|
||||
on_disconnect=DisconnectMode.continue_,
|
||||
multitask_strategy=data.get("multitask_strategy", "reject"),
|
||||
metadata=data.get("metadata") or {},
|
||||
kwargs=data.get("kwargs") or {},
|
||||
created_at=data.get("created_at") or "",
|
||||
updated_at=data.get("updated_at") or "",
|
||||
error=data.get("error"),
|
||||
)
|
||||
|
||||
async def get_persisted(self, run_id: str) -> RunRecord | None:
|
||||
"""Load a run from the backing store (process-restart fallback).
|
||||
|
||||
The in-memory registry is wiped on every Gateway restart, but the
|
||||
``RunStore`` row survives. When :meth:`get` misses, callers fall back
|
||||
here so a stale ``joinStream``/``GET run`` can report the run's final
|
||||
status instead of 404ing. Returns ``None`` when there is no store, no
|
||||
matching row, or the lookup fails (e.g. no user context).
|
||||
"""
|
||||
if self._store is None:
|
||||
return None
|
||||
try:
|
||||
data = await self._store.get(run_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to load run %s from store", run_id, exc_info=True)
|
||||
return None
|
||||
if not data:
|
||||
return None
|
||||
return self._record_from_store(data)
|
||||
|
||||
async def list_by_thread(self, thread_id: str) -> list[RunRecord]:
|
||||
"""Return all runs for a given thread, newest first."""
|
||||
async with self._lock:
|
||||
|
||||
+36
-12
@@ -39,10 +39,15 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
|
||||
"""Return the explicit skill name set declared in an agent's config.yaml.
|
||||
|
||||
Semantics:
|
||||
- ``None`` returned → no agent_id, or config missing — caller should
|
||||
fall back to "all skills visible".
|
||||
- ``set()`` (empty) → agent has ``skills: []`` declared — visibility
|
||||
is intentionally empty.
|
||||
- ``None`` returned → no agent_id, or the agent's on-disk config is
|
||||
missing entirely — caller should fall back to "all skills visible"
|
||||
(default lead agent behavior).
|
||||
- ``set()`` (empty) → custom agent with ``skills: []`` declared **or**
|
||||
with no ``skills`` key at all — visibility is intentionally empty.
|
||||
This mirrors the prompt layer (``agents_config.AgentConfig.skills``:
|
||||
``None``/``[]`` both mean "no skills" for custom agents), so an agent
|
||||
whose config omits ``skills`` can no longer enumerate every skill
|
||||
through this tool.
|
||||
- Non-empty set → only these names are visible to this agent.
|
||||
"""
|
||||
if not agent_id:
|
||||
@@ -51,9 +56,9 @@ def _load_agent_skill_whitelist(agent_id: str | None) -> set[str] | None:
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
|
||||
cfg = load_agent_config(agent_id)
|
||||
if cfg is None or cfg.skills is None:
|
||||
if cfg is None:
|
||||
return None
|
||||
return set(cfg.skills)
|
||||
return set(cfg.skills or [])
|
||||
except Exception:
|
||||
logger.debug("Failed to load agent config for skill_list filter", exc_info=True)
|
||||
return None
|
||||
@@ -119,16 +124,21 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
@tool("skill_view")
|
||||
def skill_view_tool(name: str) -> str:
|
||||
"""Read the full SKILL.md content of a skill by name.
|
||||
def skill_view_impl(name: str, agent_id: str | None) -> str:
|
||||
"""Read a skill's SKILL.md, honoring the agent's config.yaml whitelist.
|
||||
|
||||
Args:
|
||||
name: The skill name (hyphen-case, e.g. 'my-skill').
|
||||
Custom agents may only read skills on their whitelist — same filter as
|
||||
``skill_list``, so a restricted agent cannot bypass its allowlist by
|
||||
guessing skill names. Replies "not found" rather than "forbidden" to
|
||||
avoid leaking which skills exist.
|
||||
"""
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.usage import bump_view
|
||||
|
||||
allowed = _load_agent_skill_whitelist(agent_id)
|
||||
if allowed is not None and name not in allowed:
|
||||
return f"Skill '{name}' not found."
|
||||
|
||||
storage = get_or_new_skill_storage()
|
||||
try:
|
||||
if storage.custom_skill_exists(name):
|
||||
@@ -156,6 +166,16 @@ def skill_view_tool(name: str) -> str:
|
||||
return content
|
||||
|
||||
|
||||
@tool("skill_view")
|
||||
def skill_view_tool(name: str, runtime: ToolRuntime) -> str:
|
||||
"""Read the full SKILL.md content of a skill by name.
|
||||
|
||||
Args:
|
||||
name: The skill name (hyphen-case, e.g. 'my-skill').
|
||||
"""
|
||||
return skill_view_impl(name, _resolve_active_agent_id(runtime))
|
||||
|
||||
|
||||
@tool("skill_list")
|
||||
def skill_list_tool(runtime: ToolRuntime) -> str:
|
||||
"""List skills available to this agent, with metadata and usage stats.
|
||||
@@ -175,11 +195,15 @@ def skill_list_tool(runtime: ToolRuntime) -> str:
|
||||
state, use_count, view_count, patch_count, last_activity_at, pinned,
|
||||
source.
|
||||
"""
|
||||
return skill_list_impl(_resolve_active_agent_id(runtime))
|
||||
|
||||
|
||||
def skill_list_impl(agent_id: str | None) -> str:
|
||||
"""List skills visible to ``agent_id`` (None = default lead agent)."""
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
from deerflow.skills.usage import STATE_ARCHIVED, all_entries, latest_activity_at
|
||||
|
||||
agent_id = _resolve_active_agent_id(runtime)
|
||||
allowed = _load_agent_skill_whitelist(agent_id)
|
||||
|
||||
# 默认 lead agent 没有 per-agent 白名单 —— 改为只暴露"未归档"且"内置或属于
|
||||
|
||||
+6
-4
@@ -218,14 +218,16 @@ async def _skill_manage_impl(
|
||||
"""
|
||||
# Read-only aliases — delegate to the dedicated tools so a model that
|
||||
# over-uses skill_manage doesn't trip on Unsupported action errors.
|
||||
# Pass the active agent_id through so a custom agent's skill whitelist
|
||||
# is enforced on these aliases just like on skill_list/skill_view.
|
||||
if action == "list":
|
||||
from deerflow.tools.builtins.skill_tools import skill_list_tool
|
||||
from deerflow.tools.builtins.skill_tools import _resolve_active_agent_id, skill_list_impl
|
||||
|
||||
return await _to_thread(skill_list_tool.invoke, {})
|
||||
return await _to_thread(skill_list_impl, _resolve_active_agent_id(runtime))
|
||||
if action == "view":
|
||||
from deerflow.tools.builtins.skill_tools import skill_view_tool
|
||||
from deerflow.tools.builtins.skill_tools import _resolve_active_agent_id, skill_view_impl
|
||||
|
||||
return await _to_thread(skill_view_tool.invoke, {"name": name})
|
||||
return await _to_thread(skill_view_impl, name, _resolve_active_agent_id(runtime))
|
||||
|
||||
name = SkillStorage.validate_skill_name(name)
|
||||
lock = _get_lock(name)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Audit ORM index/unique-key byte sizes against MySQL InnoDB prefix limits.
|
||||
|
||||
MySQL with the legacy 767-byte InnoDB index-prefix limit (5.6, or 5.7 with
|
||||
COMPACT row format) rejects any index whose key exceeds 767 bytes — and
|
||||
utf8mb4 costs 4 bytes per char, so an indexed VARCHAR key may total at most
|
||||
191 chars. Modern servers (DYNAMIC row format) allow 3072 bytes = 768 chars.
|
||||
|
||||
Run: PYTHONPATH=. .venv/Scripts/python scripts/audit_index_key_lengths.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
|
||||
import deerflow.persistence.models # noqa: F401 (registers all ORM models)
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
LIMIT_OLD = 767
|
||||
LIMIT_NEW = 3072
|
||||
BYTES_PER_CHAR = 4 # utf8mb4
|
||||
|
||||
|
||||
def _col_key_bytes(col) -> int | None:
|
||||
t = col.type
|
||||
# unwrap TypeDecorator
|
||||
impl = getattr(t, "impl", None)
|
||||
if impl is not None:
|
||||
t = impl
|
||||
if isinstance(t, String):
|
||||
return (t.length or 255) * BYTES_PER_CHAR
|
||||
return 8 # ints/datetimes/floats — small fixed size, close enough
|
||||
|
||||
|
||||
def _key_bytes(col, mysql_length) -> int:
|
||||
full = _col_key_bytes(col) or 0
|
||||
if mysql_length is None:
|
||||
return full
|
||||
if isinstance(mysql_length, dict):
|
||||
prefix = mysql_length.get(col.name)
|
||||
else:
|
||||
prefix = mysql_length
|
||||
if prefix is None:
|
||||
return full
|
||||
return min(full, int(prefix) * BYTES_PER_CHAR)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
bad = 0
|
||||
for table in Base.metadata.sorted_tables:
|
||||
keys: list[tuple[str, str, list, object]] = []
|
||||
for idx in table.indexes:
|
||||
keys.append(("INDEX", idx.name, list(idx.columns), idx.dialect_options["mysql"].get("length")))
|
||||
for cons in table.constraints:
|
||||
if isinstance(cons, UniqueConstraint):
|
||||
keys.append(("UNIQUE", cons.name or "?", list(cons.columns), None))
|
||||
for kind, name, cols, mysql_length in keys:
|
||||
total = sum(_key_bytes(c, mysql_length) for c in cols)
|
||||
if total > LIMIT_OLD:
|
||||
bad += 1
|
||||
tag = "EXCEEDS 3072 (fails on ALL MySQL)" if total > LIMIT_NEW else "exceeds 767 (fails on legacy MySQL)"
|
||||
detail = ", ".join(f"{c.name}({getattr(getattr(c.type, 'impl', c.type), 'length', '?')})" for c in cols)
|
||||
print(f"{table.name}: {kind} {name} = {total} bytes [{detail}] -- {tag}")
|
||||
if not bad:
|
||||
print("OK: every index key fits within 767 bytes (utf8mb4 worst case)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -108,6 +108,8 @@ vision:
|
||||
model_name: deepseek-ai/DeepSeek-OCR
|
||||
# prompt: 请详细描述这张图片的内容。
|
||||
# max_tokens: 2048
|
||||
knowledge_ingest:
|
||||
url: "https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add"
|
||||
# 上传文件识别配置。
|
||||
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
|
||||
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。
|
||||
|
||||
Reference in new issue
Block a user