Merge branch 'ji' into master
This commit is contained in:
commit
e74ef4b7dc
81 files changed
+3712
-1625
No files matched your search
@@ -100,7 +100,7 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
After admin creation, migrate orphan threads from the LangGraph
|
||||
store (metadata.user_id unset) to the admin account. This is the
|
||||
"no-auth → with-auth" upgrade path: users who ran DeerFlow without
|
||||
"no-auth ? with-auth" upgrade path: users who ran DeerFlow without
|
||||
authentication have existing LangGraph thread data that needs an
|
||||
owner assigned.
|
||||
First boot (no admin exists):
|
||||
@@ -108,7 +108,7 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
- The operator must visit ``/setup`` to create the first admin.
|
||||
|
||||
Subsequent boots (admin already exists):
|
||||
- Runs the one-time "no-auth → with-auth" orphan thread migration for
|
||||
- Runs the one-time "no-auth ? with-auth" orphan thread migration for
|
||||
existing LangGraph thread metadata that has no owner_id.
|
||||
|
||||
No SQL persistence migration is needed: the four user_id columns
|
||||
@@ -138,12 +138,12 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
if admin_count == 0:
|
||||
logger.info("=" * 60)
|
||||
logger.info(" First boot detected — no admin account exists.")
|
||||
logger.info(" First boot detected ? no admin account exists.")
|
||||
logger.info(" Visit /setup to complete admin account creation.")
|
||||
logger.info("=" * 60)
|
||||
return
|
||||
|
||||
# Admin already exists — run orphan thread migration for any
|
||||
# Admin already exists ? run orphan thread migration for any
|
||||
# LangGraph thread metadata that pre-dates the auth module.
|
||||
async with sf() as session:
|
||||
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
|
||||
@@ -154,8 +154,8 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
admin_id = str(row.id)
|
||||
|
||||
# LangGraph store orphan migration — non-fatal.
|
||||
# This covers the "no-auth → with-auth" upgrade path for users
|
||||
# LangGraph store orphan migration ? non-fatal.
|
||||
# This covers the "no-auth ? with-auth" upgrade path for users
|
||||
# whose existing LangGraph thread metadata has no user_id set.
|
||||
store = getattr(app.state, "store", None)
|
||||
if store is not None:
|
||||
@@ -217,7 +217,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
# No admin yet — leave ownership null; first admin is responsible for
|
||||
# No admin yet ? leave ownership null; first admin is responsible for
|
||||
# claiming via the UI once they exist.
|
||||
admin_id: str | None = None
|
||||
else:
|
||||
@@ -278,7 +278,7 @@ async def _run_db_migrations() -> None:
|
||||
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
logger.info("Persistence backend=memory — skipping Alembic migrations")
|
||||
logger.info("Persistence backend=memory ? skipping Alembic migrations")
|
||||
return
|
||||
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
|
||||
@@ -327,28 +327,28 @@ async def _run_db_migrations() -> None:
|
||||
raise result
|
||||
logger.info("Database migrations applied (head)")
|
||||
except Exception:
|
||||
logger.exception("Database migration failed — schema may be out of date")
|
||||
logger.exception("Database migration failed ? schema may be out of date")
|
||||
|
||||
|
||||
# Skill curator 调度参数。
|
||||
# - 检查间隔:每隔这么久检查一次"需不需要进化";真正是否执行由 should_run_now() 判定。
|
||||
# - 启动延迟:启动后先等这么久再首次检查,避开启动高峰。
|
||||
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 小时
|
||||
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 分钟
|
||||
# Skill curator ?????
|
||||
# - ????:?????????"??????";??????? should_run_now() ???
|
||||
# - ????:?????????????,???????
|
||||
_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 ??
|
||||
_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 ??
|
||||
|
||||
|
||||
def _acquire_singleton_lock(lock_path):
|
||||
"""跨平台非阻塞独占文件锁,用于多 worker 下的 leader 选举。
|
||||
"""???????????,??? worker ?? leader ???
|
||||
|
||||
成功 -> 返回持有锁的文件句柄(调用方须保持其存活,直到进程退出/主动释放);
|
||||
抢不到锁(已有其它 worker 持有)或打开失败 -> 返回 None。
|
||||
进程异常退出时 OS 会自动释放该锁。
|
||||
?? -> ??????????(?????????,??????/????);
|
||||
????(???? worker ??)????? -> ?? None?
|
||||
??????? OS ????????
|
||||
"""
|
||||
try:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(lock_path, "a+")
|
||||
except OSError:
|
||||
logger.exception("Curator scheduler: 打开锁文件失败")
|
||||
logger.exception("Curator scheduler: ???????")
|
||||
return None
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
@@ -367,7 +367,7 @@ def _acquire_singleton_lock(lock_path):
|
||||
|
||||
|
||||
def _release_singleton_lock(handle) -> None:
|
||||
"""释放并关闭 _acquire_singleton_lock 拿到的锁文件句柄。"""
|
||||
"""????? _acquire_singleton_lock ?????????"""
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
@@ -388,24 +388,24 @@ def _release_singleton_lock(handle) -> None:
|
||||
|
||||
|
||||
async def _curator_scheduler_loop() -> None:
|
||||
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
|
||||
"""??????:?? _CURATOR_CHECK_INTERVAL_SECONDS ???????????
|
||||
|
||||
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
|
||||
运行调度,其余 worker 直接退出,避免 curator 被重复触发、重复调用 LLM。
|
||||
被 cancel 时 CancelledError 会穿过 ``except Exception``(它是 BaseException
|
||||
子类,不会被捕获),协程随之干净退出并在 finally 里释放锁。
|
||||
? worker ???????? leader ?? ?? ???????? worker ????
|
||||
????,?? worker ????,?? curator ?????????? LLM?
|
||||
? cancel ? CancelledError ??? ``except Exception``(?? BaseException
|
||||
??,?????),?????????? finally ?????
|
||||
"""
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.skills.curator import maybe_run_curator
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Curator scheduler: 另一个 worker 已持有调度锁,本 worker 跳过")
|
||||
logger.info("Curator scheduler: ??? worker ??????,? worker ??")
|
||||
return
|
||||
|
||||
logger.info("Curator scheduler: 取得调度锁,本 worker 负责 curator 调度")
|
||||
logger.info("Curator scheduler: ?????,? worker ?? curator ??")
|
||||
try:
|
||||
# 启动后延迟一会儿再首次检查,避开启动高峰
|
||||
# ?????????????,??????
|
||||
await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
@@ -477,7 +477,7 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
|
||||
)
|
||||
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
|
||||
# *previous* day's snapshot once, so records that landed near midnight are fully
|
||||
# captured. Default 1 → "每天凌晨 1 点重新统计前一天".
|
||||
# captured. Default 1 ? "???? 1 ????????".
|
||||
_LEADERBOARD_FINALIZE_HOUR = min(
|
||||
23,
|
||||
_env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
|
||||
@@ -648,7 +648,7 @@ async def _claim_leaderboard_snapshot(
|
||||
if age < running_ttl_seconds:
|
||||
return False
|
||||
row.status = "running"
|
||||
row.message = "排行榜后台统计任务已认领。"
|
||||
row.message = "?????????????"
|
||||
row.error = None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await _time_leaderboard_scheduler_db_operation(
|
||||
@@ -701,7 +701,7 @@ async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_has
|
||||
[yesterday],
|
||||
settings_hash,
|
||||
force=True,
|
||||
message="每日凌晨重新统计前一天完整数据,确保全部记录纳入统计。",
|
||||
message="???????????????????????????",
|
||||
)
|
||||
if queued:
|
||||
logger.info(
|
||||
@@ -734,10 +734,10 @@ async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
|
||||
session_factory,
|
||||
stat_dates,
|
||||
settings_hash,
|
||||
message="排行榜后台统计任务定期预热/补算队列。",
|
||||
message="?????????????/?????",
|
||||
)
|
||||
|
||||
# 每天凌晨(默认 1 点)后强制重算"前一天"的完整快照一次,覆盖跨午夜才落库的记录。
|
||||
# ??????? 1 ???????"???"????????????????????
|
||||
try:
|
||||
await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
|
||||
except Exception:
|
||||
@@ -816,31 +816,31 @@ async def _leaderboard_snapshot_scheduler_loop() -> None:
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
|
||||
"""?? uvicorn / gunicorn ? worker ???? ERROR ???
|
||||
|
||||
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
|
||||
维护 run_id → SSE 订阅者的内存 map。多 worker 下,用户在 worker A 发起
|
||||
``/runs/stream``、稍后在 worker B 调 join/resume,会因 in-process map 互相
|
||||
看不见而 404 / 卡死。
|
||||
? agent / canvas / AI ??????? Gateway ????? ``StreamBridge``
|
||||
?? run_id ? SSE ?????? map?? worker ????? worker A ??
|
||||
``/runs/stream``???? worker B ? join/resume??? in-process map ??
|
||||
???? 404 / ???
|
||||
|
||||
所以这里在 startup 打 ERROR 强提醒。**不抛异常 fail-fast** —— 万一
|
||||
运维确实知道自己在干嘛(比如配了 sticky session),不应该阻止启动。
|
||||
????? startup ? ERROR ????**???? fail-fast** ?? ??
|
||||
???????????????? sticky session??????????
|
||||
"""
|
||||
# uvicorn / gunicorn 都不会主动暴露 worker 数到代码层。可以从环境变量
|
||||
# 或者通过观察 sibling 进程数粗略判断。优先用 WEB_CONCURRENCY(gunicorn
|
||||
# / uvicorn 都认)。
|
||||
# uvicorn / gunicorn ??????? worker ?????????????
|
||||
# ?????? sibling ??????????? WEB_CONCURRENCY?gunicorn
|
||||
# / uvicorn ????
|
||||
web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
|
||||
if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 WEB_CONCURRENCY=%s(多 worker 部署)。AI 写作 / 主 agent "
|
||||
"流式响应依赖进程内内存状态,多 worker 下会出现「会话丢失 / "
|
||||
"SSE 断流 / resume 400」等问题。强烈建议改为 WEB_CONCURRENCY=1 "
|
||||
"或在反向代理上启用 sticky session。详见 AIWritingPage.tsx 的注释。",
|
||||
"? ??? WEB_CONCURRENCY=%s?? worker ????AI ?? / ? agent "
|
||||
"??????????????? worker ????????? / "
|
||||
"SSE ?? / resume 400??????????? WEB_CONCURRENCY=1 "
|
||||
"????????? sticky session??? AIWritingPage.tsx ????",
|
||||
web_concurrency,
|
||||
)
|
||||
return
|
||||
|
||||
# 兜底:扫 /proc 里同名 uvicorn 进程数(只在 Linux 容器里能用)
|
||||
# ???? /proc ??? uvicorn ?????? Linux ??????
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
@@ -858,13 +858,13 @@ def _warn_if_multi_worker() -> None:
|
||||
peers += 1
|
||||
if peers > 1:
|
||||
logger.error(
|
||||
"⚠ 检测到 %d 个同名 uvicorn 进程。多 worker 部署会导致 AI 写作 "
|
||||
"「会话丢失 / SSE 断流」等问题,请改为单 worker 或在反向代理上 "
|
||||
"启用 sticky session。",
|
||||
"? ??? %d ??? uvicorn ???? worker ????? AI ?? "
|
||||
"????? / SSE ??????????? worker ??????? "
|
||||
"?? sticky session?",
|
||||
peers,
|
||||
)
|
||||
except Exception:
|
||||
# 检测本身失败不算事 —— 这只是个提醒
|
||||
# ????????? ?? ??????
|
||||
pass
|
||||
|
||||
|
||||
@@ -872,8 +872,8 @@ def _warn_if_multi_worker() -> None:
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# 早一点检测多 worker 部署 —— 这是 AI 写作 / 主 agent 流式响应的常见
|
||||
# 配置陷阱,每个 worker 启动时都会打一条 ERROR 强提醒运维
|
||||
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
|
||||
# ??????? worker ???????? ERROR ?????
|
||||
_warn_if_multi_worker()
|
||||
|
||||
# Load config and check necessary environment variables at startup
|
||||
@@ -895,7 +895,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Run Alembic migrations before anything else touches the schema.
|
||||
await _run_db_migrations()
|
||||
|
||||
# 预热 recall_budget cache(从 DB 批量加载,使 get_effective_memory_config 同步可读)
|
||||
# ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
|
||||
try:
|
||||
from deerflow.config.memory_config import load_recall_budget_cache
|
||||
|
||||
@@ -911,18 +911,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Must run AFTER langgraph_runtime so app.state.store is available for thread migration
|
||||
await _ensure_admin_user(app)
|
||||
|
||||
# Migrate "ownerless" on-disk skills into the new SkillStore table —
|
||||
# Migrate "ownerless" on-disk skills into the new SkillStore table ?
|
||||
# makes the multitenancy switch transparent for existing deployments.
|
||||
try:
|
||||
await _sync_legacy_skills(app)
|
||||
except Exception:
|
||||
logger.exception("Skill ownership migration failed (non-fatal)")
|
||||
|
||||
# 圆桌功能型 agent(intent / recommender / coordinator)缺失自动补建。
|
||||
# 必须在 _sync_legacy_agents **之前**——这样刚补建的目录能在同一次启动被
|
||||
# 扫描并 upsert 进 agents 表,新机器一启动 /api/agents 就有这三个,
|
||||
# 无需用户先打开圆桌页触发懒加载。.deer-flow/ 被 gitignore,新机器 clone
|
||||
# 后这些目录不存在,seeder 从源码资源 _roundtable_seed_assets/ 复制验证版。
|
||||
# ????? agent(intent / recommender / coordinator)???????
|
||||
# ??? _sync_legacy_agents **??**??????????????????
|
||||
# ??? upsert ? agents ?,?????? /api/agents ?????,
|
||||
# ????????????????.deer-flow/ ? gitignore,??? clone
|
||||
# ????????,seeder ????? _roundtable_seed_assets/ ??????
|
||||
try:
|
||||
from app.gateway.routers._roundtable_seed import (
|
||||
ensure_roundtable_functional_agents,
|
||||
@@ -934,9 +934,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"Roundtable functional agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# AI 写作功能型 agent(researcher / outliner / writer / editor)缺失自动补建。
|
||||
# 同圆桌:必须在 _sync_legacy_agents 之前,补建目录才能在同一次启动被
|
||||
# upsert 进 agents 表,admin 启动后即可在智能体管理页编辑四个写作 agent。
|
||||
# AI ????? agent(researcher / outliner / writer / editor)???????
|
||||
# ???:??? _sync_legacy_agents ??,?????????????
|
||||
# upsert ? agents ?,admin ?????????????????? agent?
|
||||
try:
|
||||
from app.gateway.routers._ai_writing_seed import (
|
||||
ensure_ai_writing_functional_agents,
|
||||
@@ -948,7 +948,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"AI-writing functional agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 业务链条管理:幂等补建固定业务行(3Q/6BF/7BF/8BF),agent/chain 初始为空。
|
||||
# ?????????????????3Q/6BF/7BF/8BF??agent/chain ?????
|
||||
try:
|
||||
business_mapping_store = getattr(app.state, "business_mapping_store", None)
|
||||
if business_mapping_store is not None:
|
||||
@@ -958,10 +958,10 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Business mapping seeding failed at startup (non-fatal)")
|
||||
|
||||
# 一次性把 .deer-flow/agents/{id}/ 目录里历史遗留的 agent 导入 agents 表。
|
||||
# 路由层原本每次请求都跑一遍这个 sync(扫文件系统 + N 个 agent 各一次 DB
|
||||
# SELECT/UPDATE),把它挪到启动期 + 路由层只保留 TTL 兜底,显著降低
|
||||
# /api/agents 列表延迟。
|
||||
# ???? .deer-flow/agents/{id}/ ???????? agent ?? agents ??
|
||||
# ??????????????? sync(????? + N ? agent ??? DB
|
||||
# SELECT/UPDATE),??????? + ?????? TTL ??,????
|
||||
# /api/agents ?????
|
||||
try:
|
||||
from app.gateway.routers.agents import (
|
||||
_mark_legacy_sync_fresh,
|
||||
@@ -974,7 +974,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
await _sync_legacy_agents(agent_store)
|
||||
_mark_legacy_sync_fresh()
|
||||
# ``services.py`` keeps an independent TTL gate (used by
|
||||
# ``start_run`` → ``_enforce_agent_access``). Without this
|
||||
# ``start_run`` ? ``_enforce_agent_access``). Without this
|
||||
# ack, the first ``/runs/stream`` after startup would
|
||||
# re-run the heavy sync even though we just did it.
|
||||
_services_mark_sync_fresh()
|
||||
@@ -982,17 +982,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Legacy agent sync failed at startup (non-fatal)")
|
||||
|
||||
# 启动 skill curator 调度器:常驻后台任务,每小时检查一次需不需要进化
|
||||
# (per-user interval_hours 仍是真正决定要不要跑进化的闸)。
|
||||
# ?? skill curator ???:??????,?????????????
|
||||
# (per-user interval_hours ??????????????)?
|
||||
try:
|
||||
app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("Skill curator scheduler failed to start (non-fatal)")
|
||||
|
||||
# 启动 AI 写作会话定期清理:按 config.yaml 的 ai_writing.cleanup 段调度,
|
||||
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
|
||||
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
|
||||
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
|
||||
# ?? AI ????????:? config.yaml ? ai_writing.cleanup ???,
|
||||
# ?????? 3 ????,? retention_days(?? 7 ?)?????,??
|
||||
# ?? MySQL ??? + SQLite checkpointer ?? thread ???? worker
|
||||
# ?????? leader,?????? worker ??????
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
@@ -1031,7 +1031,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
yield
|
||||
|
||||
# 停止 curator 调度器
|
||||
# ?? curator ???
|
||||
curator_task = getattr(app.state, "curator_scheduler_task", None)
|
||||
if curator_task is not None:
|
||||
curator_task.cancel()
|
||||
@@ -1042,7 +1042,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler shutdown error")
|
||||
|
||||
# 停止 AI 写作清理调度器
|
||||
# ?? AI ???????
|
||||
cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
|
||||
if cleanup_task is not None:
|
||||
cleanup_task.cancel()
|
||||
@@ -1063,9 +1063,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler shutdown error")
|
||||
|
||||
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
|
||||
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
|
||||
# ``langgraph_runtime()`` 的 contextmanager 统一关闭,无需在这里特殊处理。
|
||||
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
|
||||
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
|
||||
# ``langgraph_runtime()`` ? contextmanager ???????????????
|
||||
|
||||
try:
|
||||
scheduler_service = getattr(app.state, "scheduled_task_service", None)
|
||||
@@ -1197,7 +1197,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# CORS
|
||||
if is_auth_disabled():
|
||||
logger.warning(
|
||||
"DEER_FLOW_AUTH_DISABLED is set — API authentication is bypassed. "
|
||||
"DEER_FLOW_AUTH_DISABLED is set ? API authentication is bypassed. "
|
||||
"Do not use this in production."
|
||||
)
|
||||
|
||||
@@ -1222,7 +1222,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
if _cors_allow_any_origin():
|
||||
logger.warning(
|
||||
"Permissive CORS enabled — any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
|
||||
"Permissive CORS enabled ? any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
|
||||
"DEER_FLOW_AUTH_DISABLED). Do not expose publicly."
|
||||
)
|
||||
app.add_middleware(
|
||||
@@ -1238,7 +1238,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
elif cors_origins:
|
||||
if "*" in cors_origins:
|
||||
logger.error(
|
||||
"GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True — invalid. "
|
||||
"GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True ? invalid. "
|
||||
"Set GATEWAY_CORS_ALLOW_ALL=1 instead, or list explicit origins."
|
||||
)
|
||||
cors_origins = [o for o in cors_origins if o != "*"]
|
||||
@@ -1319,10 +1319,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Roundtable-planning business chains at /api/roundtable-chains
|
||||
app.include_router(roundtable_chains.router)
|
||||
|
||||
# Global business → agent / chain mapping at /api/business-mapping
|
||||
# Global business ? agent / chain mapping at /api/business-mapping
|
||||
app.include_router(business_mapping.router)
|
||||
|
||||
# Light-app registry (轻应用中心) at /api/light-apps
|
||||
# Light-app registry (?????) at /api/light-apps
|
||||
app.include_router(light_apps.router)
|
||||
|
||||
# Standalone parallel multi-agent panel: fan-out streaming + saved tasks
|
||||
@@ -1400,7 +1400,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Public, unauthenticated read-only share snapshots (/api/public/shares/*).
|
||||
app.include_router(thread_shares.public_router)
|
||||
|
||||
# 圆桌会商草稿分享(import 复制副本 / view 只读结果报告页)
|
||||
# ?????????import ???? / view ????????
|
||||
app.include_router(roundtable_draft_shares.router)
|
||||
# Public, unauthenticated read-only roundtable result (/api/public/roundtable-shares/*).
|
||||
app.include_router(roundtable_draft_shares.public_router)
|
||||
@@ -1408,17 +1408,17 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Stateless Runs API (stream/wait without a pre-existing thread)
|
||||
app.include_router(runs.router)
|
||||
|
||||
# --- 圆桌规划三步流(前端 /page/workspace/roundtable/planning)---
|
||||
# 实现见 app/gateway/routers/{multi_agent,intent,recommend}.py;
|
||||
# 文档见 frontend-web/docs/multi-agent-backend-dev.md
|
||||
app.include_router(multi_agent.router) # Step 2:init(SSE) + run/stream(leader|special)
|
||||
app.include_router(intent.router) # Step 1:/api/intent/init + /stream
|
||||
app.include_router(recommend.router) # Step 1→2:/api/recommend/stream(无 init)
|
||||
# Step 3 产物列表/下载复用下方 artifacts.router(/api/threads/{id}/artifacts)
|
||||
# --- ?????????? /page/workspace/roundtable/planning?---
|
||||
# ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
|
||||
# ??? frontend-web/docs/multi-agent-backend-dev.md
|
||||
app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
|
||||
app.include_router(intent.router) # Step 1?/api/intent/init + /stream
|
||||
app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
|
||||
# Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
|
||||
|
||||
# 探活端点:无需鉴权(见 auth_middleware._PUBLIC_PATH_PREFIXES)。
|
||||
# 同时暴露裸 ``/health`` 与带 API 前缀的 ``/api/health``,方便监控按
|
||||
# 任意 base URL 直接探活,不会被认证中间件拦成 401。
|
||||
# ?????????(? auth_middleware._PUBLIC_PATH_PREFIXES)?
|
||||
# ????? ``/health`` ?? API ??? ``/api/health``,?????
|
||||
# ?? base URL ????,?????????? 401?
|
||||
@app.get("/health", tags=["health"])
|
||||
@app.get("/api/health", tags=["health"])
|
||||
async def health_check() -> dict:
|
||||
|
||||
@@ -31,21 +31,43 @@ from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
|
||||
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.gateway.auth.models import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P = ParamSpec("P")
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
|
||||
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
|
||||
|
||||
A checkpoint with no matching ``threads_meta`` row is the signature of a
|
||||
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
|
||||
survives in the checkpointer while the business row is gone. Normal thread
|
||||
deletion removes the checkpoint too, so this can never resurrect a
|
||||
deliberately-deleted thread.
|
||||
"""
|
||||
try:
|
||||
from app.gateway.deps import get_checkpointer
|
||||
|
||||
checkpointer = get_checkpointer(request)
|
||||
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
return tup is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Permission constants
|
||||
class Permissions:
|
||||
"""Permission constants for resource:action format."""
|
||||
@@ -205,6 +227,7 @@ def require_permission(
|
||||
action: str,
|
||||
owner_check: bool = False,
|
||||
require_existing: bool = False,
|
||||
claim_orphan: bool = False,
|
||||
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
||||
"""Decorator that checks permission for resource:action.
|
||||
|
||||
@@ -221,6 +244,15 @@ def require_permission(
|
||||
**destructive / mutating** routes (DELETE, PATCH,
|
||||
state-update) so a deleted thread can't be re-targeted
|
||||
by another user via the missing-row code path.
|
||||
claim_orphan: Only meaningful with ``owner_check=True`` and
|
||||
``require_existing=True``. When the ``threads_meta``
|
||||
row is missing **and** a LangGraph checkpoint exists
|
||||
for the thread (a DB-switch orphan), adopt it for the
|
||||
current user instead of 404ing. Use on **run-create**
|
||||
routes so a conversation whose business row was lost
|
||||
(sqlite↔mysql swap) stays answerable. Safe: normal
|
||||
deletion removes the checkpoint, so this never revives
|
||||
a deliberately-deleted thread.
|
||||
|
||||
Usage:
|
||||
# Read-style: legacy untracked threads are allowed
|
||||
@@ -289,16 +321,44 @@ def require_permission(
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
thread_store = get_thread_store(request)
|
||||
user_id = str(auth.user.id)
|
||||
allowed = await thread_store.check_access(
|
||||
thread_id,
|
||||
str(auth.user.id),
|
||||
user_id,
|
||||
require_existing=require_existing,
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not found",
|
||||
)
|
||||
# Tell apart three denial causes so the client gets an
|
||||
# accurate message and so DB-switch orphans can be adopted
|
||||
# rather than hard-404'd:
|
||||
# 1. row exists but owned by another user -> not accessible
|
||||
# 2. no row, no checkpoint -> truly missing
|
||||
# 3. no row + checkpoint + claim_orphan -> orphan, claim
|
||||
existing = await thread_store.get(thread_id, user_id=None)
|
||||
if existing is not None:
|
||||
# Distinct detail string lets the frontend show "无权访问"
|
||||
# instead of "不存在" for a cross-user / stale-session hit.
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not accessible",
|
||||
)
|
||||
claimed = False
|
||||
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
|
||||
try:
|
||||
await thread_store.create(thread_id, user_id=user_id)
|
||||
logger.info(
|
||||
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
|
||||
thread_id,
|
||||
user_id,
|
||||
)
|
||||
claimed = True
|
||||
except Exception:
|
||||
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
|
||||
if not claimed:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not found",
|
||||
)
|
||||
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -59,6 +59,11 @@ class AgentResponse(BaseModel):
|
||||
description="Recommended questions shown on the agent chat home / sidebar (常用问题). Each item is {title, prompt}; legacy string lists are coerced.",
|
||||
)
|
||||
|
||||
template_name: str | None = Field(
|
||||
default=None,
|
||||
description="入库模板名 (external knowledge-ingest templateName). When set, markdown artifacts produced in this agent's chats can be published to the external knowledge-ingest API.",
|
||||
)
|
||||
|
||||
tags: list[dict] = Field(default_factory=list, description="Tags assigned to this agent")
|
||||
|
||||
featured_order: int | None = Field(
|
||||
@@ -109,6 +114,7 @@ class AgentCreateRequest(BaseModel):
|
||||
model: str | None = Field(default=None, description="Optional model override persisted to config.yaml")
|
||||
tool_groups: list[str] | None = Field(default=None, description="Optional tool group whitelist persisted to config.yaml")
|
||||
common_questions: list[CommonQuestion] | None = Field(default=None, description="Recommended questions (常用问题) persisted to config.yaml. Each item is {title, prompt}; legacy strings are coerced.")
|
||||
template_name: str | None = Field(default=None, description="入库模板名 persisted to config.yaml. Enables publishing markdown artifacts to the external knowledge-ingest API.")
|
||||
|
||||
|
||||
class AgentUpdateRequest(BaseModel):
|
||||
@@ -124,6 +130,7 @@ class AgentUpdateRequest(BaseModel):
|
||||
model: str | None = Field(default=None, description="Updated model override (config.yaml)")
|
||||
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
|
||||
common_questions: list[CommonQuestion] | None = Field(default=None, description="Updated recommended questions (常用问题, config.yaml). Each item is {title, prompt}; legacy strings are coerced.")
|
||||
template_name: str | None = Field(default=None, description="Updated 入库模板名 (config.yaml). Empty string clears it.")
|
||||
|
||||
|
||||
class AgentSelectorResponse(BaseModel):
|
||||
@@ -228,6 +235,7 @@ def _write_agent_config(
|
||||
model: str | None = None,
|
||||
tool_groups: list[str] | None = None,
|
||||
common_questions: list[CommonQuestion] | None = None,
|
||||
template_name: str | None = None,
|
||||
) -> None:
|
||||
agent_dir = get_paths().agent_dir(agent_id)
|
||||
agent_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -254,13 +262,15 @@ def _write_agent_config(
|
||||
config_data["common_questions"] = [
|
||||
q.model_dump() for q in normalized if q.prompt.strip()
|
||||
]
|
||||
if template_name is not None and template_name.strip():
|
||||
config_data["template_name"] = template_name.strip()
|
||||
with open(config_file, "w", encoding="utf-8") as f:
|
||||
yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
|
||||
def _load_config_extras(agent_id: str) -> dict[str, Any]:
|
||||
"""Read skills/model/tool_groups/common_questions from the agent's config.yaml, tolerant of missing files."""
|
||||
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None, "template_name": None}
|
||||
try:
|
||||
cfg = load_agent_config(agent_id)
|
||||
except (FileNotFoundError, ValueError):
|
||||
@@ -272,6 +282,7 @@ def _load_config_extras(agent_id: str) -> dict[str, Any]:
|
||||
"model": cfg.model,
|
||||
"tool_groups": cfg.tool_groups,
|
||||
"common_questions": cfg.common_questions,
|
||||
"template_name": cfg.template_name,
|
||||
}
|
||||
|
||||
|
||||
@@ -432,7 +443,7 @@ def _record_to_response(
|
||||
can happen lazily when the agent is actually opened.
|
||||
"""
|
||||
soul: str | None = None
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None, "template_name": None}
|
||||
if include_extras:
|
||||
_ensure_agent_dir_from_record(record)
|
||||
extras = _load_config_extras(record["id"])
|
||||
@@ -455,6 +466,7 @@ def _record_to_response(
|
||||
tool_groups=extras["tool_groups"],
|
||||
skills=extras["skills"],
|
||||
common_questions=extras.get("common_questions"),
|
||||
template_name=extras.get("template_name"),
|
||||
featured_order=record.get("featured_order"),
|
||||
)
|
||||
|
||||
@@ -723,6 +735,10 @@ async def get_agent(agent_id: str, request: Request) -> AgentResponse:
|
||||
await _maybe_sync_legacy_agents(store)
|
||||
user_id = _current_user_id(request)
|
||||
record = await store.get_visible(agent_id, user_id)
|
||||
if record is None and _is_admin_user(request):
|
||||
# The admin console lists every agent (?admin=true), so an admin
|
||||
# must be able to open any of them — mirror that here.
|
||||
record = await store.get_any(agent_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found")
|
||||
# Detail view legitimately reads SOUL.md + config.yaml — hop to a
|
||||
@@ -773,6 +789,7 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
|
||||
model=request.model,
|
||||
tool_groups=request.tool_groups,
|
||||
common_questions=request.common_questions,
|
||||
template_name=request.template_name,
|
||||
)
|
||||
_write_agent_soul(agent_id, request.soul)
|
||||
|
||||
@@ -852,8 +869,9 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
next_model = request.model if "model" in fields_set else existing_extras["model"]
|
||||
next_tool_groups = request.tool_groups if "tool_groups" in fields_set else existing_extras["tool_groups"]
|
||||
next_common_questions = request.common_questions if "common_questions" in fields_set else existing_extras["common_questions"]
|
||||
next_template_name = request.template_name if "template_name" in fields_set else existing_extras["template_name"]
|
||||
|
||||
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions"))
|
||||
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions", "template_name"))
|
||||
if config_dirty:
|
||||
_write_agent_config(
|
||||
agent_id,
|
||||
@@ -863,6 +881,7 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
model=next_model,
|
||||
tool_groups=next_tool_groups,
|
||||
common_questions=next_common_questions,
|
||||
template_name=next_template_name,
|
||||
)
|
||||
# Keep the listing cache in lock-step with config.yaml, but only
|
||||
# when the cached fields were actually touched — a partial update
|
||||
|
||||
@@ -424,6 +424,7 @@ async def list_notes(
|
||||
request: Request,
|
||||
keyword: str | None = None,
|
||||
tag: str | None = None,
|
||||
entity: str | None = None,
|
||||
source_type: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
@@ -432,7 +433,7 @@ async def list_notes(
|
||||
service = get_knowledge_service(request)
|
||||
limit = max(1, min(limit, 100))
|
||||
offset = max(0, offset)
|
||||
return await service.list_notes(keyword=keyword, tag=tag, source_type=source_type, status=status, limit=limit, offset=offset)
|
||||
return await service.list_notes(keyword=keyword, tag=tag, entity=entity, source_type=source_type, status=status, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@router.get("/notes/{note_id}")
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""外部知识入库接口(发布 Markdown 产物到 knowes 知识库)。
|
||||
|
||||
智能体 config.yaml 配置了 ``template_name`` 后,该智能体会话中生成的 Markdown
|
||||
产物可以通过 ``POST /api/knowledge-ingest/publish`` 一键入库。
|
||||
|
||||
入库目标接口的字段名不固定:标题字段可能叫「名称」也可能叫「标题」,因此先按
|
||||
「名称」发送一次,若判定失败则用 **同一个 nodeName** 把标题字段换成「标题」再
|
||||
发一次。nodeName 是入库记录的唯一标识,取「md 标题 + 时间戳」。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/knowledge-ingest", tags=["knowledge-ingest"])
|
||||
|
||||
# 入库接口地址,可用环境变量覆盖(如切换 UAT/生产)。
|
||||
_REQUEST_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
# 标题字段在目标模板里可能叫「名称」或「标题」,按此顺序逐个尝试。
|
||||
_TITLE_FIELD_CANDIDATES = ("名称", "标题")
|
||||
_CONTENT_FIELD = "内容"
|
||||
|
||||
|
||||
class KnowledgeIngestPublishRequest(BaseModel):
|
||||
"""Request body for publishing one markdown artifact."""
|
||||
|
||||
agent_id: str = Field(..., description="智能体 id,用于读取其 config.yaml 中的 template_name")
|
||||
title: str = Field(..., description="md 标题(一级标题或文件名)")
|
||||
content: str = Field(..., description="md 正文全文")
|
||||
|
||||
|
||||
class KnowledgeIngestAttempt(BaseModel):
|
||||
"""One delivery attempt against the ingest API."""
|
||||
|
||||
title_field: str = Field(..., description="本次尝试使用的标题字段名(名称/标题)")
|
||||
status_code: int | None = Field(default=None, description="HTTP 状态码;请求异常时为 null")
|
||||
response_text: str = Field(default="", description="入库接口原始响应(截断到 2000 字符)")
|
||||
success: bool = False
|
||||
error: str | None = Field(default=None, description="请求层异常信息(网络/超时等)")
|
||||
|
||||
|
||||
class KnowledgeIngestPublishResponse(BaseModel):
|
||||
success: bool
|
||||
template_name: str
|
||||
node_name: str = Field(..., description="入库唯一标识(md 标题 + 时间戳),两次尝试共用")
|
||||
batch_number: str
|
||||
attempts: list[KnowledgeIngestAttempt]
|
||||
|
||||
|
||||
def _judge_success(status_code: int, response_text: str) -> bool:
|
||||
"""判断入库接口响应是否表示成功。
|
||||
|
||||
接口未提供正式契约:优先看响应 JSON 中常见的 success/code/status/state
|
||||
字段;JSON 不可解析或没有这类字段时,退化为按 HTTP 2xx 判定。
|
||||
"""
|
||||
if not (200 <= status_code < 300):
|
||||
return False
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except (ValueError, TypeError):
|
||||
return True
|
||||
if not isinstance(data, dict):
|
||||
return True
|
||||
if isinstance(data.get("success"), bool):
|
||||
return data["success"]
|
||||
for key in ("code", "status", "state"):
|
||||
if key in data:
|
||||
return str(data[key]).lower() in {"0", "200", "success", "ok", "true"}
|
||||
return True
|
||||
|
||||
|
||||
def _build_payload(template_name: str, batch_number: str, node_name: str, title_field: str, title: str, content: str) -> dict[str, Any]:
|
||||
return {
|
||||
"templateName": template_name,
|
||||
"batchNumber": batch_number,
|
||||
"list": [
|
||||
{
|
||||
"rec": {"nodeName": node_name},
|
||||
"data": [
|
||||
{"fieldName": title_field, "value": title},
|
||||
{"fieldName": _CONTENT_FIELD, "value": content},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_node_name(title: str, timestamp: str) -> str:
|
||||
"""Build a stable node name as md5(title) + timestamp + .md."""
|
||||
title_md5 = hashlib.md5(title.encode("utf-8")).hexdigest()
|
||||
return f"{title_md5}-{timestamp}.md"
|
||||
|
||||
|
||||
def _resolve_ingest_url() -> str:
|
||||
"""Resolve the external ingest URL from config.yaml, with env fallback."""
|
||||
config_url = getattr(get_app_config().knowledge_ingest, "url", "").strip()
|
||||
return os.environ.get(
|
||||
"KNOWES_INGEST_URL",
|
||||
config_url or "https://ch1.b.uat.4.cn/knowes/api/knowledge-outer/add",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/publish", response_model=KnowledgeIngestPublishResponse, summary="Publish markdown artifact to external knowledge base")
|
||||
async def publish_to_knowledge_base(request: KnowledgeIngestPublishRequest) -> KnowledgeIngestPublishResponse:
|
||||
title = request.title.strip()
|
||||
content = request.content.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=422, detail="标题不能为空")
|
||||
if not content:
|
||||
raise HTTPException(status_code=422, detail="内容为空,无法入库")
|
||||
|
||||
try:
|
||||
agent_cfg = load_agent_config(request.agent_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
template_name = (agent_cfg.template_name or "").strip() if agent_cfg else ""
|
||||
if not template_name:
|
||||
raise HTTPException(status_code=400, detail=f"智能体 '{request.agent_id}' 未配置入库模板名(template_name),无法发布")
|
||||
|
||||
now = datetime.now()
|
||||
# nodeName 是入库记录的唯一标识:md 标题 + 时间戳。两次尝试必须共用同一个
|
||||
# nodeName,避免第一次实际已入库时第二次产生重复记录。
|
||||
timestamp = now.strftime("%Y%m%d%H%M%S")
|
||||
node_name = _build_node_name(title, timestamp)
|
||||
batch_number = f"md{now.strftime('%Y%m%d')}"
|
||||
ingest_url = _resolve_ingest_url()
|
||||
|
||||
logger.info(
|
||||
"[knowledge-ingest] 开始入库: agent=%s template=%s nodeName=%s batchNumber=%s title=%r content_len=%d url=%s",
|
||||
request.agent_id,
|
||||
template_name,
|
||||
node_name,
|
||||
batch_number,
|
||||
title,
|
||||
len(content),
|
||||
ingest_url,
|
||||
)
|
||||
|
||||
attempts: list[KnowledgeIngestAttempt] = []
|
||||
headers = {"Content-Type": "application/json", "User-Agent": "DeerFlow/1.0"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=_REQUEST_TIMEOUT_SECONDS) as client:
|
||||
for title_field in _TITLE_FIELD_CANDIDATES:
|
||||
payload = _build_payload(template_name, batch_number, node_name, title_field, title, content)
|
||||
logger.info(
|
||||
"[knowledge-ingest] 第 %d 次发送 (标题字段=%s): %s",
|
||||
len(attempts) + 1,
|
||||
title_field,
|
||||
json.dumps({**payload, "list": [{**payload["list"][0], "data": [{"fieldName": d["fieldName"], "value": d["value"][:200]} for d in payload["list"][0]["data"]]}]}, ensure_ascii=False),
|
||||
)
|
||||
try:
|
||||
response = await client.post(ingest_url, json=payload, headers=headers)
|
||||
except httpx.HTTPError as exc:
|
||||
logger.error("[knowledge-ingest] 第 %d 次发送请求异常 (标题字段=%s): %s", len(attempts) + 1, title_field, exc)
|
||||
attempts.append(KnowledgeIngestAttempt(title_field=title_field, error=str(exc)))
|
||||
continue
|
||||
|
||||
text = response.text or ""
|
||||
success = _judge_success(response.status_code, text)
|
||||
logger.info(
|
||||
"[knowledge-ingest] 第 %d 次响应 (标题字段=%s): status=%d success=%s body=%s",
|
||||
len(attempts) + 1,
|
||||
title_field,
|
||||
response.status_code,
|
||||
success,
|
||||
text[:2000],
|
||||
)
|
||||
attempts.append(
|
||||
KnowledgeIngestAttempt(
|
||||
title_field=title_field,
|
||||
status_code=response.status_code,
|
||||
response_text=text[:2000],
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
if success:
|
||||
break
|
||||
logger.warning("[knowledge-ingest] 第 %d 次入库判定失败 (标题字段=%s),准备换字段重试", len(attempts), title_field)
|
||||
|
||||
overall = any(a.success for a in attempts)
|
||||
if overall:
|
||||
logger.info("[knowledge-ingest] 入库成功: nodeName=%s 使用标题字段=%s", node_name, next(a.title_field for a in attempts if a.success))
|
||||
else:
|
||||
logger.error("[knowledge-ingest] 入库失败(全部 %d 次尝试均未成功): nodeName=%s", len(attempts), node_name)
|
||||
|
||||
return KnowledgeIngestPublishResponse(
|
||||
success=overall,
|
||||
template_name=template_name,
|
||||
node_name=node_name,
|
||||
batch_number=batch_number,
|
||||
attempts=attempts,
|
||||
)
|
||||
@@ -21,7 +21,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer, get_current_user, get_feedback_repo, get_run_event_store, get_run_manager, get_run_store, get_stream_bridge
|
||||
from app.gateway.services import sse_consumer, start_run
|
||||
from app.gateway.services import immediate_end_sse, sse_consumer, start_run
|
||||
from deerflow.runtime import RunRecord, serialize_channel_values
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -96,7 +96,7 @@ def _record_to_response(record: RunRecord) -> RunResponse:
|
||||
|
||||
|
||||
@router.post("/{thread_id}/runs", response_model=RunResponse)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
|
||||
async def create_run(thread_id: str, body: RunCreateRequest, request: Request) -> RunResponse:
|
||||
"""Create a background run (returns immediately)."""
|
||||
record = await start_run(body, thread_id, request)
|
||||
@@ -104,7 +104,7 @@ async def create_run(thread_id: str, body: RunCreateRequest, request: Request) -
|
||||
|
||||
|
||||
@router.post("/{thread_id}/runs/stream")
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
|
||||
async def stream_run(thread_id: str, body: RunCreateRequest, request: Request) -> StreamingResponse:
|
||||
"""Create a run and stream events via SSE.
|
||||
|
||||
@@ -132,7 +132,7 @@ async def stream_run(thread_id: str, body: RunCreateRequest, request: Request) -
|
||||
|
||||
|
||||
@router.post("/{thread_id}/runs/wait", response_model=dict)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True)
|
||||
@require_permission("runs", "create", owner_check=True, require_existing=True, claim_orphan=True)
|
||||
async def wait_run(thread_id: str, body: RunCreateRequest, request: Request) -> dict:
|
||||
"""Create a run and block until it completes, returning the final state."""
|
||||
record = await start_run(body, thread_id, request)
|
||||
@@ -173,7 +173,11 @@ async def get_run(thread_id: str, run_id: str, request: Request) -> RunResponse:
|
||||
run_mgr = get_run_manager(request)
|
||||
record = run_mgr.get(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
# In-memory miss: fall back to the persistent store so a run still
|
||||
# resolves after a Gateway restart instead of 404ing.
|
||||
record = await run_mgr.get_persisted(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
return _record_to_response(record)
|
||||
|
||||
|
||||
@@ -222,11 +226,27 @@ async def join_run(thread_id: str, run_id: str, request: Request) -> StreamingRe
|
||||
bridge = get_stream_bridge(request)
|
||||
run_mgr = get_run_manager(request)
|
||||
record = run_mgr.get(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
if record is not None and record.thread_id == thread_id:
|
||||
return StreamingResponse(
|
||||
sse_consumer(bridge, record, request, run_mgr),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
# Not in memory: a run that still exists in the store but has no live task
|
||||
# means the Gateway restarted (or the finished run was cleaned up). There
|
||||
# is nothing to reconnect to, so stream a clean terminal end instead of a
|
||||
# 404 — the SDK then clears its reconnect key and renders history.
|
||||
persisted = await run_mgr.get_persisted(run_id)
|
||||
if persisted is None or persisted.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
|
||||
return StreamingResponse(
|
||||
sse_consumer(bridge, record, request, run_mgr),
|
||||
immediate_end_sse(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
@@ -255,7 +275,22 @@ async def stream_existing_run(
|
||||
run_mgr = get_run_manager(request)
|
||||
record = run_mgr.get(run_id)
|
||||
if record is None or record.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
# In-memory miss. If the run survives in the store it's a dead record
|
||||
# from before a restart (or already finished) — nothing to join or
|
||||
# cancel, so emit a clean terminal end so the SDK stops reconnecting
|
||||
# instead of 404ing. A truly unknown run still 404s.
|
||||
persisted = await run_mgr.get_persisted(run_id)
|
||||
if persisted is None or persisted.thread_id != thread_id:
|
||||
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
|
||||
return StreamingResponse(
|
||||
immediate_end_sse(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
# Cancel if an action was requested (stop-button / interrupt flow)
|
||||
if action is not None:
|
||||
|
||||
@@ -42,6 +42,20 @@ logger = logging.getLogger(__name__)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def immediate_end_sse():
|
||||
"""Yield a single terminal ``end`` SSE frame and stop.
|
||||
|
||||
Used when a run exists in the persistent store but not in the in-memory
|
||||
registry — typically after a Gateway restart, or once a finished run's
|
||||
record has been cleaned up. There is no live stream to join, so instead of
|
||||
returning 404 (which the SDK's ``joinStream`` surfaces as a hard error and
|
||||
never clears its reconnect key for) we stream a clean end. The SDK treats
|
||||
this as success, clears ``lg:stream:{threadId}``, and the conversation
|
||||
falls back to history rendering — killing the repeating switch-back 404.
|
||||
"""
|
||||
yield format_sse("end", None)
|
||||
|
||||
|
||||
def format_sse(event: str, data: Any, *, event_id: str | None = None) -> str:
|
||||
"""Format a single SSE frame.
|
||||
|
||||
@@ -477,6 +491,9 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
|
||||
store = get_agent_store(request)
|
||||
await _maybe_sync_legacy_agent_records(store)
|
||||
user_id = get_effective_user_id()
|
||||
# Admins see every agent in the admin console listing (?admin=true), so
|
||||
# they may also start a run against any of them.
|
||||
is_admin = getattr(getattr(request.state, "user", None), "system_role", None) == "admin"
|
||||
for agent_id in agent_ids:
|
||||
try:
|
||||
normalized = validate_agent_id(agent_id)
|
||||
@@ -484,8 +501,11 @@ async def _enforce_agent_access(body: Any, request: Request) -> None:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
if normalized is None:
|
||||
continue
|
||||
if await store.get_visible(normalized, user_id) is None:
|
||||
raise HTTPException(status_code=404, detail=f"Agent {normalized} not found")
|
||||
if await store.get_visible(normalized, user_id) is not None:
|
||||
continue
|
||||
if is_admin and await store.get_any(normalized) is not None:
|
||||
continue
|
||||
raise HTTPException(status_code=404, detail=f"Agent {normalized} not found")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in new issue
Block a user