sync snapshot from LAN repo (source codex/merge-tjlnew-classic-retrieval-20261019 @ ded012d9)

This commit is contained in:
gozeros committed 2026-11-05 11:04:44 +08:00
1 parent ea9dcff9c0
commit c753e6c9b5
818 files changed
+14037 -108991

No files matched your search

@@ -1,8 +1,7 @@
description: AI 写作流程的素材收集专家,负责为写作任务生成检索关键词、用配置的技能编排检索、并对检索回来的素材按相关度排序筛选
description: AI 写作流程的素材收集专家,负责为写作任务生成检索关键词、从全部启用技能中自由编排检索、并对检索回来的素材按相关度排序筛选
id: ai-writing-researcher
name: 素材收集专家
# 通用检索使用的技能白名单(按顺序编排)。技能目录动辄数百个,全量列进
# 选择提示词会把模型上下文撑爆,因此候选池限定在这里配置的技能;内置
# 「知识库检索」始终保底。admin 可在智能体管理页替换或追加。
# 通用检索不再使用这里的 skills 白名单:素材收集从全部启用技能里由模型按
# 写作意图自由挑选(内置「知识库检索」始终保底)。此字段仅保留兼容旧版。
skills:
- knowledge-base-search
@@ -124,35 +124,23 @@ _SEED_FILES: tuple[str, ...] = ("config.yaml", "SOUL.md")
_ACTION_PLAN_SKILL = "position-roundtable-action-create"
def _ensure_action_plan_contract(agent_dir: Path) -> bool:
"""Refresh the action-type contract and retain the batch-create skill binding.
def _ensure_action_plan_skill(agent_dir: Path) -> bool:
"""Merge the owned batch-create Skill into an existing action planner.
``actionType`` changed from a task-system enum to a model-derived name. An
already seeded planner must receive this contract update; otherwise its
persisted SOUL would continue to require frontend ``dictValue`` input after
a deployment. Other optional skills on a deployed agent are preserved.
The planner SOUL remains editable in the agent configuration drawer and is
therefore never overwritten here. The skill binding, however, is a runtime
integration contract: without it an already-deployed planner cannot read
the new SKILL.md or invoke the batch API script after a code upgrade.
"""
changed = False
source_soul = _ASSETS_DIR / ACTION_PLAN_AGENT_ID / "SOUL.md"
target_soul = agent_dir / "SOUL.md"
try:
source_bytes = source_soul.read_bytes()
if not target_soul.exists() or target_soul.read_bytes() != source_bytes:
target_soul.write_bytes(source_bytes)
changed = True
except OSError:
logger.warning("Unable to refresh action planner contract: %s", target_soul)
return changed
config_path = agent_dir / "config.yaml"
try:
config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if config_path.exists() else {}
except (OSError, yaml.YAMLError):
logger.warning("Unable to read action planner config for skill binding: %s", config_path)
return changed
return False
if not isinstance(config, dict):
logger.warning("Action planner config is not a mapping; skip skill binding: %s", config_path)
return changed
return False
skills = config.get("skills")
if skills is None:
normalized: list[str] = []
@@ -160,9 +148,9 @@ def _ensure_action_plan_contract(agent_dir: Path) -> bool:
normalized = [str(name).strip() for name in skills if str(name).strip()]
else:
logger.warning("Action planner skills is not a list; skip skill binding: %s", config_path)
return changed
return False
if _ACTION_PLAN_SKILL in normalized:
return changed
return False
config["skills"] = [*normalized, _ACTION_PLAN_SKILL]
try:
@@ -172,7 +160,7 @@ def _ensure_action_plan_contract(agent_dir: Path) -> bool:
)
except OSError:
logger.warning("Unable to write action planner skill binding: %s", config_path)
return changed
return False
return True
@@ -180,11 +168,10 @@ def _ensure_one(agent_id: str) -> bool:
"""Copy the verified SOUL.md + config.yaml for one functional agent into
``.deer-flow/agents/<id>/`` if that directory is missing.
The position-roundtable intent agent and action planner are implementation-
owned compatibility contracts: unlike user-managed agents, their required
prompts must stay aligned with the position workspace. Their affected seed
files are therefore refreshed when they differ, so a previously seeded
incorrect prompt is repaired after deployment.
The position-roundtable intent agent is an implementation-owned compatibility
contract: unlike user-managed agents, its schema must stay aligned with the
position workspace. Its two seed files are therefore refreshed when they
differ, so a previously seeded incorrect prompt is repaired after deployment.
Returns True if files were created or refreshed, False if no work was needed
or the seed assets are missing.
@@ -203,7 +190,7 @@ def _ensure_one(agent_id: str) -> bool:
agent_dir = get_paths().agent_dir(agent_id)
if agent_dir.exists():
if agent_id == ACTION_PLAN_AGENT_ID:
return _ensure_action_plan_contract(agent_dir)
return _ensure_action_plan_skill(agent_dir)
if agent_id not in {POSITION_INTENT_AGENT_ID, REPORT_COLLABORATION_INTENT_AGENT_ID}:
# Other built-in agents may have been tuned on a deployed machine;
# preserve the long-standing no-overwrite behavior for them.
@@ -2,7 +2,7 @@
## 核心定位
你是岗位多智能体会商流程的最后一个内置智能体。上游已经完成意图识别和各岗位 Markdown 交付,可能另有方案总结。你的职责是把这些已确认材料拆解成能执行、能验收、能跟踪的行动规划;方案总结存在时应综合使用,不存在时直接依据任务信息、已确认意图和岗位交付完成规划。你不重新调研、不联网、不派发子智能体。
你是岗位多智能体会商流程的最后一个内置智能体。上游已经完成意图识别、各岗位 Markdown 交付和方案总结。你的职责是把这些已确认材料拆解成能执行、能验收、能跟踪的行动规划;你不重新调研、不联网、不派发子智能体。
## 产出规则
@@ -17,7 +17,7 @@
- 先说明规划目标、关键依据和执行边界。
- 按优先级和依赖顺序给出阶段安排、关键动作、责任岗位建议、里程碑、风险控制和验收方式。
- 每项行动必须能追溯到方案总结(如有)或至少一个岗位有效交付;没有依据时不要假装是事实,可明确写为建议性安排。
- 每项行动必须能追溯到方案总结或至少一个岗位有效交付;没有依据时不要假装是事实,可明确写为建议性安排。
- 用户提出修改时,只调整受影响部分,并同时重写两个最终文件,使 Markdown 与 JSON 始终一致。
## JSON 子任务契约
@@ -28,7 +28,7 @@
{
"actionName": "行动名称",
"actionDescription": "行动描述",
"actionType": "行动类型名称",
"actionType": 0,
"publishChannel": 0,
"infoFormat": 0,
"responsiblePosition": 0,
@@ -38,17 +38,17 @@
}
```
`actionType` 必须由你根据方案总结、任务意图和岗位交付自行分析归纳,填写简洁、明确的中文类型名称(例如“风险研判”“内容发布”“协同协调”);不要从前端字典选择它,也不要填写 `dictValue` 或数字编码。`publishChannel`、`infoFormat`、`responsiblePosition` 必须从隐藏上下文传入的对应字典选择,并将该项 `dictValue` 写为整数。`taskId` 必须等于隐藏上下文中前端传入的任务 ID。不要输出注释、尾逗号、旧的 `id/title/goal/basis` 等字段,不能用 0 或空值代替已提供的字典项。
`actionType`、`publishChannel`、`infoFormat`、`responsiblePosition` 必须从隐藏上下文传入的对应字典选择,并将该项 `dictValue` 写为整数。`taskId` 必须等于隐藏上下文中前端传入的任务 ID。不要输出注释、尾逗号、旧的 `id/title/goal/basis` 等字段,不能用 0 或空值代替未选择的字典项。
## 行动批量新增
首次生成完整行动清单后:仅当隐藏上下文的 `taskId` 是正整数且三类字典(发布渠道、信息形式、责任岗位)均完整时,必须先用 `read_file` 读取 `position-roundtable-action-create` Skill 的 `SKILL.md` 全文,再按其说明调用脚本批量新增行动。若 taskId 为本地演示标识/为空,或字典接口未就绪,只生成报告与 JSON;`actionType` 仍须填写分析出的中文名称,未就绪的三个枚举字段可暂填 `0` 占位,并明确说明未调用外部任务系统。不得逐条调用旧的 `/task/action/add`,也不得自行用 curl 或 requests 替代脚本。
首次生成完整行动清单后:仅当隐藏上下文的 `taskId` 是正整数且四类字典(发布渠道、行动类型、信息形式、责任岗位)均完整时,必须先用 `read_file` 读取 `position-roundtable-action-create` Skill 的 `SKILL.md` 全文,再按其说明调用脚本批量新增行动。若 taskId 为本地演示标识/为空,或字典接口未就绪,只生成报告与 JSON;枚举字段可暂填 `0` 占位,并明确说明未调用外部任务系统。不得逐条调用旧的 `/task/action/add`,也不得自行用 curl 或 requests 替代脚本。
脚本成功时才可以说明行动已入库;脚本失败时如实报告失败原因。相同任务、相同行动清单会由脚本自动幂等跳过。已经成功新增后,如用户要求修改行动,先重写 Markdown 和 JSON;因为当前没有删除/更新行动接口,只有用户明确说“确认新增行动”或“重新批量新增”时才可以按 Skill 使用 `--force` 再次新增,并需提示重复风险。
## 边界
- 只依据用户消息中提供的任务信息、意图、方案总结(如有)和岗位交付。
- 只依据用户消息中提供的任务信息、意图、方案总结和岗位交付。
- 不调用联网、派活、澄清、展示文件等工具;仅可为行动批量新增按专属 Skill 使用 `read_file` 与 `bash`。
- 不删除或修改上游岗位产物。
- 不在报告中添加“AI 生成”类水印。
@@ -10,8 +10,6 @@
4. 按统一量表独立评分,再给出横向比较和推荐组合。
5. 只使用输入中的 `agentId`,不得改名或虚构不存在的候选项。
本阶段没有用户中途回答入口。不得调用 `ask_clarification`,也不得在正文中向用户提问、索要材料、要求确认或要求重试。若证据不足,采用保守假设、降低对应维度评分,并将缺口写入“数据限制与待核验”,仍须一次性完成所有候选项的评判。
## 百分制量表
- 任务匹配度:25
@@ -26,7 +24,7 @@
## 输出要求
先用简洁 Markdown 给出总体评价、主要差异、推荐组合和需要人工确认的冲突,不要重复复述同一结论或输出多个评判版本。随后在回复末尾输出:
先用简洁 Markdown 给出总体评价、主要差异、推荐组合和需要人工确认的冲突。随后在回复末尾输出:
````markdown
[JUDGEMENT_READY]
@@ -42,18 +42,6 @@ _DEFAULT_LOOKBACK_HOURS = {"minute": 3, "hour": 48}
# assistant_id values that are not real custom-agent ids (the default lead agent).
_DEFAULT_ASSISTANT_IDS = {"", "default", "lead_agent", "agent"}
_DEFAULT_AGENT_NAME = "默认助手"
_UNKNOWN_USER_NAME = "未知用户"
def _display_agent_name(assistant_id: str, agent_names: dict[str, str]) -> str:
if assistant_id in _DEFAULT_ASSISTANT_IDS:
return _DEFAULT_AGENT_NAME
return agent_names.get(assistant_id) or assistant_id
def _display_user_name(user_id: str, user_emails: dict[str, str]) -> str:
return user_emails.get(user_id) or user_id or _UNKNOWN_USER_NAME
class ActiveRunItem(BaseModel):
@@ -256,7 +244,10 @@ async def list_active_runs(request: Request) -> ActiveRunsResponse:
user_id = r.user_id or (meta_row.user_id if meta_row else "") or ""
metadata = (meta_row.metadata_json if meta_row else None) or {}
assistant_id = r.assistant_id or ""
agent_name = _display_agent_name(assistant_id, agent_names)
if assistant_id in _DEFAULT_ASSISTANT_IDS:
agent_name = "榛樿鍔╂墜"
else:
agent_name = agent_names.get(assistant_id) or assistant_id
items.append(
ActiveRunItem(
run_id=r.run_id,
@@ -265,7 +256,7 @@ async def list_active_runs(request: Request) -> ActiveRunsResponse:
created_at=datetime.fromtimestamp(r.created_at, tz=UTC).isoformat() if r.created_at else "",
elapsed_seconds=_elapsed_seconds_from_epoch(r.created_at, now),
user_id=user_id,
user_name=_display_user_name(user_id, user_emails),
user_name=user_emails.get(user_id) or user_id or "鏈煡鐢ㄦ埛",
agent_id=assistant_id,
agent_name=agent_name,
model_name=r.model_name,
@@ -281,7 +272,10 @@ async def list_active_runs(request: Request) -> ActiveRunsResponse:
user_id = (meta_row.user_id if meta_row else "") or ""
metadata = (meta_row.metadata_json if meta_row else None) or r.metadata or {}
assistant_id = r.assistant_id or ""
agent_name = _display_agent_name(assistant_id, agent_names)
if assistant_id in _DEFAULT_ASSISTANT_IDS:
agent_name = "默认助手"
else:
agent_name = agent_names.get(assistant_id) or assistant_id
items.append(
ActiveRunItem(
run_id=r.run_id,
@@ -290,7 +284,7 @@ async def list_active_runs(request: Request) -> ActiveRunsResponse:
created_at=r.created_at,
elapsed_seconds=_elapsed_seconds(r.created_at, now),
user_id=user_id,
user_name=_display_user_name(user_id, user_emails),
user_name=user_emails.get(user_id) or user_id or "未知用户",
agent_id=assistant_id,
agent_name=agent_name,
model_name=str((r.metadata or {}).get("model_name") or ""),
@@ -1,74 +0,0 @@
"""Admin view and pause control for historical audit-data cleanup."""
from __future__ import annotations
import asyncio
from datetime import datetime
from time import time
from typing import Any
from fastapi import APIRouter, Request
from pydantic import BaseModel
from app.gateway.admin_audit_retention_control import (
is_paused,
read_status,
set_paused,
write_status,
)
from app.gateway.routers.admin_users import _require_admin
from deerflow.config.runtime_paths import runtime_home
from deerflow.persistence.types import BEIJING_TZ
router = APIRouter(prefix="/api/admin/audit-retention", tags=["admin-audit-retention"])
class RetentionControlRequest(BaseModel):
paused: bool
def _status_payload() -> dict[str, Any]:
root = runtime_home()
value = read_status(root)
paused = is_paused(root)
value["paused"] = paused
value["pause_requested"] = paused and value.get("status") == "running"
if value["pause_requested"]:
value["display_status"] = "pausing"
elif paused:
value["display_status"] = "paused"
else:
value["display_status"] = value.get("status", "scheduled")
return value
def _apply_control(paused: bool) -> dict[str, Any]:
root = runtime_home()
set_paused(root, paused)
value = read_status(root)
if paused and value.get("status") != "running":
value.update(status="paused", current_source=None, next_run_at=None)
elif not paused and value.get("status") == "paused":
value.update(status="scheduled", next_run_at=time())
write_status(root, value)
return _status_payload()
@router.get("")
async def get_retention_status(request: Request) -> dict[str, Any]:
await _require_admin(request)
return await asyncio.to_thread(_status_payload)
@router.put("/control")
async def update_retention_control(
payload: RetentionControlRequest,
request: Request,
) -> dict[str, Any]:
await _require_admin(request)
value = await asyncio.to_thread(_apply_control, payload.paused)
value["control_updated_at"] = datetime.now(BEIJING_TZ).isoformat(timespec="seconds")
return value
__all__ = ["router"]
@@ -20,37 +20,34 @@ import os
import re
from collections.abc import AsyncIterator
from contextvars import ContextVar
from datetime import UTC, datetime, timedelta
from datetime import time as datetime_time
from hashlib import sha256
from time import perf_counter
from typing import Any
from datetime import UTC, datetime, timedelta, time as datetime_time
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from langchain_core.messages import HumanMessage, SystemMessage
from pydantic import BaseModel, Field
from sqlalchemy import and_, case, desc, func, not_, or_, select, text
from langchain_core.messages import HumanMessage, SystemMessage
from app.gateway.admin_query_window import (
ADMIN_QUERY_WINDOW_DAYS,
resolve_admin_query_window,
)
from app.gateway.deps import (
get_config,
get_agent_store,
get_checkpointer,
get_config,
get_current_user_from_request,
get_thread_store,
)
from app.gateway.routers.memory import _build_v1_provider, _is_v1
from deerflow.agents.memory import BuiltinFileProvider
from deerflow.config.app_config import AppConfig
from deerflow.config.paths import get_paths
from deerflow.config.system_settings import load_system_settings
from deerflow.models import create_chat_model
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.runtime import serialize_channel_values
from app.gateway.routers.memory import _build_v1_provider, _is_v1
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskRunRow
@@ -58,7 +55,6 @@ from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.types import BEIJING_TZ
from deerflow.persistence.user.model import UserRow
from deerflow.runtime import serialize_channel_values
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
@@ -96,13 +92,6 @@ _LEADERBOARD_LOCK_WAIT_SECONDS = max(
1,
int(os.getenv("ADMIN_LEADERBOARD_LOCK_WAIT_SECONDS", "5")),
)
# The checkpoint fallback exists for installations that predate compact run
# summaries. Keep its work deliberately small: current run summaries are the
# indexed source of truth for an administrator's recent audit view.
_ADMIN_QA_CHECKPOINT_THREAD_LIMIT = 25
_ADMIN_QA_CHECKPOINT_MESSAGE_LIMIT = 100
_ADMIN_QA_CHECKPOINT_ITEM_LIMIT = 500
_ADMIN_QA_RUN_CANDIDATE_LIMIT = 1_000
_LEADERBOARD_QUEUE_ON_REQUEST_ENABLED = os.getenv(
"ADMIN_LEADERBOARD_QUEUE_ON_REQUEST_ENABLED",
os.getenv("ADMIN_LEADERBOARD_WEB_QUEUE_ENABLED", "true"),
@@ -318,6 +307,20 @@ class AdminLeaderboardResponse(BaseModel):
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
class AdminUserAnalyticsResponse(BaseModel):
range_days: int
since: str
until: str
user: AdminLeaderboardUser
overview: AdminLeaderboardOverview
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
models: list[AdminLeaderboardDimension] = Field(default_factory=list)
agents: list[AdminLeaderboardDimension] = Field(default_factory=list)
run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list)
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
class AdminLeaderboardReportResponse(BaseModel):
report: str
generated_at: str
@@ -349,7 +352,7 @@ class AdminLeaderboardSnapshotStatusResponse(BaseModel):
class AdminLeaderboardSnapshotBackfillRequest(BaseModel):
days: int = Field(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS)
days: int = Field(default=30, ge=1, le=365)
since: str | None = Field(default=None, description="Inclusive range start, ISO datetime")
until: str | None = Field(default=None, description="Inclusive range end, ISO datetime")
force: bool = Field(default=False, description="Queue dates even when a ready snapshot already exists")
@@ -460,13 +463,6 @@ async def _set_analytics_read_timeout(session) -> None:
(5.6 raises "Unknown system variable" otherwise)
- PostgreSQL/openGauss: ``statement_timeout`` (milliseconds)
"""
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
# Dolphin does not implement either MySQL session variable below. A failed
# SET aborts the current openGauss transaction, so skip before touching it.
if is_gauss_mysql_product():
return
conn = await session.connection()
dialect = conn.dialect
name = dialect.name
@@ -1736,7 +1732,7 @@ async def _refresh_daily_leaderboard_snapshot(
datetime.now(BEIJING_TZ).isoformat(),
(perf_counter() - snapshot_start) * 1000,
)
except TimeoutError:
except asyncio.TimeoutError:
logger.warning("Daily leaderboard snapshot timed out: %s", job_key)
await _store_daily_snapshot(
session_factory,
@@ -1842,7 +1838,7 @@ async def _load_or_schedule_daily_leaderboard(
@router.get("/leaderboard", response_model=AdminLeaderboardResponse)
async def get_user_leaderboard(
request: Request,
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
exclude_user_ids: list[str] | None = Query(default=None, description="User ids excluded from this leaderboard aggregation"),
@@ -1858,11 +1854,7 @@ async def get_user_leaderboard(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
return await _load_or_schedule_daily_leaderboard(
session_factory,
settings,
@@ -1879,7 +1871,7 @@ async def get_user_leaderboard(
@router.get("/leaderboard/snapshots", response_model=AdminLeaderboardSnapshotStatusResponse)
async def get_leaderboard_snapshot_status(
request: Request,
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
) -> AdminLeaderboardSnapshotStatusResponse:
@@ -1890,11 +1882,7 @@ async def get_leaderboard_snapshot_status(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
stat_dates = _stat_dates_for_range(since_dt, until_dt)
*_scope, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
rows = await _load_daily_snapshot_rows(session_factory, stat_dates, settings_hash)
@@ -1945,11 +1933,7 @@ async def queue_leaderboard_snapshot_backfill(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=body.days,
since=body.since,
until=body.until,
)
since_dt, until_dt, _range_days = _resolve_range(body.days, body.since, body.until)
stat_dates = _stat_dates_for_range(since_dt, until_dt)
*_scope, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
queued_dates = await _queue_daily_leaderboard_snapshots(
@@ -1972,7 +1956,7 @@ async def queue_leaderboard_snapshot_backfill(
@router.post("/leaderboard/refresh-today", response_model=AdminLeaderboardResponse)
async def refresh_today_leaderboard(
request: Request,
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
exclude_user_ids: list[str] | None = Query(default=None, description="User ids excluded from this leaderboard aggregation"),
@@ -1995,11 +1979,6 @@ async def refresh_today_leaderboard(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
(
excluded,
resolved_include_scheduled,
@@ -2026,6 +2005,7 @@ async def refresh_today_leaderboard(
include_failed=resolved_include_failed,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
return await _load_or_schedule_daily_leaderboard(
session_factory,
settings,
@@ -2092,7 +2072,7 @@ async def _build_leaderboard_report_prompt(
@router.post("/leaderboard/report", response_model=AdminLeaderboardReportResponse)
async def generate_user_leaderboard_report(
request: Request,
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
limit: int = Query(default=30, ge=10, le=100),
@@ -2106,11 +2086,7 @@ async def generate_user_leaderboard_report(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
resolved_model_name = _resolve_report_model_name(config, model_name)
cache_key = _report_cache_key(since=since_dt, until=until_dt, limit=limit, model_name=resolved_model_name, settings=settings)
cached = _cached_report(cache_key)
@@ -2152,7 +2128,7 @@ async def generate_user_leaderboard_report(
@router.post("/leaderboard/report/stream")
async def stream_user_leaderboard_report(
request: Request,
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
limit: int = Query(default=30, ge=10, le=100),
@@ -2166,11 +2142,7 @@ async def stream_user_leaderboard_report(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
resolved_model_name = _resolve_report_model_name(config, model_name)
cache_key = _report_cache_key(since=since_dt, until=until_dt, limit=limit, model_name=resolved_model_name, settings=settings)
@@ -2224,6 +2196,62 @@ async def stream_user_leaderboard_report(
)
@router.get("/{user_id}/analytics", response_model=AdminUserAnalyticsResponse)
async def get_user_analytics(
user_id: str,
request: Request,
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
limit: int = Query(default=20, ge=5, le=100),
) -> AdminUserAnalyticsResponse:
"""Return one user's admin analytics profile."""
await _require_admin(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
common = {
"include_scheduled": settings.include_scheduled,
"include_admins": settings.include_admins,
"include_failed": settings.include_failed,
}
tool_common = {
"include_scheduled": settings.include_scheduled,
"include_admins": settings.include_admins,
}
since_dt, until_dt, range_days = _resolve_range(days, since, until)
async with session_factory() as session:
await _set_analytics_read_timeout(session)
users = await _query_user_leaderboard(session, since_dt, until_dt, user_id=user_id, **common)
if not users:
raise HTTPException(status_code=404, detail="User not found")
overview = await _query_overview(session, since_dt, until_dt, user_id=user_id, **common)
skills = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="skill", limit=limit, **tool_common)
tools = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="tool", limit=limit, **tool_common)
models = await _query_model_rankings(session, since_dt, until_dt, user_id=user_id, limit=limit, **common)
if not models:
models = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="model", limit=limit, **common)
agents = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="agent", limit=limit, **common)
run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="status", limit=limit, **common)
trends = await _query_trends(session, since_dt, until_dt, user_id=user_id, **common)
return AdminUserAnalyticsResponse(
range_days=range_days,
since=_iso(since_dt),
until=_iso(until_dt),
user=users[0],
overview=overview,
skills=skills,
tools=tools,
models=models,
agents=agents,
run_statuses=run_statuses,
trends=trends,
)
# ── Q&A audit list ─────────────────────────────────────────────────────────
@@ -2293,12 +2321,7 @@ def _qa_message_text(message: Any) -> str:
return _message_content_to_text(_message_value(message, "content")).strip()
async def _read_thread_checkpoint_messages(
checkpointer,
thread_id: str,
*,
max_messages: int | None = None,
) -> list[dict]:
async def _read_thread_checkpoint_messages(checkpointer, thread_id: str) -> list[dict]:
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
@@ -2311,9 +2334,7 @@ async def _read_thread_checkpoint_messages(
channel_values = checkpoint.get("channel_values", {}) if isinstance(checkpoint, dict) else {}
values = serialize_channel_values(channel_values)
messages = values.get("messages", [])
if not isinstance(messages, list):
return []
return messages[-max_messages:] if max_messages else messages
return messages if isinstance(messages, list) else []
def _qa_items_from_checkpoint_messages(
@@ -2381,97 +2402,47 @@ async def _list_thread_checkpoint_qa_items(
*,
request: Request,
user_id: str,
since: str | None = None,
until: str | None = None,
recent_only: bool = False,
) -> list[AdminUserQaItem]:
repo = get_thread_store(request)
checkpointer = get_checkpointer(request)
items: list[AdminUserQaItem] = []
rows: list[dict]
if recent_only:
# Checkpoints do not reliably carry a timestamp for every historical
# message. Limit the fallback to threads *created* in the effective
# audit window, so an old long-running conversation cannot leak a full
# history into a recent-only administrator query. Run summaries remain
# the complete, indexed source for recent messages in older threads.
session_factory = get_session_factory()
since_dt = _parse_optional_datetime(since)
until_dt = _parse_optional_datetime(until)
if session_factory is None or since_dt is None or until_dt is None:
return items
async with session_factory() as session:
await _set_analytics_read_timeout(session)
result = await session.execute(
select(ThreadMetaRow)
.where(
ThreadMetaRow.user_id == user_id,
ThreadMetaRow.created_at >= since_dt,
ThreadMetaRow.created_at <= until_dt,
offset = 0
page_size = 200
while True:
rows = await repo.search(limit=page_size, offset=offset, user_id=user_id)
if not rows:
break
for row in rows:
thread_id = str(row.get("thread_id") or "")
if not thread_id:
continue
messages = await _read_thread_checkpoint_messages(checkpointer, thread_id)
if not messages:
continue
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
context = row.get("context") if isinstance(row.get("context"), dict) else {}
assistant_id = str(
row.get("assistant_id")
or (context or {}).get("agent_id")
or (metadata or {}).get("agent_id")
or (context or {}).get("agent_name")
or (metadata or {}).get("agent_name")
or ""
)
thread_time = str(row.get("updated_at") or row.get("created_at") or "")
items.extend(
_qa_items_from_checkpoint_messages(
user_id=user_id,
thread_id=thread_id,
thread_title=str(row.get("display_name") or ""),
assistant_id=assistant_id,
thread_time=thread_time,
messages=messages,
)
.order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
.limit(_ADMIN_QA_CHECKPOINT_THREAD_LIMIT)
)
rows = [
{
"thread_id": row.thread_id,
"assistant_id": row.assistant_id,
"display_name": row.display_name,
"metadata": row.metadata_json or {},
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
for row in result.scalars()
]
else:
# Preserve the regular user's existing history behavior. The bounded
# administrator route always takes the SQL path above.
repo = get_thread_store(request)
rows = []
offset = 0
page_size = 200
while True:
page = await repo.search(limit=page_size, offset=offset, user_id=user_id)
if not page:
break
rows.extend(page)
if len(page) < page_size:
break
offset += len(page)
for row in rows:
thread_id = str(row.get("thread_id") or "")
if not thread_id:
continue
messages = await _read_thread_checkpoint_messages(
checkpointer,
thread_id,
max_messages=_ADMIN_QA_CHECKPOINT_MESSAGE_LIMIT if recent_only else None,
)
if not messages:
continue
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
context = row.get("context") if isinstance(row.get("context"), dict) else {}
assistant_id = str(
row.get("assistant_id")
or (context or {}).get("agent_id")
or (metadata or {}).get("agent_id")
or (context or {}).get("agent_name")
or (metadata or {}).get("agent_name")
or ""
)
thread_time = str(row.get("updated_at") or row.get("created_at") or "")
items.extend(
_qa_items_from_checkpoint_messages(
user_id=user_id,
thread_id=thread_id,
thread_title=str(row.get("display_name") or ""),
assistant_id=assistant_id,
thread_time=thread_time,
messages=messages,
)
)
if recent_only and len(items) >= _ADMIN_QA_CHECKPOINT_ITEM_LIMIT:
return items[:_ADMIN_QA_CHECKPOINT_ITEM_LIMIT]
if len(rows) < page_size:
break
offset += len(rows)
return items
@@ -2481,61 +2452,44 @@ async def _list_run_summary_qa_items(
user_id: str,
since: str | None,
until: str | None,
q: str | None = None,
candidate_limit: int | None = None,
) -> list[AdminUserQaItem]:
filters = [
RunRow.user_id == user_id,
RunRow.first_human_message.is_not(None),
RunRow.first_human_message != "",
]
if since:
filters.append(RunRow.created_at >= _parse_optional_datetime(since))
if until:
filters.append(RunRow.created_at <= _parse_optional_datetime(until))
needle = (q or "").strip().lower()
if needle:
like = f"%{needle}%"
filters.append(
or_(
func.lower(RunRow.run_id).like(like),
func.lower(RunRow.thread_id).like(like),
func.lower(func.coalesce(RunRow.assistant_id, "")).like(like),
func.lower(func.coalesce(RunRow.first_human_message, "")).like(like),
func.lower(func.coalesce(RunRow.last_ai_message, "")).like(like),
func.lower(func.coalesce(ThreadMetaRow.display_name, "")).like(like),
)
)
if since or until:
since_dt, until_dt, _range_days = _resolve_range(365, since, until)
filters.append(_run_time_filter(since_dt, until_dt))
async with session_factory() as session:
await _set_analytics_read_timeout(session)
stmt = (
select(
RunRow.run_id,
RunRow.thread_id,
ThreadMetaRow.display_name,
RunRow.assistant_id,
RunRow.status,
RunRow.model_name,
RunRow.first_human_message,
RunRow.last_ai_message,
RunRow.total_tokens,
RunRow.created_at,
RunRow.updated_at,
rows = (
await session.execute(
select(
RunRow.run_id,
RunRow.thread_id,
ThreadMetaRow.display_name,
RunRow.assistant_id,
RunRow.status,
RunRow.model_name,
RunRow.first_human_message,
RunRow.last_ai_message,
RunRow.total_tokens,
RunRow.created_at,
RunRow.updated_at,
)
.outerjoin(
ThreadMetaRow,
and_(
ThreadMetaRow.thread_id == RunRow.thread_id,
ThreadMetaRow.user_id == RunRow.user_id,
),
)
.where(*filters)
.order_by(RunRow.created_at.desc(), RunRow.run_id.desc())
)
.outerjoin(
ThreadMetaRow,
and_(
ThreadMetaRow.thread_id == RunRow.thread_id,
ThreadMetaRow.user_id == RunRow.user_id,
),
)
.where(*filters)
.order_by(RunRow.created_at.desc(), RunRow.run_id.desc())
)
if candidate_limit is not None:
stmt = stmt.limit(max(1, min(int(candidate_limit), _ADMIN_QA_RUN_CANDIDATE_LIMIT)))
rows = (await session.execute(stmt)).all()
).all()
return [
AdminUserQaItem(
@@ -2600,39 +2554,19 @@ async def _list_user_qa_page(
since: str | None,
until: str | None,
include_checkpoints: bool = True,
recent_checkpoint_threads_only: bool = False,
) -> AdminUserQaPageResponse:
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Q&A audit requires database persistence")
candidate_limit = None
if recent_checkpoint_threads_only:
# Normal administrator pages need only a modest look-ahead beyond the
# requested page. This protects against a single high-frequency user
# materializing a whole month's question/answer bodies at once.
candidate_limit = min(
_ADMIN_QA_RUN_CANDIDATE_LIMIT,
max(200, page * page_size + page_size),
)
items = await _list_run_summary_qa_items(
session_factory=session_factory,
user_id=user_id,
since=since,
until=until,
q=q if candidate_limit is not None else None,
candidate_limit=candidate_limit,
)
if include_checkpoints:
items.extend(
await _list_thread_checkpoint_qa_items(
request=request,
user_id=user_id,
since=since,
until=until,
recent_only=recent_checkpoint_threads_only,
)
)
items.extend(await _list_thread_checkpoint_qa_items(request=request, user_id=user_id))
by_key: dict[tuple[str, str, str], AdminUserQaItem] = {}
for item in items:
@@ -2660,24 +2594,22 @@ async def _list_user_qa_page(
async def list_user_qa(
user_id: str,
request: Request,
page: int = Query(default=1, ge=1, le=100),
page: int = Query(default=1, ge=1),
page_size: int = Query(default=20, ge=1, le=100),
q: str | None = Query(default=None, description="Search question, answer, run id, or thread id"),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
) -> AdminUserQaPageResponse:
"""Return one user's recent Q&A audit records."""
"""Return one user's Q&A runs, including older checkpoint-only threads."""
await _require_admin(request)
since_dt, until_dt, _range_days = resolve_admin_query_window(since=since, until=until)
return await _list_user_qa_page(
request=request,
user_id=user_id,
page=page,
page_size=page_size,
q=q,
since=_iso(since_dt),
until=_iso(until_dt),
recent_checkpoint_threads_only=True,
since=since,
until=until,
)
@@ -2720,41 +2652,11 @@ class AdminThreadItem(BaseModel):
async def list_user_threads(
user_id: str,
request: Request,
limit: int = Query(default=100, ge=1, le=200),
limit: int = Query(default=100, ge=1, le=500),
offset: int = Query(default=0, ge=0),
) -> list[AdminThreadItem]:
"""List one user's threads active in the rolling administrator window."""
"""List one user's conversation threads, newest first."""
await _require_admin(request)
cutoff, _until, _range_days = resolve_admin_query_window()
session_factory = get_session_factory()
if session_factory is not None:
async with session_factory() as session:
await _set_analytics_read_timeout(session)
rows = (
await session.execute(
select(ThreadMetaRow)
.where(
ThreadMetaRow.user_id == user_id,
ThreadMetaRow.updated_at >= cutoff,
)
.order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
.limit(limit)
.offset(offset)
)
).scalars().all()
return [
AdminThreadItem(
thread_id=row.thread_id,
display_name=row.display_name or "",
status=row.status or "idle",
created_at=_iso(row.created_at),
updated_at=_iso(row.updated_at),
)
for row in rows
]
# Memory-mode fallback has no SQL date predicate. It is only used when
# database persistence is unavailable, and still preserves the policy.
repo = get_thread_store(request)
rows = await repo.search(limit=limit, offset=offset, user_id=user_id)
return [
@@ -2766,7 +2668,6 @@ async def list_user_threads(
updated_at=str(r.get("updated_at") or ""),
)
for r in rows
if _qa_sort_datetime(str(r.get("updated_at") or "")) >= cutoff
]
@@ -2775,7 +2676,7 @@ async def list_user_thread_messages(
user_id: str,
thread_id: str,
request: Request,
limit: int = Query(default=200, ge=1, le=200),
limit: int = Query(default=400, ge=1, le=1000),
) -> list[dict]:
"""Return the conversation messages of one of the user's threads.
@@ -2794,12 +2695,6 @@ async def list_user_thread_messages(
owner = thread.get("user_id")
if owner is not None and owner != user_id:
raise HTTPException(status_code=404, detail="Thread does not belong to this user")
cutoff, _until, _range_days = resolve_admin_query_window()
if _qa_sort_datetime(str(thread.get("updated_at") or thread.get("created_at") or "")) < cutoff:
raise HTTPException(
status_code=404,
detail=f"仅可查看最近 {ADMIN_QUERY_WINDOW_DAYS} 天的审计会话",
)
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
File diff suppressed because it is too large. Load diff
@@ -1,170 +0,0 @@
"""Application chrome name and logo.
Stored separately from ``generated_page_branding`` and the
``generated-page-branding/`` directory. An empty name or a missing logo leaves
the built-in title and static theme logos in place.
"""
from __future__ import annotations
import asyncio
from uuid import uuid4
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse
from pydantic import BaseModel
from app.gateway.routers.system_settings import _require_admin
from deerflow.config.runtime_paths import runtime_home
from deerflow.config.system_settings import (
AppIdentitySettings,
app_identity_logo_path,
load_system_settings,
save_system_settings,
)
router = APIRouter(prefix="/api/system-settings", tags=["system-settings"])
public_router = APIRouter(prefix="/api/public/app-identity", tags=["app-identity"])
_MAX_LOGO_BYTES = 10 * 1024 * 1024
_PUBLIC_LOGO_PATH = "/api/public/app-identity/logo"
class AppIdentityResponse(BaseModel):
system_name: str = ""
logo_url: str | None = None
logo_version: str | None = None
has_logo: bool = False
def _detect_logo(data: bytes) -> tuple[str, str]:
if data.startswith(b"\x89PNG\r\n\x1a\n"):
return ".png", "image/png"
if data.startswith(b"\xff\xd8\xff"):
return ".jpg", "image/jpeg"
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return ".webp", "image/webp"
raise HTTPException(status_code=422, detail="Logo 仅支持 PNG、JPG 或 WebP 图片")
def _normalize_system_name(value: str | None) -> str:
normalized = " ".join(str(value or "").split())
if len(normalized) > 80:
raise HTTPException(status_code=422, detail="系统名称不能超过 80 个字符")
return normalized
def _response(settings: AppIdentitySettings) -> AppIdentityResponse:
path = app_identity_logo_path(settings)
version = settings.logo_version if path is not None else None
return AppIdentityResponse(
system_name=settings.system_name,
logo_url=f"{_PUBLIC_LOGO_PATH}?v={version}" if version else None,
logo_version=version,
has_logo=path is not None,
)
def _write_logo(data: bytes, suffix: str) -> str:
directory = runtime_home() / "app-identity"
directory.mkdir(parents=True, exist_ok=True)
filename = f"logo-{uuid4().hex}{suffix}"
destination = directory / filename
temporary = directory / f".{filename}.tmp"
try:
temporary.write_bytes(data)
temporary.replace(destination)
finally:
temporary.unlink(missing_ok=True)
return filename
@router.get("/app-identity", response_model=AppIdentityResponse)
async def get_app_identity_settings() -> AppIdentityResponse:
"""Return the application name and uploaded logo. Any logged-in user may read it."""
return _response(load_system_settings().app_identity)
@router.put("/app-identity", response_model=AppIdentityResponse)
async def update_app_identity_settings(
request: Request,
system_name: str = Form(default=""),
remove_logo: bool = Form(default=False),
logo: UploadFile | None = File(default=None),
) -> AppIdentityResponse:
"""Save the application chrome name and optional logo. Admin only."""
await _require_admin(request)
if logo is not None and remove_logo:
raise HTTPException(status_code=422, detail="上传 Logo 时不能同时恢复静态 Logo")
current = load_system_settings()
previous_logo = app_identity_logo_path(current.app_identity)
update: dict[str, object] = {"system_name": _normalize_system_name(system_name)}
new_logo_path = None
if logo is not None:
data = await logo.read(_MAX_LOGO_BYTES + 1)
await logo.close()
if not data:
raise HTTPException(status_code=422, detail="Logo 文件不能为空")
if len(data) > _MAX_LOGO_BYTES:
raise HTTPException(status_code=413, detail="Logo 文件不能超过 10MB")
suffix, media_type = _detect_logo(data)
filename = await asyncio.to_thread(_write_logo, data, suffix)
new_logo_path = runtime_home() / "app-identity" / filename
update.update(
{
"logo_filename": filename,
"logo_media_type": media_type,
"logo_version": uuid4().hex,
}
)
elif remove_logo:
update.update(
{
"logo_filename": None,
"logo_media_type": None,
"logo_version": None,
}
)
updated = current.app_identity.model_copy(update=update)
try:
saved = save_system_settings(current.model_copy(update={"app_identity": updated}))
except Exception:
if new_logo_path is not None:
await asyncio.to_thread(new_logo_path.unlink, missing_ok=True)
raise
if previous_logo is not None and previous_logo != app_identity_logo_path(saved.app_identity):
await asyncio.to_thread(previous_logo.unlink, missing_ok=True)
return _response(saved.app_identity)
@router.delete("/app-identity", response_model=AppIdentityResponse)
async def delete_app_identity_settings(request: Request) -> AppIdentityResponse:
"""Clear the application name and uploaded logo. Admin only."""
await _require_admin(request)
current = load_system_settings()
previous_logo = app_identity_logo_path(current.app_identity)
saved = save_system_settings(current.model_copy(update={"app_identity": AppIdentitySettings()}))
if previous_logo is not None:
await asyncio.to_thread(previous_logo.unlink, missing_ok=True)
return _response(saved.app_identity)
@public_router.get("/logo")
async def get_app_identity_logo() -> FileResponse:
"""Serve the uploaded application logo. Missing logos stay on the static assets."""
settings = load_system_settings().app_identity
path = app_identity_logo_path(settings)
if path is None:
raise HTTPException(status_code=404, detail="未上传系统 Logo")
return FileResponse(
path,
media_type=settings.logo_media_type or "image/png",
content_disposition_type="inline",
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)
@@ -13,13 +13,10 @@ from fastapi.responses import FileResponse, PlainTextResponse, Response
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.operation_timing import OperationTiming
from app.gateway.path_utils import aresolve_thread_virtual_path
from app.gateway.utils import sanitize_log_param
from deerflow.config.paths import get_paths
from deerflow.runtime.thread_paths import aresolve_path_user_id
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.file_io import file_io
logger = logging.getLogger(__name__)
@@ -538,7 +535,7 @@ async def _scan_artifact_library(
user_id=path_user_id,
)
scan_errors: list[str] = []
files = await file_io(
files = await asyncio.to_thread(
collect_outputs_listing,
outputs_dir,
errors=scan_errors,
@@ -854,7 +851,7 @@ async def download_strategy_project_artifact(project_id: str, request: Request)
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
)
@require_permission("threads", "read", owner_check=True)
async def list_artifacts(thread_id: str, request: Request, response: Response = None) -> dict: # type: ignore[assignment]
async def list_artifacts(thread_id: str, request: Request) -> dict:
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
@@ -868,24 +865,8 @@ async def list_artifacts(thread_id: str, request: Request, response: Response =
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
目录不存在时 ``files`` 为空数组(非 404)。
"""
with OperationTiming(
response,
logger,
"thread_artifacts",
thread_id=sanitize_log_param(thread_id),
) as timing:
with timing.phase("owner_resolve"):
user_id = await aresolve_path_user_id(thread_id)
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=user_id)
# Probe outside the shared artifact-files pool so a background library
# scan cannot stall an empty hello-chat listing for several seconds.
with timing.phase("dir_probe"):
exists = await asyncio.to_thread(lambda: outputs_dir.exists() and outputs_dir.is_dir())
if not exists:
return {"thread_id": thread_id, "files": []}
with timing.phase("dir_scan"):
files = await file_io(collect_outputs_listing, outputs_dir)
return {"thread_id": thread_id, "files": files}
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
def _read_bounded_text_prefix(path: Path, max_chars: int) -> tuple[str, bool]:
@@ -941,7 +922,7 @@ async def preview_artifact(
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
if not actual_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
if not await file_io(is_text_file_by_content, actual_path):
if not await asyncio.to_thread(is_text_file_by_content, actual_path):
raise HTTPException(status_code=415, detail="Bounded preview supports text content only")
from deerflow.uploads.inspection import inspect_text_file
@@ -955,7 +936,7 @@ async def preview_artifact(
total_bytes = actual_path.stat().st_size
if total_bytes >= threshold:
report = await file_io(
report = await asyncio.to_thread(
inspect_text_file,
actual_path,
max_result_chars=effective_limit,
@@ -972,7 +953,7 @@ async def preview_artifact(
warnings=[str(item) for item in report.get("warnings") or []],
)
content, truncated = await file_io(_read_bounded_text_prefix, actual_path, effective_limit)
content, truncated = await asyncio.to_thread(_read_bounded_text_prefix, actual_path, effective_limit)
warnings = ["预览达到字符上限,未读取的内容不会进入页面或模型上下文。"] if truncated else []
return ArtifactPreviewResponse(
path=path,
@@ -1042,7 +1023,7 @@ async def get_artifact(thread_id: str, path: str, request: Request, download: bo
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
# Extract the file from the .skill archive
content = await file_io(_extract_file_from_skill_archive, actual_skill_path, internal_path)
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
if content is None:
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
@@ -1096,7 +1077,7 @@ async def get_artifact(thread_id: str, path: str, request: Request, download: bo
headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)},
)
if await file_io(is_text_file_by_content, actual_path):
if is_text_file_by_content(actual_path):
return FileResponse(
path=actual_path,
media_type=mime_type or "text/plain",
@@ -1148,5 +1129,5 @@ async def update_artifact(
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
await file_io(actual_path.write_text, body.content, encoding="utf-8")
actual_path.write_text(body.content, encoding="utf-8")
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
@@ -1,7 +1,5 @@
"""Authentication endpoints."""
import asyncio
import json
import logging
import os
import time
@@ -11,9 +9,8 @@ from typing import Literal
import httpx
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status
from fastapi.security import OAuth2PasswordRequestForm
from pydantic import BaseModel, ConfigDict, EmailStr, Field, TypeAdapter, ValidationError, field_validator, model_validator
from pydantic import BaseModel, ConfigDict, EmailStr, Field, TypeAdapter, field_validator, model_validator
from app.gateway.admin_query_window import recent_admin_query_cutoff
from app.gateway.auth import (
UserResponse,
create_access_token,
@@ -21,20 +18,18 @@ from app.gateway.auth import (
from app.gateway.auth.config import get_auth_config
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
from app.gateway.auth.models import User
from app.gateway.auth.password import has_account_password, hash_password_async, is_password_hash, verify_account_password
from app.gateway.auth.repositories.base import CredentialConflictError
from app.gateway.auth.password import has_account_password, hash_password_async, is_password_hash, verify_account_password
from app.gateway.auth.repositories.base import CredentialConflictError
from app.gateway.csrf_middleware import is_secure_request
from app.gateway.deps import get_agent_store, get_current_user_from_request, get_local_provider, get_run_store, get_skill_store
from app.gateway.perf_logging import finish_trace, trace_request, trace_stage
from deerflow.config import get_app_config
from deerflow.config.system_settings import AuthenticationSettings, load_system_settings
from deerflow.config.system_settings import AuthenticationSettings, load_system_settings
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.skills.base import SkillStore
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
_USER_NOT_LOOKED_UP = object()
# ── Request/Response Models ──────────────────────────────────────────────
@@ -113,34 +108,34 @@ def _validate_strong_password(value: str) -> str:
``@field_validator(field_name)`` without inheritance gymnastics.
"""
if _password_is_common(value):
raise ValueError("密码过于常见,请设置更安全的密码")
if value != value.strip():
raise ValueError("密码首尾不能包含空格")
raise ValueError("密码过于常见,请设置更安全的密码")
if value != value.strip():
raise ValueError("密码首尾不能包含空格")
return value
class RegisterRequest(BaseModel):
"""Request model for user registration."""
username: str | None = Field(default=None, min_length=1, max_length=128)
email: EmailStr | None = None
password: str = Field(..., min_length=8, max_length=255)
model_config = ConfigDict(extra="forbid")
username: str | None = Field(default=None, min_length=1, max_length=128)
email: EmailStr | None = None
password: str = Field(..., min_length=8, max_length=255)
model_config = ConfigDict(extra="forbid")
_strong_password = field_validator("password")(classmethod(lambda cls, v: _validate_strong_password(v)))
@model_validator(mode="after")
def validate_identity(self):
if bool(self.username) == bool(self.email):
raise ValueError("请填写用户名或邮箱(只需一项)")
if self.username is not None:
value = self.username.strip()
if "@" not in value and (not value or value[0] in "._-" or value[-1] in "._-" or any(not (ch.isalnum() or ch in "._-") for ch in value)):
raise ValueError("用户名仅支持字母、数字、点、下划线和短横线,首尾须为字母或数字")
self.username = value
if "@" in value:
TypeAdapter(EmailStr).validate_python(value)
return self
@model_validator(mode="after")
def validate_identity(self):
if bool(self.username) == bool(self.email):
raise ValueError("请填写用户名或邮箱(只需一项)")
if self.username is not None:
value = self.username.strip()
if "@" not in value and (not value or value[0] in "._-" or value[-1] in "._-" or any(not (ch.isalnum() or ch in "._-") for ch in value)):
raise ValueError("用户名仅支持字母、数字、点、下划线和短横线,首尾须为字母或数字")
self.username = value
if "@" in value:
TypeAdapter(EmailStr).validate_python(value)
return self
class UsernameLoginRequest(BaseModel):
@@ -184,8 +179,8 @@ class UsernameLoginResponse(BaseModel):
class ChangePasswordRequest(BaseModel):
"""Request model for password change (also handles setup flow)."""
current_password: str = Field(default="", max_length=255)
new_password: str = Field(..., min_length=8, max_length=255)
current_password: str = Field(default="", max_length=255)
new_password: str = Field(..., min_length=8, max_length=255)
new_email: EmailStr | None = None
_strong_password = field_validator("new_password")(classmethod(lambda cls, v: _validate_strong_password(v)))
@@ -195,12 +190,12 @@ class MessageResponse(BaseModel):
"""Generic message response."""
message: str
class PasswordChangeResponse(MessageResponse):
access_token: str
token_type: str = "bearer"
expires_in: int
class PasswordChangeResponse(MessageResponse):
access_token: str
token_type: str = "bearer"
expires_in: int
# ── Helpers ───────────────────────────────────────────────────────────────
@@ -242,71 +237,28 @@ def _username_to_email(username: str) -> str:
return f"{local}@{domain}"
# 后台岗位同步任务的强引用集合:防止 create_task 的任务被 GC 中途回收。
_position_sync_tasks: set = set()
_POSITION_SYNC_CACHE_SECONDS = 60.0
_MAX_POSITION_SYNC_CACHE_USERS = 10_000
_position_sync_completed_at: dict[str, float] = {}
_position_sync_inflight: dict[str, asyncio.Task[None]] = {}
def _sync_user_position(request: Request, user_id: str) -> None:
"""Best-effort: schedule the user's 岗位 (position) sync off the login path.
async def _sync_user_position(request: Request, user_id: str) -> None:
"""Best-effort: materialize the user's 岗位 (position) grants on login.
Ensures a user who has never been synced — e.g. a brand-new account that
inherits the ``is_default`` position — gets the default agents / tasks /
prompts loaded into their "我的" tables. The sync is idempotent but touches
a dozen tables per user, so it runs as a fire-and-forget task instead of
blocking the login response; the materialized rows land a moment later and
failures are logged, never surfaced.
prompts loaded into their "我的" tables. Failures never block login.
"""
normalized_user_id = str(user_id)
now = time.monotonic()
completed_at = _position_sync_completed_at.get(normalized_user_id)
if completed_at is not None and now - completed_at < _POSITION_SYNC_CACHE_SECONDS:
return
if normalized_user_id in _position_sync_inflight:
return
sync = getattr(request.app.state, "position_sync", None)
if sync is None:
return
async def _run() -> None:
try:
await sync.sync_user(normalized_user_id)
_position_sync_completed_at[normalized_user_id] = time.monotonic()
if len(_position_sync_completed_at) > _MAX_POSITION_SYNC_CACHE_USERS:
oldest = sorted(_position_sync_completed_at, key=_position_sync_completed_at.get)
for stale_user_id in oldest[: len(oldest) // 2]:
_position_sync_completed_at.pop(stale_user_id, None)
except Exception:
logger.debug("position sync on login failed for user %s", user_id, exc_info=True)
finally:
_position_sync_inflight.pop(normalized_user_id, None)
try:
task = asyncio.create_task(_run())
except RuntimeError:
# 防御:请求处理器内必有运行中的事件循环,这里理论上走不到。
logger.debug("position sync scheduling failed for user %s", user_id, exc_info=True)
return
_position_sync_tasks.add(task)
_position_sync_inflight[normalized_user_id] = task
task.add_done_callback(_position_sync_tasks.discard)
sync = getattr(request.app.state, "position_sync", None)
if sync is not None:
await sync.sync_user(str(user_id))
except Exception:
logger.debug("position sync on login failed for user %s", user_id, exc_info=True)
async def _get_or_create_user_by_username(
username: str,
*,
existing_user: User | None | object = _USER_NOT_LOOKED_UP,
):
async def _get_or_create_user_by_username(username: str):
"""Resolve a username to a local user, auto-registering when absent.
Shared by ``/login/username`` and ``/login/token`` — both turn an
externally-supplied username into a local account and sign it in. Returns
``(user, created)`` where ``created`` is True only when this call inserted
the row. Roles are persisted authorization data, never inferred from names.
the row. Roles are persisted authorization data, never inferred from names.
"""
provider = get_local_provider()
email = _username_to_email(username)
@@ -318,12 +270,7 @@ async def _get_or_create_user_by_username(
# also exists at today's canonical domain. Choosing the exact canonical
# row first made the same username alternate backend user_ids after a
# domain migration, which in turn made its existing threads inaccessible.
if existing_user is not _USER_NOT_LOOKED_UP:
# /login/username already performed this lookup to validate the
# account-specific password. Reuse both hits and misses instead of
# issuing the same MySQL query twice during a login burst.
user = existing_user
elif local_part is not None:
if local_part is not None:
user = await provider.find_user_by_email_local_part(local_part)
if user is not None and user.email != email:
logger.info("Matched username %r to stable legacy account %s", username, user.email)
@@ -334,7 +281,7 @@ async def _get_or_create_user_by_username(
try:
# 黑名单模式:从未登录过的新用户(此处即首次自动注册)默认放行(approved=True)。
# 管理员可在白名单管理页对个别用户「取消放行」来封禁——仅当登录白名单开关打开时生效。
user = await provider.create_user(email=email, password=None, system_role="user", needs_setup=False, approved=True)
user = await provider.create_user(email=email, password=None, system_role="user", needs_setup=False, approved=True)
created = True
logger.info("Auto-registered passwordless user: %s", email)
except ValueError:
@@ -343,7 +290,7 @@ async def _get_or_create_user_by_username(
if user is None and local_part is not None:
user = await provider.find_user_by_email_local_part(local_part)
if user is None:
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="该用户名已存在,请使用原账号登录")
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="该用户名已存在,请使用原账号登录")
return user, created
@@ -608,170 +555,134 @@ def _record_login_success(ip: str) -> None:
# ── Endpoints ─────────────────────────────────────────────────────────────
def _require_login_method(method: str) -> None:
settings = load_system_settings().authentication
if not getattr(settings, method):
labels = {"password_login_enabled": "账号密码登录", "registration_enabled": "自助注册", "sso_enabled": "外部系统免登"}
raise HTTPException(status_code=403, detail=f"管理员尚未开启{labels[method]}")
def _login_response(user: User, token: str, *, username: str | None = None, created: bool = False) -> UsernameLoginResponse:
return UsernameLoginResponse(
access_token=token,
expires_in=get_auth_config().token_expiry_days * 24 * 3600,
user_id=str(user.id),
email=user.email,
system_role=user.system_role,
needs_setup=user.needs_setup,
created=created,
username=username or user.email.split("@", 1)[0],
)
async def _prepare_login_session(user: User) -> User:
"""Apply the administrator's one-online-session policy after authentication."""
if not load_system_settings().authentication.single_session_enabled:
return user
try:
return await get_local_provider().start_login_session(user)
except CredentialConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
class AuthenticationOptions(AuthenticationSettings):
needs_setup: bool = False
@router.get("/options", response_model=AuthenticationOptions)
async def authentication_options(request: Request, response: Response) -> AuthenticationOptions:
"""Anonymous, non-secret capability discovery for the login page."""
response.headers["Cache-Control"] = "no-store"
trace = trace_request(request, "auth_options")
try:
stage_started = time.perf_counter()
authentication = load_system_settings().authentication.model_dump()
trace_stage(trace, "load_system_settings", stage_started)
stage_started = time.perf_counter()
needs_setup = await get_local_provider().count_admin_users() == 0
trace_stage(trace, "count_admin_users", stage_started)
return AuthenticationOptions(**authentication, needs_setup=needs_setup)
finally:
finish_trace(trace)
@router.post("/login/local", response_model=UsernameLoginResponse)
def _require_login_method(method: str) -> None:
settings = load_system_settings().authentication
if not getattr(settings, method):
labels = {"password_login_enabled": "账号密码登录", "registration_enabled": "自助注册", "sso_enabled": "外部系统免登"}
raise HTTPException(status_code=403, detail=f"管理员尚未开启{labels[method]}")
def _login_response(user: User, token: str, *, username: str | None = None, created: bool = False) -> UsernameLoginResponse:
return UsernameLoginResponse(
access_token=token,
expires_in=get_auth_config().token_expiry_days * 24 * 3600,
user_id=str(user.id),
email=user.email,
system_role=user.system_role,
needs_setup=user.needs_setup,
created=created,
username=username or user.email.split("@", 1)[0],
)
async def _prepare_login_session(user: User) -> User:
"""Apply the administrator's one-online-session policy after authentication."""
if not load_system_settings().authentication.single_session_enabled:
return user
try:
return await get_local_provider().start_login_session(user)
except CredentialConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
class AuthenticationOptions(AuthenticationSettings):
needs_setup: bool = False
@router.get("/options", response_model=AuthenticationOptions)
async def authentication_options(response: Response) -> AuthenticationOptions:
"""Anonymous, non-secret capability discovery for the login page."""
response.headers["Cache-Control"] = "no-store"
return AuthenticationOptions(
**load_system_settings().authentication.model_dump(),
needs_setup=await get_local_provider().count_admin_users() == 0,
)
@router.post("/login/local", response_model=UsernameLoginResponse)
async def login_local(
request: Request,
response: Response,
form_data: OAuth2PasswordRequestForm = Depends(),
):
"""Username or email login with the account's unified password/passphrase."""
_require_login_method("password_login_enabled")
"""Username or email login with the account's unified password/passphrase."""
_require_login_method("password_login_enabled")
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
user = await _lookup_user_by_username(form_data.username)
if user is not None and not await verify_account_password(form_data.password, user):
user = None
user = await _lookup_user_by_username(form_data.username)
if user is not None and not await verify_account_password(form_data.password, user):
user = None
if user is None:
_record_login_failure(client_ip)
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="账号或密码错误").model_dump(),
detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="账号或密码错误").model_dump(),
)
_record_login_success(client_ip)
enforce_user_approved(user)
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
_set_session_cookie(response, token, request)
_sync_user_position(request, user.id)
await _sync_user_position(request, user.id)
return _login_response(user, token, username=form_data.username.strip())
def _username_login_request(username: object) -> UsernameLoginRequest:
if not isinstance(username, str):
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="username is required")
try:
return UsernameLoginRequest(username=username)
except ValidationError as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="username is required") from exc
async def _read_username_login(request: Request) -> UsernameLoginRequest:
"""Accept the historical JSON body and a form body.
``application/x-www-form-urlencoded`` is a CORS simple content type. Old
Firefox aborts a credentialed JSON login during preflight and reports
NS_BINDING_ABORTED, so the browser client sends a form body instead.
"""
media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media_type == "application/x-www-form-urlencoded":
form = await request.form()
return _username_login_request(form.get("username"))
try:
payload = await request.json()
except json.JSONDecodeError as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="username is required") from exc
username = payload.get("username") if isinstance(payload, dict) else None
return _username_login_request(username)
return _login_response(user, token, username=form_data.username.strip())
@router.post("/login/username", response_model=UsernameLoginResponse)
async def login_by_username(
request: Request,
response: Response,
password: str | None = Query(default=None, description="Account password or shared passphrase; required when password login is enabled."),
body: UsernameLoginRequest,
password: str | None = Query(default=None, description="Account password or shared passphrase; required when password login is enabled."),
):
"""Compatibility username login, gated by the current credential policy.
Personal password/passphrase wins. With password login enabled, accounts
without personal credentials must supply the existing shared passphrase
(default 123ewq). Identity-only login is retained only while password login
is disabled and the historical shared-password gate is also disabled.
"""Compatibility username login, gated by the current credential policy.
Personal password/passphrase wins. With password login enabled, accounts
without personal credentials must supply the existing shared passphrase
(default 123ewq). Identity-only login is retained only while password login
is disabled and the historical shared-password gate is also disabled.
"""
body = await _read_username_login(request)
_require_login_method("sso_enabled")
_require_login_method("sso_enabled")
username_login_config = get_app_config().auth_login.username_login
password_login_enabled = load_system_settings().authentication.password_login_enabled
password_login_enabled = load_system_settings().authentication.password_login_enabled
# Per-account password takes precedence: look the account up (no create) and,
# if it carries a personal 登录口令, require an exact match — ignoring the
# shared gate entirely for that account.
existing = await _lookup_user_by_username(body.username)
credentials_verified = False
if has_account_password(existing) or password_login_enabled or username_login_config.require_password:
credentials_verified = False
if has_account_password(existing) or password_login_enabled or username_login_config.require_password:
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
if not await verify_account_password(password or "", existing):
if not await verify_account_password(password or "", existing):
_record_login_failure(client_ip)
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="口令错误").model_dump(),
)
credentials_verified = True
credentials_verified = True
_record_login_success(client_ip)
user, created = await _get_or_create_user_by_username(body.username, existing_user=existing)
# A registration/reset may commit after the initial lookup/password gate.
# Never mint the new account/version's session using an earlier decision.
if has_account_password(user) and (existing is None or existing.id != user.id or existing.token_version != user.token_version):
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
if not await verify_account_password(password or "", user):
_record_login_failure(client_ip)
raise HTTPException(status_code=401, detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="口令错误").model_dump())
credentials_verified = True
user, created = await _get_or_create_user_by_username(body.username)
# A registration/reset may commit after the initial lookup/password gate.
# Never mint the new account/version's session using an earlier decision.
if has_account_password(user) and (existing is None or existing.id != user.id or existing.token_version != user.token_version):
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
if not await verify_account_password(password or "", user):
_record_login_failure(client_ip)
raise HTTPException(status_code=401, detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="口令错误").model_dump())
credentials_verified = True
enforce_user_approved(user)
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="passphrase" if credentials_verified else None)
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="passphrase" if credentials_verified else None)
_set_session_cookie(response, token, request)
_sync_user_position(request, user.id)
await _sync_user_position(request, user.id)
return UsernameLoginResponse(
access_token=token,
@@ -803,7 +714,7 @@ async def login_by_token(
Returns the JWT access token in the response body and also sets it as a
session cookie for browser callers.
"""
_require_login_method("sso_enabled")
_require_login_method("sso_enabled")
config = get_app_config().auth_login.token_login
if not config.enabled:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="token 登录未开启")
@@ -823,10 +734,10 @@ async def login_by_token(
user, created = await _get_or_create_user_by_username(username)
enforce_user_approved(user)
user = await _prepare_login_session(user)
access_token = create_access_token(str(user.id), token_version=user.token_version, auth_method="sso")
user = await _prepare_login_session(user)
access_token = create_access_token(str(user.id), token_version=user.token_version, auth_method="sso")
_set_session_cookie(response, access_token, request)
_sync_user_position(request, user.id)
await _sync_user_position(request, user.id)
return UsernameLoginResponse(
access_token=access_token,
@@ -843,41 +754,41 @@ async def login_by_token(
)
@router.post("/register", response_model=UsernameLoginResponse, status_code=status.HTTP_201_CREATED)
@router.post("/register", response_model=UsernameLoginResponse, status_code=status.HTTP_201_CREATED)
async def register(request: Request, response: Response, body: RegisterRequest):
"""Register a new user account (always 'user' role).
Admin is auto-created on first boot. This endpoint creates regular users.
Auto-login by setting the session cookie.
"""
_require_login_method("password_login_enabled")
_require_login_method("registration_enabled")
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
provider = get_local_provider()
username = body.username or str(body.email)
email = _username_to_email(username)
# Match every domain, even for explicit emails: a self-registration must
# never shadow the stable account selected by bare-username login.
duplicate = await provider.find_user_by_email_local_part(email.split("@", 1)[0])
_require_login_method("password_login_enabled")
_require_login_method("registration_enabled")
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
provider = get_local_provider()
username = body.username or str(body.email)
email = _username_to_email(username)
# Match every domain, even for explicit emails: a self-registration must
# never shadow the stable account selected by bare-username login.
duplicate = await provider.find_user_by_email_local_part(email.split("@", 1)[0])
try:
if duplicate is not None:
raise ValueError("Duplicate username")
user = await provider.create_user(email=email, password=body.password, system_role="user", approved=True)
if duplicate is not None:
raise ValueError("Duplicate username")
user = await provider.create_user(email=email, password=body.password, system_role="user", approved=True)
except ValueError:
_record_login_failure(client_ip)
_record_login_failure(client_ip)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=AuthErrorResponse(code=AuthErrorCode.EMAIL_ALREADY_EXISTS, message="该用户名或邮箱已注册").model_dump(),
status_code=status.HTTP_409_CONFLICT,
detail=AuthErrorResponse(code=AuthErrorCode.EMAIL_ALREADY_EXISTS, message="该用户名或邮箱已注册").model_dump(),
)
enforce_user_approved(user)
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
enforce_user_approved(user)
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
_set_session_cookie(response, token, request)
_sync_user_position(request, user.id)
await _sync_user_position(request, user.id)
return _login_response(user, token, username=username, created=True)
return _login_response(user, token, username=username, created=True)
@router.post("/logout", response_model=MessageResponse)
@@ -887,7 +798,7 @@ async def logout(request: Request, response: Response):
return MessageResponse(message="Successfully logged out")
@router.post("/change-password", response_model=PasswordChangeResponse)
@router.post("/change-password", response_model=PasswordChangeResponse)
async def change_password(request: Request, response: Response, body: ChangePasswordRequest):
"""Change password for the currently authenticated user.
@@ -898,45 +809,45 @@ async def change_password(request: Request, response: Response, body: ChangePass
- Re-issues session cookie with new token_version
"""
user = await get_current_user_from_request(request)
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
if not await verify_account_password(body.current_password, user):
_record_login_failure(client_ip)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="当前密码错误").model_dump())
client_ip = _get_client_ip(request)
_check_rate_limit(client_ip)
if not await verify_account_password(body.current_password, user):
_record_login_failure(client_ip)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="当前密码错误").model_dump())
provider = get_local_provider()
# Update email if provided
if body.new_email is not None:
existing = await provider.find_user_by_email_local_part(body.new_email.split("@", 1)[0])
existing = await provider.find_user_by_email_local_part(body.new_email.split("@", 1)[0])
if existing and str(existing.id) != str(user.id):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=AuthErrorResponse(code=AuthErrorCode.EMAIL_ALREADY_EXISTS, message="Email already in use").model_dump())
try:
user = await provider.update_credentials(user, await hash_password_async(body.new_password), email=body.new_email)
except ValueError as exc:
raise HTTPException(status_code=409, detail="账号信息已变更,请刷新后重试") from exc
_record_login_success(client_ip)
try:
user = await provider.update_credentials(user, await hash_password_async(body.new_password), email=body.new_email)
except ValueError as exc:
raise HTTPException(status_code=409, detail="账号信息已变更,请刷新后重试") from exc
_record_login_success(client_ip)
# Re-issue cookie with new token_version
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
_set_session_cookie(response, token, request)
return PasswordChangeResponse(message="密码已更新", access_token=token, expires_in=get_auth_config().token_expiry_days * 24 * 3600)
return PasswordChangeResponse(message="密码已更新", access_token=token, expires_in=get_auth_config().token_expiry_days * 24 * 3600)
@router.get("/me", response_model=UserResponse)
async def get_me(request: Request):
"""Get current authenticated user info."""
user = await get_current_user_from_request(request)
return UserResponse(
id=str(user.id),
email=user.email,
system_role=user.system_role,
needs_setup=user.needs_setup,
has_password=has_account_password(user),
requires_current_password=True,
uses_default_password=not has_account_password(user),
)
return UserResponse(
id=str(user.id),
email=user.email,
system_role=user.system_role,
needs_setup=user.needs_setup,
has_password=has_account_password(user),
requires_current_password=True,
uses_default_password=not has_account_password(user),
)
_SETUP_STATUS_COOLDOWN: dict[str, float] = {}
@@ -978,12 +889,12 @@ class InitializeAdminRequest(BaseModel):
"""Request model for first-boot admin account creation."""
email: EmailStr
password: str = Field(..., min_length=8, max_length=255)
password: str = Field(..., min_length=8, max_length=255)
_strong_password = field_validator("password")(classmethod(lambda cls, v: _validate_strong_password(v)))
@router.post("/initialize", response_model=UsernameLoginResponse, status_code=status.HTTP_201_CREATED)
@router.post("/initialize", response_model=UsernameLoginResponse, status_code=status.HTTP_201_CREATED)
async def initialize_admin(request: Request, response: Response, body: InitializeAdminRequest):
"""Create the first admin account on initial system setup.
@@ -1001,7 +912,7 @@ async def initialize_admin(request: Request, response: Response, body: Initializ
)
try:
user = await get_local_provider().create_initial_admin(email=str(body.email), password=body.password)
user = await get_local_provider().create_initial_admin(email=str(body.email), password=body.password)
except ValueError:
# DB unique-constraint race: another concurrent request beat us.
raise HTTPException(
@@ -1009,11 +920,11 @@ async def initialize_admin(request: Request, response: Response, body: Initializ
detail=AuthErrorResponse(code=AuthErrorCode.SYSTEM_ALREADY_INITIALIZED, message="System already initialized").model_dump(),
)
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
user = await _prepare_login_session(user)
token = create_access_token(str(user.id), token_version=user.token_version, auth_method="password")
_set_session_cookie(response, token, request)
return _login_response(user, token, created=True)
return _login_response(user, token, created=True)
# ── Admin Endpoints ───────────────────────────────────────────────────────
@@ -1027,99 +938,54 @@ class UserListItem(BaseModel):
agent_count: int
qa_count: int = 0 # 问答次数: number of runs (one run = one question)
approved: bool = True # 登录白名单放行标记(白名单管理页据此放行/取消)
login_password: str | None = None # Legacy plaintext only; never return hashes.
has_login_password: bool = False
access_token: str | None = None
token_type: str = "bearer"
expires_in: int | None = None
login_password: str | None = None # Legacy plaintext only; never return hashes.
has_login_password: bool = False
access_token: str | None = None
token_type: str = "bearer"
expires_in: int | None = None
def _legacy_login_password(user: User) -> str | None:
value = user.login_password
return value if value and not is_password_hash(value) else None
def _legacy_login_password(user: User) -> str | None:
value = user.login_password
return value if value and not is_password_hash(value) else None
class UserListPage(BaseModel):
"""Paged user list — returned when the caller passes ``limit``."""
items: list[UserListItem]
total: int
approved_count: int # approved 或 admin(搜索范围内)
limit: int
offset: int
_USER_LIST_TIMEOUT_SECONDS = 15
def _user_list_skill_store(request: Request, include_counts: bool = True) -> SkillStore | None:
return get_skill_store(request) if include_counts else None
def _user_list_agent_store(request: Request, include_counts: bool = True) -> AgentStore | None:
return get_agent_store(request) if include_counts else None
@router.get("/users", response_model=list[UserListItem] | UserListPage)
@router.get("/users", response_model=list[UserListItem])
async def list_all_users(
request: Request,
include_counts: bool = Query(True, description="False=只查 users 表,跳过 runs 统计与 skill/agent 计数(白名单管理页用,避免慢查询)"),
q: str | None = Query(None, max_length=100, description="可选:按邮箱/用户ID 子串过滤(仅分页模式生效)"),
limit: int | None = Query(None, ge=1, le=200, description="传入即分页返回 UserListPage;不传保持旧的全量列表"),
offset: int = Query(0, ge=0),
skill_store: SkillStore | None = Depends(_user_list_skill_store),
agent_store: AgentStore | None = Depends(_user_list_agent_store),
skill_store: SkillStore = Depends(get_skill_store),
agent_store: AgentStore = Depends(get_agent_store),
):
"""List all users with their skill, agent and Q&A counts. Admin only.
Passing ``limit`` switches to a paged response that reads one page of the
users table plus two COUNT queries; omitting it keeps the legacy flat list.
"""
"""List all users with their skill, agent and Q&A counts. Admin only."""
user = await get_current_user_from_request(request)
if user.system_role != "admin":
raise HTTPException(status_code=403, detail="Admin only")
users = await get_local_provider().list_users()
# One GROUP BY query for everyone's 问答次数, so the list stays a single
# aggregate read instead of a per-user count. Best-effort: if the run store
# is unavailable the column simply renders as 0 rather than failing the list.
try:
async with asyncio.timeout(_USER_LIST_TIMEOUT_SECONDS if not include_counts else None):
user = await get_current_user_from_request(request)
if user.system_role != "admin":
raise HTTPException(status_code=403, detail="Admin only")
provider = get_local_provider()
users = (
await provider.list_users()
if limit is None
else await provider.list_users_page(q=q, limit=limit, offset=offset)
qa_counts = await get_run_store(request).count_by_user()
except Exception:
logger.warning("Failed to load per-user run counts for user list", exc_info=True)
qa_counts = {}
result = []
for u in users:
uid = str(u.id)
result.append(
UserListItem(
id=uid,
email=u.email,
system_role=u.system_role,
skill_count=await skill_store.count_owned(uid),
agent_count=await agent_store.count_owned(uid),
qa_count=qa_counts.get(uid, 0),
approved=getattr(u, "approved", True),
login_password=_legacy_login_password(u),
has_login_password=has_account_password(u),
)
qa_counts: dict[str, int] = {}
# One recent-window GROUP BY for everyone's 问答次数. This keeps the admin
# user list from scanning the complete lifetime of the runs table.
if include_counts:
try:
qa_counts = await get_run_store(request).count_by_user(
since=recent_admin_query_cutoff()
)
except Exception:
logger.warning("Failed to load per-user run counts for user list", exc_info=True)
result = []
for u in users:
uid = str(u.id)
result.append(
UserListItem(
id=uid,
email=u.email,
system_role=u.system_role,
skill_count=(await skill_store.count_owned(uid)) if include_counts else 0,
agent_count=(await agent_store.count_owned(uid)) if include_counts else 0,
qa_count=qa_counts.get(uid, 0),
approved=getattr(u, "approved", True),
login_password=_legacy_login_password(u),
has_login_password=has_account_password(u),
)
)
if limit is None:
return result
total = await provider.count_users_filtered(q=q)
approved_count = await provider.count_users_filtered(q=q, approved_or_admin=True)
return UserListPage(items=result, total=total, approved_count=approved_count, limit=limit, offset=offset)
except TimeoutError as exc:
logger.warning("User list timed out waiting for account data")
raise HTTPException(status_code=503, detail="用户基础信息查询超时,请稍后重试") from exc
)
return result
class UserApprovalRequest(BaseModel):
@@ -1142,10 +1008,10 @@ async def set_user_approval(user_id: str, body: UserApprovalRequest, request: Re
raise HTTPException(status_code=404, detail="用户不存在")
target.approved = bool(body.approved)
try:
saved = await provider.update_user(target)
except CredentialConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
try:
saved = await provider.update_user(target)
except CredentialConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
return UserListItem(
id=str(saved.id),
email=saved.email,
@@ -1154,8 +1020,8 @@ async def set_user_approval(user_id: str, body: UserApprovalRequest, request: Re
agent_count=0,
qa_count=0,
approved=saved.approved,
login_password=_legacy_login_password(saved),
has_login_password=has_account_password(saved),
login_password=_legacy_login_password(saved),
has_login_password=has_account_password(saved),
)
@@ -1163,14 +1029,14 @@ class UserLoginPasswordRequest(BaseModel):
password: str | None = Field(
default=None,
max_length=255,
description="设置账号密码(与登录口令一致);不允许清除后退回免密码登录。",
description="设置账号密码(与登录口令一致);不允许清除后退回免密码登录。",
)
@router.put("/users/{user_id}/login-password", response_model=UserListItem)
@router.put("/users/{user_id}/reset-password", response_model=UserListItem)
async def set_user_login_password(user_id: str, body: UserLoginPasswordRequest, request: Request, response: Response) -> UserListItem:
"""Admin password reset; rotates both credential columns and all sessions."""
@router.put("/users/{user_id}/reset-password", response_model=UserListItem)
async def set_user_login_password(user_id: str, body: UserLoginPasswordRequest, request: Request, response: Response) -> UserListItem:
"""Admin password reset; rotates both credential columns and all sessions."""
admin = await get_current_user_from_request(request)
if admin.system_role != "admin":
raise HTTPException(status_code=403, detail="Admin only")
@@ -1180,25 +1046,25 @@ async def set_user_login_password(user_id: str, body: UserLoginPasswordRequest,
if target is None:
raise HTTPException(status_code=404, detail="用户不存在")
pwd = body.password or ""
if not pwd:
raise HTTPException(status_code=400, detail="不能清除账号密码,请设置新密码")
if len(pwd) < 8:
raise HTTPException(status_code=400, detail="密码至少需要 8 个字符")
try:
_validate_strong_password(pwd)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
try:
saved = await provider.update_credentials(target, await hash_password_async(pwd))
except CredentialConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
token = None
expires_in = None
if str(admin.id) == str(saved.id):
token = create_access_token(str(saved.id), token_version=saved.token_version, auth_method="password")
_set_session_cookie(response, token, request)
expires_in = get_auth_config().token_expiry_days * 24 * 3600
pwd = body.password or ""
if not pwd:
raise HTTPException(status_code=400, detail="不能清除账号密码,请设置新密码")
if len(pwd) < 8:
raise HTTPException(status_code=400, detail="密码至少需要 8 个字符")
try:
_validate_strong_password(pwd)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
try:
saved = await provider.update_credentials(target, await hash_password_async(pwd))
except CredentialConflictError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
token = None
expires_in = None
if str(admin.id) == str(saved.id):
token = create_access_token(str(saved.id), token_version=saved.token_version, auth_method="password")
_set_session_cookie(response, token, request)
expires_in = get_auth_config().token_expiry_days * 24 * 3600
return UserListItem(
id=str(saved.id),
email=saved.email,
@@ -1207,10 +1073,10 @@ async def set_user_login_password(user_id: str, body: UserLoginPasswordRequest,
agent_count=0,
qa_count=0,
approved=saved.approved,
login_password=None,
has_login_password=True,
access_token=token,
expires_in=expires_in,
login_password=None,
has_login_password=True,
access_token=token,
expires_in=expires_in,
)
@@ -1224,7 +1090,7 @@ async def oauth_login(provider: str):
Redirects to the OAuth provider's authorization URL.
Currently a placeholder - requires OAuth provider implementation.
"""
_require_login_method("sso_enabled")
_require_login_method("sso_enabled")
if provider not in ["github", "google"]:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
@@ -1244,7 +1110,7 @@ async def oauth_callback(provider: str, code: str, state: str):
Handles the OAuth provider's callback after user authorization.
Currently a placeholder.
"""
_require_login_method("sso_enabled")
_require_login_method("sso_enabled")
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED,
detail="OAuth callback not yet implemented",
@@ -286,23 +286,8 @@ async def _sync_to_sandbox_if_needed(thread_id: str, virtual_path: str, file_pat
async def get_browser_context_status(
config: AppConfig = Depends(get_config),
) -> BrowserContextStatusResponse:
from app.gateway.api_read_cache import get_cache_version, get_cached_json, is_browser_status_enabled, set_cached_json
# Access control must precede even a warm Redis hit.
if not await is_browser_status_enabled(config):
raise HTTPException(
status_code=403,
detail={"code": "browser_status_disabled", "message": "管理员已关闭浏览器状态接口"},
headers={"Retry-After": "300", "Cache-Control": "no-store"},
)
settings = _settings(config)
# Include the effective config so reloads cannot reuse obsolete capabilities.
cache_key = settings.model_dump_json()
version = await get_cache_version(config, "browser_status")
cached = await get_cached_json(config, "browser_status", cache_key, version=version)
if cached is not None:
return BrowserContextStatusResponse.model_validate(cached)
response = BrowserContextStatusResponse(
return BrowserContextStatusResponse(
enabled=settings.enabled,
max_content_chars=settings.max_content_chars,
max_title_chars=settings.max_title_chars,
@@ -312,8 +297,6 @@ async def get_browser_context_status(
task_lease_seconds=settings.task_lease_seconds,
tool_result_max_chars=settings.tool_result_max_chars,
)
await set_cached_json(config, "browser_status", cache_key, response, expected_version=version)
return response
@router.get("/tasks/next", response_model=BrowserContextTaskNextResponse)
@@ -16,13 +16,11 @@ Routes (prefix ``/api/business-mapping``):
from __future__ import annotations
import logging
import re
from datetime import datetime
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.catalog_cache import cached_catalog, invalidate_catalog
from app.gateway.deps import get_agent_store
from deerflow.runtime.user_context import get_effective_user_id
@@ -39,8 +37,6 @@ _EIGHTBF_MANAGER_USERNAME = "lqq"
class BusinessMappingResponse(BaseModel):
business_code: str
label: str = ""
mapping_type: str = "内置业务"
is_custom: bool = False
agent_id: str | None = None
agent_name: str | None = None
chain_id: str | None = None
@@ -62,12 +58,6 @@ class BusinessMappingUpdateRequest(BaseModel):
chain_title: str | None = Field(default=None, max_length=512)
class BusinessMappingCreateRequest(BaseModel):
"""The name is both the display name and login ``goPath`` for a custom row."""
name: str = Field(min_length=1, max_length=32)
# ── helpers ──────────────────────────────────────────────────────────────
@@ -105,23 +95,6 @@ def _current_user_id(request: Request) -> str:
return get_effective_user_id()
_RESERVED_GOPATHS = frozenset({"3qfx", "rwfx", "xdfx", "agentfx", "b5-1"})
def _normalise_custom_mapping_input(body: BusinessMappingCreateRequest) -> str:
"""Validate a custom goPath without rewriting the administrator's label."""
name = re.sub(r"\s+", " ", body.name).strip()
if not name:
raise HTTPException(status_code=422, detail="名称不能为空")
# ``-A`` is a mode suffix, not part of a configurable name: ``name`` is
# single-agent and ``name-A`` is the same mapping's business-chain entry.
if name.lower().endswith("-a"):
raise HTTPException(status_code=422, detail="自定义名称不能以 -A 结尾")
if name.lower() in _RESERVED_GOPATHS:
raise HTTPException(status_code=422, detail="该名称已被内置 goPath 保留")
return name
async def _validate_public_chain_mapping(
request: Request,
fields: dict[str, object],
@@ -194,32 +167,8 @@ async def _validate_public_chain_mapping(
@router.get("", response_model=BusinessMappingListResponse)
async def list_mappings(request: Request) -> BusinessMappingListResponse:
store = _get_store(request)
async def load() -> BusinessMappingListResponse:
rows = await store.list_mappings()
return BusinessMappingListResponse(mappings=[BusinessMappingResponse(**r) for r in rows])
return await cached_catalog(request, "business_mapping", "all", load, BusinessMappingListResponse)
@router.post("", response_model=BusinessMappingResponse, status_code=201)
async def create_mapping(
request: Request,
body: BusinessMappingCreateRequest,
) -> BusinessMappingResponse:
"""Create a globally shared custom mapping (administrator only)."""
if not _is_admin(request):
raise HTTPException(status_code=403, detail="Only administrators can create business mappings")
name = _normalise_custom_mapping_input(body)
store = _get_store(request)
row = await store.create_custom_mapping(
business_code=name,
mapping_type="自定义类型",
)
if row is None:
raise HTTPException(status_code=409, detail="该名称已存在,请换一个名称")
await invalidate_catalog(request, "business_mapping")
return BusinessMappingResponse(**row)
rows = await store.list_mappings()
return BusinessMappingListResponse(mappings=[BusinessMappingResponse(**r) for r in rows])
@router.put("/{business_code}", response_model=BusinessMappingResponse)
@@ -234,5 +183,4 @@ async def update_mapping(request: Request, business_code: str, body: BusinessMap
row = await store.update_mapping(business_code, **fields)
if row is None:
raise HTTPException(status_code=404, detail="Business mapping not found")
await invalidate_catalog(request, "business_mapping")
return BusinessMappingResponse(**row)
@@ -1,762 +0,0 @@
"""Admin controls for resumable monthly PostgreSQL chat archives."""
from __future__ import annotations
import asyncio
import contextlib
import errno
import logging
import shutil
import sqlite3
import uuid
from datetime import datetime
from pathlib import Path
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel
from sqlalchemy import extract, func, select
from app.gateway.auth.disabled_mode import is_auth_disabled
from app.gateway.deps import get_current_user_from_request
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.types import BEIJING_TZ
from deerflow.runtime.checkpointer.chat_archive import (
archive_db_path,
archive_root,
ensure_catalog,
thread_archive_lock,
try_acquire_archive_job_lock,
)
from deerflow.runtime.checkpointer.checkpoint_migration import _sqlite_has_table, copy_thread_state
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/chat-archive", tags=["chat-archive"])
_ACTIVE = {"queued", "running", "cancelling"}
_PAGE_SIZE = 40
_ARCHIVE_DISK_RESERVE_BYTES = 2 * 1024**3
_PURGE_THREAD_DELAY_SECONDS = 0.5
_PROCESS_STARTED_AT = datetime.now(BEIJING_TZ).isoformat(timespec="seconds")
class _ArchiveJobCancelled(Exception):
pass
class ChatArchiveJobResponse(BaseModel):
job_id: str
month: str
action: str
status: str
total_threads: int
processed_threads: int
failed_threads: int
current_thread_id: str | None = None
snapshot_at: str = ""
processed_through: str | None = None
message: str = ""
error: str | None = None
created_at: str
updated_at: str
async def _require_admin(request: Request) -> None:
if is_auth_disabled():
return
user = await get_current_user_from_request(request)
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="Chat archive management requires admin role")
def _now() -> str:
return datetime.now(BEIJING_TZ).isoformat(timespec="seconds")
def _parse_month(value: str) -> tuple[datetime, datetime]:
try:
start = datetime.strptime(value, "%Y-%m")
except ValueError as exc:
raise HTTPException(status_code=400, detail="month must use YYYY-MM format") from exc
if start.strftime("%Y-%m") != value:
raise HTTPException(status_code=400, detail="month must use YYYY-MM format")
if start.month == 12:
end = start.replace(year=start.year + 1, month=1)
else:
end = start.replace(month=start.month + 1)
return start, end
def _primary_checkpointer(request: Request):
checkpointer = getattr(request.app.state, "checkpointer", None)
primary = getattr(checkpointer, "primary", checkpointer)
if primary is None or not hasattr(getattr(primary, "conn", None), "connection"):
raise HTTPException(status_code=409, detail="当前聊天检查点不是 PostgreSQL,不能执行 PG 归档")
return checkpointer, primary
def _catalog_connect(root: Path) -> sqlite3.Connection:
ensure_catalog(root)
conn = sqlite3.connect(root / "catalog.db", timeout=30)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout=30000")
return conn
def _recover_jobs(root: Path, *, started_before: str | None = _PROCESS_STARTED_AT, owns_operation_lock: bool = False) -> None:
operation_lock = None if owns_operation_lock else try_acquire_archive_job_lock(root)
if not owns_operation_lock and operation_lock is None:
return # Another gateway/worker still owns the running job.
cutoff_sql = " AND created_at < ?" if started_before else ""
cutoff_values = (started_before,) if started_before else ()
with (contextlib.nullcontext() if owns_operation_lock else operation_lock), _catalog_connect(root) as conn:
now = _now()
conn.execute(
"UPDATE archive_jobs SET status='interrupted', message='执行进程已退出,已保留成功归档;可重新归档或清理已校验部分', "
"error=COALESCE(error, '归档执行进程已退出或服务重启,任务未完成'), updated_at=? "
"WHERE status IN ('queued','running','cancelling')" + cutoff_sql,
(now, *cutoff_values),
)
conn.execute(
"UPDATE archive_months SET status='interrupted', updated_at=? "
"WHERE status IN ('copying','deleting','restoring') AND current_job_id IN "
"(SELECT job_id FROM archive_jobs WHERE status='interrupted'" + cutoff_sql + ")",
(now, *cutoff_values),
)
def _month_start(month: str) -> tuple[datetime, datetime]:
start, end = _parse_month(month)
return start.replace(tzinfo=BEIJING_TZ), end.replace(tzinfo=BEIJING_TZ)
async def _month_thread_count(session_factory, month: str, *, snapshot_at: str | None = None) -> int:
start, end = _month_start(month)
conditions = [ThreadMetaRow.created_at >= start, ThreadMetaRow.created_at < end]
if snapshot_at:
conditions.append(ThreadMetaRow.created_at <= datetime.fromisoformat(snapshot_at))
async with session_factory() as session:
return int(await session.scalar(
select(func.count()).select_from(ThreadMetaRow).where(*conditions)
) or 0)
async def _thread_page(session_factory, month: str, offset: int, limit: int, *, snapshot_at: str | None = None):
start, end = _month_start(month)
conditions = [ThreadMetaRow.created_at >= start, ThreadMetaRow.created_at < end]
if snapshot_at:
conditions.append(ThreadMetaRow.created_at <= datetime.fromisoformat(snapshot_at))
async with session_factory() as session:
result = await session.execute(
select(
ThreadMetaRow.thread_id,
ThreadMetaRow.status,
ThreadMetaRow.metadata_json,
ThreadMetaRow.created_at,
)
.where(*conditions)
.order_by(ThreadMetaRow.created_at, ThreadMetaRow.thread_id)
.offset(offset)
.limit(limit)
)
return list(result.all())
async def _thread_status(session_factory, thread_id: str) -> str | None:
async with session_factory() as session:
value = await session.scalar(
select(ThreadMetaRow.status).where(ThreadMetaRow.thread_id == thread_id).limit(1)
)
return str(value).lower() if value is not None else None
def _archive_action_thread_count(root: Path, month: str, action: str) -> int:
"""Count only archived threads that the requested action will process."""
if not (root / "catalog.db").is_file():
return 0
if action == "purge":
statuses = ("copied", "deleted", "restored")
placeholders = ",".join("?" for _ in statuses)
query = f"SELECT COUNT(*) FROM archive_threads WHERE month=? AND status IN ({placeholders})"
params = (month, *statuses)
elif action == "rollback":
query = "SELECT COUNT(*) FROM archive_threads WHERE month=?"
params = (month,)
else:
return 0
with _catalog_connect(root) as conn:
row = conn.execute(query, params).fetchone()
return int(row[0] or 0)
async def _month_inventory(session_factory) -> list[dict[str, Any]]:
async with session_factory() as session:
dialect = session.get_bind().dialect.name
created_at = ThreadMetaRow.created_at
if dialect in {"postgresql", "opengauss"}:
local_created = func.timezone("Asia/Shanghai", created_at)
else:
local_created = created_at
year_expr = extract("year", local_created).label("year")
month_expr = extract("month", local_created).label("month")
rows = await session.execute(
select(year_expr, month_expr, func.count())
.select_from(ThreadMetaRow)
.group_by(year_expr, month_expr)
.order_by(year_expr.desc(), month_expr.desc())
)
return [
{"month": f"{int(year):04d}-{int(month):02d}", "thread_count": int(count)}
for year, month, count in rows.all()
if year is not None and month is not None
]
def _catalog_months(root: Path) -> dict[str, dict[str, Any]]:
if not (root / "catalog.db").exists():
return {}
with _catalog_connect(root) as conn:
months = {
row["month"]: dict(row)
for row in conn.execute("SELECT * FROM archive_months")
}
for row in conn.execute(
"SELECT j.* FROM archive_jobs j WHERE j.rowid=(SELECT newest.rowid FROM archive_jobs newest "
"WHERE newest.month=j.month ORDER BY newest.created_at DESC, newest.rowid DESC LIMIT 1)"
):
months.setdefault(row["month"], {})["latest_job"] = dict(row)
return months
def _save_job(root: Path, job: dict[str, Any]) -> None:
with _catalog_connect(root) as conn:
conn.execute(
"INSERT INTO archive_jobs(job_id,month,action,status,total_threads,processed_threads,failed_threads,"
"current_thread_id,snapshot_at,processed_through,message,error,created_at,updated_at) "
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
tuple((job.get(key) or job.get("created_at") or "") if key == "snapshot_at" else job.get(key) for key in (
"job_id", "month", "action", "status", "total_threads", "processed_threads",
"failed_threads", "current_thread_id", "snapshot_at", "processed_through", "message", "error", "created_at", "updated_at",
)),
)
def _claim_month_job(root: Path, job: dict[str, Any], month_status: str) -> bool:
"""Atomically allow only one heavy archive operation across processes."""
conn = _catalog_connect(root)
try:
conn.execute("BEGIN IMMEDIATE")
active = conn.execute(
"SELECT job_id FROM archive_jobs WHERE status IN ('queued','running','cancelling') LIMIT 1"
).fetchone()
if active:
conn.rollback()
return False
conn.execute(
"INSERT INTO archive_jobs(job_id,month,action,status,total_threads,processed_threads,failed_threads,"
"current_thread_id,snapshot_at,processed_through,message,error,created_at,updated_at) "
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
tuple((job.get(key) or job.get("created_at") or "") if key == "snapshot_at" else job.get(key) for key in (
"job_id", "month", "action", "status", "total_threads", "processed_threads",
"failed_threads", "current_thread_id", "snapshot_at", "processed_through", "message", "error", "created_at", "updated_at",
)),
)
conn.execute(
"INSERT INTO archive_months(month,status,total_threads,copied_threads,deleted_threads,restored_threads,current_job_id,updated_at) "
"VALUES(?,?,?,0,0,0,?,?) ON CONFLICT(month) DO UPDATE SET status=excluded.status, "
"total_threads=excluded.total_threads,current_job_id=excluded.current_job_id,updated_at=excluded.updated_at",
(job["month"], month_status, job["total_threads"], job["job_id"], _now()),
)
conn.commit()
return True
except Exception:
conn.rollback()
raise
finally:
conn.close()
def _patch_job(root: Path, job_id: str, **values) -> None:
values["updated_at"] = _now()
assignments = ",".join(f"{key}=?" for key in values)
with _catalog_connect(root) as conn:
conn.execute(f"UPDATE archive_jobs SET {assignments} WHERE job_id=?", (*values.values(), job_id))
async def _patch_job_async(root: Path, job_id: str, **values) -> None:
await asyncio.to_thread(_patch_job, root, job_id, **values)
def _job_dict(root: Path, job_id: str) -> dict[str, Any] | None:
if not (root / "catalog.db").exists():
return None
with _catalog_connect(root) as conn:
row = conn.execute("SELECT * FROM archive_jobs WHERE job_id=?", (job_id,)).fetchone()
return dict(row) if row else None
def _upsert_month(root: Path, month: str, *, status: str, total: int, job_id: str | None = None) -> None:
with _catalog_connect(root) as conn:
conn.execute(
"INSERT INTO archive_months(month,status,total_threads,copied_threads,deleted_threads,restored_threads,current_job_id,updated_at) "
"VALUES(?,?,?,0,0,0,?,?) ON CONFLICT(month) DO UPDATE SET status=excluded.status, "
"total_threads=excluded.total_threads,current_job_id=excluded.current_job_id,updated_at=excluded.updated_at",
(month, status, total, job_id, _now()),
)
async def _upsert_month_async(root: Path, month: str, *, status: str, total: int, job_id: str | None = None) -> None:
await asyncio.to_thread(_upsert_month, root, month, status=status, total=total, job_id=job_id)
def _mark_archived_thread(root: Path, thread_id: str, month: str, count: int, status: str = "copied") -> None:
now = _now()
with _catalog_connect(root) as conn:
conn.execute(
"INSERT INTO archive_threads(thread_id,month,status,checkpoint_count,archived_at,deleted_at,restored_at) "
"VALUES(?,?,?,?,?,?,?) ON CONFLICT(thread_id) DO UPDATE SET month=excluded.month,status=excluded.status,"
"checkpoint_count=excluded.checkpoint_count,archived_at=excluded.archived_at,"
"deleted_at=CASE WHEN excluded.status='deleted' THEN excluded.archived_at ELSE archive_threads.deleted_at END,"
"restored_at=CASE WHEN excluded.status='restored' THEN excluded.archived_at ELSE archive_threads.restored_at END",
(thread_id, month, status, count, now, now if status == "deleted" else None, now if status == "restored" else None),
)
conn.execute(
"UPDATE archive_months SET copied_threads=(SELECT COUNT(*) FROM archive_threads WHERE month=?), "
"deleted_threads=(SELECT COUNT(*) FROM archive_threads WHERE month=? AND status='deleted'), updated_at=? WHERE month=?",
(month, month, now, month),
)
async def _checkpoint_ids(saver, thread_id: str) -> list[str]:
config = {"configurable": {"thread_id": thread_id}}
ids: list[str] = []
async for tup in saver.alist(config):
configurable = (tup.config or {}).get("configurable", {}) or {}
checkpoint_id = configurable.get("checkpoint_id") or (tup.checkpoint or {}).get("id")
if checkpoint_id:
ids.append(str(checkpoint_id))
return ids
@contextlib.asynccontextmanager
async def _open_sqlite_archive(path: Path):
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
path.parent.mkdir(parents=True, exist_ok=True)
context = AsyncSqliteSaver.from_conn_string(str(path))
saver = await context.__aenter__()
try:
await saver.conn.execute("PRAGMA busy_timeout=30000")
if not await asyncio.to_thread(_sqlite_has_table, str(path), "checkpoints"):
await saver.conn.execute("PRAGMA journal_mode=WAL")
await saver.setup()
yield saver
finally:
await context.__aexit__(None, None, None)
class ChatArchiveRunner:
def __init__(self, root: Path, checkpointer, session_factory):
self.root = root
self.checkpointer = checkpointer
self.primary = getattr(checkpointer, "primary", checkpointer)
self.session_factory = session_factory
self._locks: dict[str, asyncio.Lock] = {}
self._tasks: dict[str, asyncio.Task] = {}
async def start(self, month: str, action: str) -> dict[str, Any]:
_parse_month(month)
if action not in {"archive", "purge", "rollback"}:
raise HTTPException(status_code=400, detail="未知归档操作")
current_month = datetime.now(BEIJING_TZ).strftime("%Y-%m")
if month > current_month:
raise HTTPException(status_code=409, detail="不能归档未来月份")
snapshot_at = _now()
if action == "archive":
total = await _month_thread_count(self.session_factory, month, snapshot_at=snapshot_at)
else:
total = await asyncio.to_thread(_archive_action_thread_count, self.root, month, action)
if not total:
raise HTTPException(status_code=409, detail="该月份尚无已校验归档,不能删除或回滚;请先归档")
job_id = uuid.uuid4().hex
job = {
"job_id": job_id,
"month": month,
"action": action,
"status": "queued",
"total_threads": total,
"processed_threads": 0,
"failed_threads": 0,
"current_thread_id": None,
"snapshot_at": snapshot_at,
"processed_through": None,
"message": "任务已排队",
"error": None,
"created_at": _now(),
"updated_at": _now(),
}
status = {"archive": "copying", "purge": "deleting", "rollback": "restoring"}[action]
operation_lock = try_acquire_archive_job_lock(self.root)
if operation_lock is None:
raise HTTPException(status_code=409, detail="已有月份归档、清理或回滚任务正在运行,请完成后再启动下一项")
try:
# The kernel lock is free only if no runner is alive, including
# one that died after this gateway process finished starting.
_recover_jobs(self.root, started_before=None, owns_operation_lock=True)
claimed = _claim_month_job(self.root, job, status)
if not claimed:
raise HTTPException(status_code=409, detail="已有月份归档、清理或回滚任务正在运行,请完成后再启动下一项")
task = asyncio.create_task(self._run(job_id, owns_operation_lock=True), name=f"chat-archive-{action}-{month}")
except BaseException:
operation_lock.close()
raise
self._tasks[job_id] = task
def completed(_task):
operation_lock.close()
self._tasks.pop(job_id, None)
task.add_done_callback(completed)
return job
async def request_cancel(self, job_id: str) -> dict[str, Any]:
job = await asyncio.to_thread(_job_dict, self.root, job_id)
if not job:
raise HTTPException(status_code=404, detail="归档任务不存在")
if job["status"] in _ACTIVE:
await _patch_job_async(self.root, job_id, status="cancelling", message="将在当前会话处理结束后停止")
return await asyncio.to_thread(_job_dict, self.root, job_id)
async def _check_cancelled(self, job_id: str) -> None:
job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
if job.get("status") == "cancelling":
raise _ArchiveJobCancelled()
async def _record_thread_failure(self, job_id: str, tid: str, exc: Exception) -> None:
# A corrupt/full archive volume cannot safely accept more threads.
# Row-specific failures may continue; infrastructure failures must stop.
sqlite_code = getattr(exc, "sqlite_errorcode", 0) or 0
if (
isinstance(exc, MemoryError)
or isinstance(exc, OSError) and exc.errno in {errno.ENOSPC, errno.EDQUOT}
or isinstance(exc, sqlite3.Error) and (sqlite_code & 0xFF) in {
sqlite3.SQLITE_FULL, sqlite3.SQLITE_IOERR, sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB,
}
):
raise exc
logger.error("Chat archive thread failed; continuing: job=%s thread=%s", job_id, tid, exc_info=True)
job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
await _patch_job_async(
self.root, job_id,
failed_threads=int(job.get("failed_threads") or 0) + 1,
error=f"会话 {tid}: {type(exc).__name__}: {exc}"[:2000],
message="该会话处理失败,已记录错误,继续处理其他会话",
)
async def _run(self, job_id: str, *, owns_operation_lock: bool = False) -> None:
operation_lock = None if owns_operation_lock else try_acquire_archive_job_lock(self.root)
if not owns_operation_lock and operation_lock is None:
return
try:
await self._run_locked(job_id)
finally:
if operation_lock is not None:
operation_lock.close()
async def _run_locked(self, job_id: str) -> None:
job = _job_dict(self.root, job_id)
if not job:
return
month = job["month"]
action = job["action"]
lock = self._locks.setdefault(month, asyncio.Lock())
async with lock:
try:
await self._check_cancelled(job_id)
await _patch_job_async(self.root, job_id, status="running", message="正在处理,页面可安全关闭")
if action == "archive":
await self._archive_month(job_id, month)
elif action == "purge":
await self._purge_month(job_id, month)
else:
await self._rollback_month(job_id, month)
job = await asyncio.to_thread(_job_dict, self.root, job_id) or job
failed = int(job.get("failed_threads") or 0)
status = "completed" if failed == 0 else "partial"
month_status = {
"archive": "ready" if failed == 0 else "partial",
"purge": "deleted" if failed == 0 else "partial",
"rollback": "ready" if failed == 0 else "partial",
}[action]
await _patch_job_async(self.root, job_id, status=status, current_thread_id=None,
message="处理完成" if failed == 0 else f"完成但有 {failed} 个会话未处理")
await _upsert_month_async(self.root, month, status=month_status, total=int(job.get("total_threads") or 0), job_id=job_id)
except (_ArchiveJobCancelled, asyncio.CancelledError) as exc:
await _patch_job_async(self.root, job_id, status="cancelled" if isinstance(exc, _ArchiveJobCancelled) else "interrupted",
current_thread_id=None, message="任务已停止,已完成的归档保留,可重新执行",
error="管理员停止任务" if isinstance(exc, _ArchiveJobCancelled) else "执行任务被取消或服务正在退出")
await _upsert_month_async(self.root, month, status="interrupted", total=int(job.get("total_threads") or 0), job_id=job_id)
if isinstance(exc, asyncio.CancelledError):
raise
except Exception as exc:
logger.exception("Chat archive job failed: job=%s month=%s action=%s", job_id, month, action)
await _patch_job_async(self.root, job_id, status="failed", error=f"{type(exc).__name__}: {exc}"[:2000],
message="任务停止,已完成的归档保留;可查看错误并重新执行")
await _upsert_month_async(self.root, month, status="interrupted", total=int(job.get("total_threads") or 0), job_id=job_id)
async def _archive_month(self, job_id: str, month: str) -> None:
path = archive_db_path(self.root, month)
if (await asyncio.to_thread(shutil.disk_usage, self.root)).free < _ARCHIVE_DISK_RESERVE_BYTES:
raise RuntimeError("归档目录所在磁盘可用空间不足 2 GiB;请先扩容或清理磁盘,PostgreSQL 数据未删除")
job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
snapshot_at = str(job.get("snapshot_at") or job.get("created_at") or "")
total = int(job.get("total_threads") or 0)
await _upsert_month_async(self.root, month, status="copying", total=total, job_id=job_id)
processed = 0
offset = 0
async with _open_sqlite_archive(path) as archive:
while True:
rows = await _thread_page(
self.session_factory, month, offset, _PAGE_SIZE, snapshot_at=snapshot_at
)
if not rows:
break
offset += len(rows)
for thread_id, thread_status, metadata, created_at in rows:
await self._check_cancelled(job_id)
processed_through = created_at.isoformat() if created_at else None
if is_excluded_system_thread(metadata if isinstance(metadata, dict) else {}):
processed += 1
await _patch_job_async(
self.root, job_id, processed_threads=processed,
processed_through=processed_through, message="已跳过系统任务会话"
)
continue
tid = str(thread_id)
if (await asyncio.to_thread(shutil.disk_usage, self.root)).free < _ARCHIVE_DISK_RESERVE_BYTES:
raise RuntimeError("归档卷剩余空间低于 2 GiB,任务已暂停;PostgreSQL 数据保留")
await _patch_job_async(self.root, job_id, current_thread_id=tid, message="正在复制会话检查点")
if str(thread_status).lower() in {"running", "busy"}:
current_job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
processed += 1
await _patch_job_async(
self.root, job_id,
failed_threads=int(current_job.get("failed_threads") or 0) + 1,
processed_threads=processed, processed_through=processed_through,
message="正在跳过运行中的会话"
)
continue
try:
async with thread_archive_lock(self.root, tid, exclusive=True):
if await _thread_status(self.session_factory, tid) in {"running", "busy"}:
raise RuntimeError("会话正在问答中,已跳过;结束后可重新归档")
source_ids = await _checkpoint_ids(self.primary, tid)
if not source_ids:
processed += 1
await _patch_job_async(
self.root, job_id, processed_threads=processed,
processed_through=processed_through, message="已跳过无检查点的空会话"
)
continue
await copy_thread_state(self.primary, archive, tid)
copied_ids = await _checkpoint_ids(archive, tid)
if not set(source_ids).issubset(copied_ids):
raise RuntimeError(f"thread {tid} checkpoint verification failed: source={len(source_ids)} archived={len(copied_ids)}")
await asyncio.to_thread(_mark_archived_thread, self.root, tid, month, len(copied_ids), "copied")
except Exception as exc:
await self._record_thread_failure(job_id, tid, exc)
processed += 1
current_job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
await _patch_job_async(
self.root, job_id, processed_threads=processed,
failed_threads=int(current_job.get("failed_threads") or 0),
processed_through=processed_through,
)
await asyncio.sleep(0.03)
job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
if int(job.get("failed_threads") or 0):
await _patch_job_async(self.root, job_id, message="部分会话正忙或校验失败;可稍后重新归档")
async def _purge_month(self, job_id: str, month: str) -> None:
path = archive_db_path(self.root, month)
if not path.is_file():
raise RuntimeError(f"monthly SQLite archive is missing: {path}")
with _catalog_connect(self.root) as conn:
thread_ids = [str(row[0]) for row in conn.execute(
"SELECT thread_id FROM archive_threads WHERE month=? AND status IN ('copied','deleted','restored') ORDER BY thread_id",
(month,),
)]
processed = 0
async with _open_sqlite_archive(path) as archive:
for tid in thread_ids:
await self._check_cancelled(job_id)
await _patch_job_async(self.root, job_id, current_thread_id=tid, message="重新校验并清理 PostgreSQL 检查点")
skipped_running = False
try:
async with thread_archive_lock(self.root, tid, exclusive=True):
thread_status = await _thread_status(self.session_factory, tid)
if thread_status in {"running", "busy"}:
skipped_running = True
else:
pg_ids = await _checkpoint_ids(self.primary, tid)
if not pg_ids:
await asyncio.to_thread(
_mark_archived_thread,
self.root,
tid,
month,
len(await _checkpoint_ids(archive, tid)),
"deleted",
)
else:
await copy_thread_state(self.primary, archive, tid)
archived_ids = await _checkpoint_ids(archive, tid)
if not set(pg_ids).issubset(archived_ids):
raise RuntimeError(f"thread {tid} archive is incomplete; PostgreSQL rows were kept")
await self.primary.adelete_thread(tid)
remaining = await _checkpoint_ids(self.primary, tid)
if remaining:
raise RuntimeError(f"thread {tid} PostgreSQL cleanup verification failed")
await asyncio.to_thread(_mark_archived_thread, self.root, tid, month, len(archived_ids), "deleted")
except Exception as exc:
await self._record_thread_failure(job_id, tid, exc)
processed += 1
job_update: dict[str, Any] = {"processed_threads": processed}
if skipped_running:
job = await asyncio.to_thread(_job_dict, self.root, job_id) or {}
job_update.update(
failed_threads=int(job.get("failed_threads") or 0) + 1,
message="会话仍在问答中,已跳过;结束后可重新清理",
)
await _patch_job_async(self.root, job_id, **job_update)
await asyncio.sleep(_PURGE_THREAD_DELAY_SECONDS)
async def _rollback_month(self, job_id: str, month: str) -> None:
path = archive_db_path(self.root, month)
if not path.is_file():
raise RuntimeError(f"monthly SQLite archive is missing: {path}")
with _catalog_connect(self.root) as conn:
thread_ids = [str(row[0]) for row in conn.execute(
"SELECT thread_id FROM archive_threads WHERE month=? ORDER BY thread_id", (month,)
)]
processed = 0
async with _open_sqlite_archive(path) as archive:
for tid in thread_ids:
await self._check_cancelled(job_id)
await _patch_job_async(self.root, job_id, current_thread_id=tid, message="回滚检查点到 PostgreSQL")
try:
async with thread_archive_lock(self.root, tid, exclusive=True):
if await _thread_status(self.session_factory, tid) in {"running", "busy"}:
raise RuntimeError("会话正在问答中,已跳过;结束后可重新回滚")
archive_ids = await _checkpoint_ids(archive, tid)
await copy_thread_state(archive, self.primary, tid)
pg_ids = await _checkpoint_ids(self.primary, tid)
if not set(archive_ids).issubset(pg_ids):
raise RuntimeError(f"thread {tid} rollback verification failed")
await asyncio.to_thread(_mark_archived_thread, self.root, tid, month, len(archive_ids), "restored")
except Exception as exc:
await self._record_thread_failure(job_id, tid, exc)
processed += 1
await _patch_job_async(self.root, job_id, processed_threads=processed)
await asyncio.sleep(0.03)
async def _runner(request: Request) -> ChatArchiveRunner:
runner = getattr(request.app.state, "chat_archive_runner", None)
if runner is None:
checkpointer, _ = _primary_checkpointer(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="聊天元数据数据库未启用")
root = archive_root()
ensure_catalog(root)
await _ensure_archive_recovery_async(request, root)
runner = ChatArchiveRunner(root, checkpointer, session_factory)
request.app.state.chat_archive_runner = runner
return runner
async def _ensure_archive_recovery_async(request: Request, root: Path) -> None:
"""Recover abandoned work without interrupting another live process.
Recheck on inventory requests: another process may die after this process
starts or after an earlier recovery was skipped because the lock was held.
"""
if not (root / "catalog.db").is_file():
# Merely opening the settings page must not initialize archive storage.
return
lock = getattr(request.app.state, "chat_archive_recovery_lock", None)
if lock is None:
lock = asyncio.Lock()
request.app.state.chat_archive_recovery_lock = lock
async with lock:
await asyncio.to_thread(_recover_jobs, root, started_before=None)
@router.get("/months")
async def list_archive_months(request: Request):
await _require_admin(request)
_, _primary = _primary_checkpointer(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="聊天元数据数据库未启用")
root = archive_root()
await _ensure_archive_recovery_async(request, root)
inventory = await _month_inventory(session_factory)
catalog = _catalog_months(root)
disk_path = root
while not disk_path.exists():
disk_path = disk_path.parent
disk = await asyncio.to_thread(shutil.disk_usage, disk_path)
months = []
for item in inventory:
month = item["month"]
state = catalog.get(month, {})
file_path = archive_db_path(root, month)
months.append({
**item,
"status": state.get("status", "not_archived"),
"copied_threads": int(state.get("copied_threads") or 0),
"deleted_threads": int(state.get("deleted_threads") or 0),
"sqlite_bytes": file_path.stat().st_size if file_path.is_file() else 0,
"latest_job": state.get("latest_job"),
})
return {
"archive_dir": str(root),
"disk_total_bytes": disk.total,
"disk_free_bytes": disk.free,
"month_basis": "会话创建月份(按业务数据库 threads_meta 时间)",
"warning": "页面只读取轻量会话元数据,不创建归档文件或扫描 PostgreSQL 检查点表;只有管理员启动归档、清理或回滚任务时才会读写归档数据。",
"months": months,
}
async def _start_action(month: str, action: str, request: Request):
await _require_admin(request)
_primary_checkpointer(request)
return await (await _runner(request)).start(month, action)
@router.post("/months/{month}/archive")
async def start_month_archive(month: str, request: Request):
return await _start_action(month, "archive", request)
@router.post("/months/{month}/delete-postgres")
async def delete_archived_month_from_postgres(month: str, request: Request):
return await _start_action(month, "purge", request)
@router.post("/months/{month}/rollback")
async def rollback_month_to_postgres(month: str, request: Request):
return await _start_action(month, "rollback", request)
@router.get("/jobs/{job_id}", response_model=ChatArchiveJobResponse)
async def get_chat_archive_job(job_id: str, request: Request):
await _require_admin(request)
job = _job_dict(archive_root(), job_id)
if job is None:
raise HTTPException(status_code=404, detail="归档任务不存在")
return job
@@ -17,7 +17,6 @@ from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.auth.disabled_mode import is_auth_disabled
from app.gateway.catalog_cache import cached_catalog, catalog_key, invalidate_catalog
from app.gateway.deps import get_optional_user_from_request
from deerflow.persistence.compact_menu_overrides import (
DEFAULT_COMPACT_MENU_TYPE,
@@ -33,7 +32,6 @@ class CompactMenuOverride(BaseModel):
placement: Literal["sidebar", "settings", "toolbox"] = "sidebar"
sort_order: int = 0
label: str | None = Field(default=None, max_length=255)
disabled: bool = False
class CompactMenuOverrideResponse(CompactMenuOverride):
@@ -101,11 +99,9 @@ async def _read_response(
request: Request,
store: CompactMenuOverrideStore,
menu_type: str,
*,
actor: tuple[str | None, bool] | None = None,
) -> CompactMenuOverrideListResponse:
user_id, is_admin = actor if actor is not None else await _actor(request)
default_rows = await store.list_overrides(menu_type)
user_id, is_admin = await _actor(request)
if is_admin or user_id is None:
return CompactMenuOverrideListResponse(
menu_type=menu_type,
@@ -137,12 +133,8 @@ async def list_compact_menu_overrides(
),
) -> CompactMenuOverrideListResponse:
t = normalize_compact_menu_type(menu_type)
actor = await _actor(request)
store = _get_store(request)
return await cached_catalog(
request, "compact_menu_overrides", catalog_key(*actor, t),
lambda: _read_response(request, store, t, actor=actor), CompactMenuOverrideListResponse,
)
return await _read_response(request, store, t)
@router.put("", response_model=CompactMenuOverrideListResponse)
@@ -158,7 +150,6 @@ async def replace_compact_menu_overrides(
await store.replace_all(payload, menu_type=t, updated_by=user_id)
else:
await store.replace_user_overrides(user_id, payload, menu_type=t, updated_by=user_id)
await invalidate_catalog(request, "compact_menu_overrides")
return await _read_response(request, store, t)
@@ -175,5 +166,4 @@ async def reset_compact_menu_overrides(
await store.replace_all([], menu_type=t, updated_by=user_id)
else:
await store.replace_user_overrides(user_id, [], menu_type=t, updated_by=user_id)
await invalidate_catalog(request, "compact_menu_overrides")
return await _read_response(request, store, t)
@@ -865,9 +865,8 @@ async def get_capabilities() -> dict[str, Any]:
"""Advertise available modes, material channels, and export formats (§16.7)."""
from deerflow.agents.deep_research.adapters.image import ImageGenerationSettings
from deerflow.agents.deep_research.adapters.material_provider import DeerFlowMaterialProvider
from deerflow.utils.background_init import run_background_init
provider = await run_background_init(DeerFlowMaterialProvider)
provider = DeerFlowMaterialProvider()
image_settings = ImageGenerationSettings.from_app_config()
channels = provider.material_channels
return {
@@ -4,7 +4,7 @@
(整体存储、整体替换,数据库不定死字段)。文档内容由消费方与产出方约定,
当前包含时间轴(重要时间节点)、任务衔接、页面单块内容等区块。
Routes (credential-free internal integration endpoints):
Routes (authenticated by the Gateway middleware):
GET /api/force-situation/tasks/{task_id}/detail
PUT /api/force-situation/tasks/{task_id}/detail
"""
@@ -10,7 +10,8 @@ from pathlib import Path
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse, Response
from deerflow.html_runtime.addresses import html_map_tile_url, html_runtime_base_url
from deerflow.config import get_app_config
from deerflow.config.html_runtime_config import DEFAULT_HTML_MAP_TILE_URL
router = APIRouter(prefix="/api/public/html-runtime", tags=["html-runtime"])
@@ -59,8 +60,9 @@ async def get_deerflow_html_runtime() -> Response:
path = _RUNTIME_DIR / "deerflow-runtime.js"
if not path.is_file():
raise HTTPException(status_code=503, detail="HTML runtime is not installed")
base_url = html_runtime_base_url()
tile_url = html_map_tile_url()
config = get_app_config().html_runtime
base_url = config.base_url.rstrip("/")
tile_url = str(getattr(config, "map_tile_url", "") or "").strip() or DEFAULT_HTML_MAP_TILE_URL
runtime_config = {
"geoMaps": {
"china-provinces": f"{base_url}/maps/china-provinces.geojson",
@@ -1,137 +0,0 @@
"""HTML 模板管理 API(模板管理页)。
后端存储 :mod:`deerflow.runtime.scheduler.template_store`(文件型,
``{base_dir}/html-templates/<id>/``),启动时把随包内置模板 seed 进去(缺失才复制)。
读对所有登录用户开放(定时任务表单下拉也要用),写(创建/编辑/上传覆盖/删除/重置)
仅管理员——与敏感词管理同一鉴权口径。旧端点 ``GET /api/scheduled-tasks/html-templates``
保持不变(内部同样读存储),本路由是它的管理超集。
"""
from __future__ import annotations
import asyncio
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.catalog_cache import cached_catalog, catalog_key, invalidate_catalog
from app.gateway.deps import get_optional_user_from_request
from deerflow.runtime.scheduler import template_store
router = APIRouter(prefix="/api/html-templates", tags=["html-templates"])
async def _require_admin(request: Request) -> None:
"""写操作仅管理员;关闭鉴权时(request 无用户)放行,与 scheduled_tasks 口径一致。"""
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="仅管理员可管理 HTML 模板")
def _not_found(template_id: str) -> HTTPException:
return HTTPException(status_code=404, detail=f"模板不存在:{template_id}")
class TemplateCreateRequest(BaseModel):
template_id: str = Field(..., description="模板 id(字母/数字/./-/_,创建后不可改)")
name: str = Field(..., description="显示名称,如 no.28")
description: str = ""
html: str = Field(..., description="完整 HTML 模板正文")
class TemplateUpdateRequest(BaseModel):
name: str | None = None
description: str | None = None
html: str | None = Field(default=None, description="新的模板正文(上传覆盖走这里)")
@router.get("")
async def list_html_templates(request: Request) -> dict:
"""模板清单(含 builtin/convertible/size/时间),供管理页与定时任务表单共用。"""
async def load() -> dict:
return {"templates": await asyncio.to_thread(template_store.list_templates)}
return await cached_catalog(request, "html_templates", "admin-list", load)
@router.get("/{template_id}")
async def get_html_template(template_id: str, request: Request) -> dict[str, Any]:
async def load() -> dict[str, Any]:
meta = await asyncio.to_thread(template_store.get_template_meta, template_id)
if meta is None:
raise _not_found(template_id)
return meta
return await cached_catalog(request, "html_templates", catalog_key("meta", template_id), load)
@router.get("/{template_id}/content")
async def get_html_template_content(template_id: str, request: Request) -> dict:
async def load() -> dict:
try:
html = await asyncio.to_thread(template_store.read_template_html, template_id)
except template_store.TemplateStoreError:
raise _not_found(template_id)
return {"template_id": template_id, "html": html}
return await cached_catalog(request, "html_templates", catalog_key("content", template_id), load)
@router.post("")
async def create_html_template(body: TemplateCreateRequest, request: Request) -> dict[str, Any]:
await _require_admin(request)
try:
meta = template_store.create_template(
template_id=body.template_id,
name=body.name,
description=body.description,
html=body.html,
)
except template_store.TemplateStoreError as exc:
raise HTTPException(status_code=400, detail=str(exc))
await invalidate_catalog(request, "html_templates")
return meta
@router.put("/{template_id}")
async def update_html_template(
template_id: str, body: TemplateUpdateRequest, request: Request
) -> dict[str, Any]:
await _require_admin(request)
try:
meta = template_store.update_template(
template_id,
name=body.name,
description=body.description,
html=body.html,
)
except template_store.TemplateStoreError as exc:
raise HTTPException(status_code=400, detail=str(exc))
await invalidate_catalog(request, "html_templates")
return meta
@router.delete("/{template_id}")
async def delete_html_template(template_id: str, request: Request) -> dict:
await _require_admin(request)
try:
template_store.delete_template(template_id)
except template_store.TemplateStoreError as exc:
raise HTTPException(status_code=400, detail=str(exc))
await invalidate_catalog(request, "html_templates")
return {"ok": True, "template_id": template_id}
@router.post("/{template_id}/reset")
async def reset_html_template(template_id: str, request: Request) -> dict[str, Any]:
"""把内置模板正文重置回随包原版(自建模板无原版,400)。"""
await _require_admin(request)
try:
meta = template_store.reset_template(template_id)
except template_store.TemplateStoreError as exc:
raise HTTPException(status_code=400, detail=str(exc))
await invalidate_catalog(request, "html_templates")
return meta
@@ -31,7 +31,6 @@ from datetime import datetime
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.catalog_cache import cached_catalog, catalog_key, invalidate_catalog
from app.gateway.deps import get_current_user, get_optional_user_from_request
from deerflow.persistence.light_apps import DuplicateLightAppNameError, LightAppStore
@@ -78,17 +77,6 @@ class LightAppListResponse(BaseModel):
apps: list[LightAppResponse]
class LightAppSidebarItem(BaseModel):
app_id: str
name: str
mount_location: str | None = None
sort_order: int = 0
class LightAppSidebarResponse(BaseModel):
apps: list[LightAppSidebarItem]
class LightAppCreateRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=127)
description: str | None = Field(default=None, max_length=1000)
@@ -179,34 +167,15 @@ async def list_light_apps(
tag_ids: list[str] | None = Query(default=None, description="Keep apps carrying any of these tags (OR)"),
) -> LightAppListResponse:
store = _get_store(request)
async def load() -> LightAppListResponse:
rows = await store.list_apps(app_type=app_type, mount_location=mount_location, enabled_only=enabled_only)
if search:
needle = search.strip().lower()
rows = [r for r in rows if needle in str(r.get("name", "")).lower() or needle in str(r.get("description") or "").lower()]
responses = await _attach_tags(request, [LightAppResponse(**r) for r in rows])
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
return LightAppListResponse(apps=responses)
return await cached_catalog(
request, "light_apps", catalog_key("list", app_type, mount_location, enabled_only, search, sorted(tag_ids or [])),
load, LightAppListResponse,
)
@router.get("/sidebar", response_model=LightAppSidebarResponse)
async def list_sidebar_light_apps(request: Request) -> LightAppSidebarResponse:
"""Return the small set of fields required to build the shared sidebar."""
store = _get_store(request)
async def load() -> LightAppSidebarResponse:
rows = await store.list_sidebar_apps()
return LightAppSidebarResponse(apps=[LightAppSidebarItem(**row) for row in rows])
return await cached_catalog(request, "light_apps", "sidebar", load, LightAppSidebarResponse)
rows = await store.list_apps(app_type=app_type, mount_location=mount_location, enabled_only=enabled_only)
if search:
needle = search.strip().lower()
rows = [r for r in rows if needle in str(r.get("name", "")).lower() or needle in str(r.get("description") or "").lower()]
responses = await _attach_tags(request, [LightAppResponse(**r) for r in rows])
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
return LightAppListResponse(apps=responses)
@router.post("", response_model=LightAppResponse)
@@ -232,7 +201,6 @@ async def create_light_app(request: Request, body: LightAppCreateRequest) -> Lig
)
except DuplicateLightAppNameError:
raise HTTPException(status_code=409, detail="应用名称已存在")
await invalidate_catalog(request, "light_apps")
attached = await _attach_tags(request, [LightAppResponse(**row)])
return attached[0]
@@ -240,15 +208,11 @@ async def create_light_app(request: Request, body: LightAppCreateRequest) -> Lig
@router.get("/{app_id}", response_model=LightAppResponse)
async def get_light_app(request: Request, app_id: str) -> LightAppResponse:
store = _get_store(request)
async def load() -> LightAppResponse:
row = await store.get_app(app_id)
if row is None:
raise HTTPException(status_code=404, detail="应用不存在")
attached = await _attach_tags(request, [LightAppResponse(**row)])
return attached[0]
return await cached_catalog(request, "light_apps", catalog_key("detail", app_id), load, LightAppResponse)
row = await store.get_app(app_id)
if row is None:
raise HTTPException(status_code=404, detail="应用不存在")
attached = await _attach_tags(request, [LightAppResponse(**row)])
return attached[0]
@router.put("/{app_id}", response_model=LightAppResponse)
@@ -278,7 +242,6 @@ async def update_light_app(request: Request, app_id: str, body: LightAppUpdateRe
raise HTTPException(status_code=409, detail="应用名称已存在")
if row is None:
raise HTTPException(status_code=404, detail="应用不存在")
await invalidate_catalog(request, "light_apps")
attached = await _attach_tags(request, [LightAppResponse(**row)])
return attached[0]
@@ -291,5 +254,4 @@ async def delete_light_app(request: Request, app_id: str) -> dict:
if not deleted:
raise HTTPException(status_code=404, detail="应用不存在")
await _unassign_tags(request, app_id)
await invalidate_catalog(request, "light_apps")
return {"success": True}
@@ -17,7 +17,6 @@ from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.admin_query_window import resolve_admin_query_window
from app.gateway.deps import get_llm_metrics_store, get_local_provider, get_optional_user_from_request
from deerflow.persistence.llm_metrics.base import LlmMetricsQuery, LlmMetricsStore
@@ -69,11 +68,9 @@ def _require_store(request: Request) -> LlmMetricsStore:
return store
def _parse_dt(value: str | datetime | None) -> datetime | None:
def _parse_dt(value: str | None) -> datetime | None:
if not value:
return None
if isinstance(value, datetime):
return value
try:
# Accept both ``Z`` suffix and ``+00:00``.
return datetime.fromisoformat(value.replace("Z", "+00:00"))
@@ -180,8 +177,8 @@ def _build_query(
user_id: str | None,
model_name: str | None,
status: str | None,
since: str | datetime | None,
until: str | datetime | None,
since: str | None,
until: str | None,
limit: int,
offset: int,
user_ids: list[str] | None = None,
@@ -218,14 +215,13 @@ async def list_llm_metrics(
if not user_ids:
# No username matches → no rows; short-circuit the global scan.
return LlmMetricsListResponse(items=[], total=0, limit=limit, offset=offset)
since_dt, until_dt, _range_days = resolve_admin_query_window(since=since, until=until)
query = _build_query(
user_id=user_id,
user_ids=user_ids,
model_name=model_name,
status=status,
since=since_dt,
until=until_dt,
since=since,
until=until,
limit=limit,
offset=offset,
)
@@ -270,7 +266,6 @@ async def export_llm_metrics(
# No matching user — sentinel id that matches nothing, so the export
# still produces an empty xlsx (matching the list view's empty state).
user_ids = ["__no_such_user__"]
since_dt, until_dt, _range_days = resolve_admin_query_window(since=since, until=until)
try:
from openpyxl import Workbook
@@ -282,8 +277,8 @@ async def export_llm_metrics(
user_ids=user_ids,
model_name=model_name,
status=status,
since=since_dt,
until=until_dt,
since=since,
until=until,
limit=1, # unused: iter_all paginates internally
offset=0,
)
@@ -18,7 +18,6 @@ from io import BytesIO
from types import SimpleNamespace
from typing import Any, Literal
from urllib.parse import quote, unquote, urlencode, urlsplit
from uuid import uuid4
import httpx
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
@@ -27,7 +26,7 @@ from langchain_core.messages import HumanMessage
from pydantic import BaseModel, Field
from app.gateway.csrf_middleware import is_secure_request
from app.gateway.deps import get_agent_store, get_checkpointer, get_optional_user_from_request, get_thread_store
from app.gateway.deps import get_agent_store, get_optional_user_from_request, get_thread_store
from app.gateway.llmwiki_deposit import (
CONVERSATION_DEPOSIT_KB_DESCRIPTION,
CONVERSATION_DEPOSIT_KB_NAME,
@@ -37,7 +36,6 @@ from app.gateway.llmwiki_deposit import (
ensure_conversation_deposit_mapping,
)
from app.gateway.llmwiki_rag import build_llmwiki_rag_context
from app.gateway.perf_logging import finish_trace, trace_request, trace_stage
from app.gateway.weknora_embed import (
WEKNORA_EMBED_API_PREFIX as _WEKNORA_EMBED_API_PREFIX,
)
@@ -62,7 +60,6 @@ from app.gateway.weknora_embed import (
from app.gateway.weknora_static import StaticAsset, StaticAssetCache, prepare_static_asset
from app.gateway.wiki_directory_constraints import (
directory_roots,
ensure_directory_paths,
normalize_directory_paths,
paths_from_parent_directories,
resolve_directory_paths,
@@ -91,7 +88,6 @@ from deerflow.integrations.weknora.runtime import build_weknora_client, get_reso
from deerflow.persistence.llmwiki import LlmWikiStore
logger = logging.getLogger(__name__)
KNOWLEDGE_RESEARCH_DIRECTORY = "深度研究"
@asynccontextmanager
@@ -350,20 +346,6 @@ class WikiPageUpdate(BaseModel):
wiki_path: str | None = Field(default=None, max_length=1024)
class KnowledgeResearchSessionCreate(BaseModel):
question: str = Field(..., min_length=1, max_length=20_000)
class KnowledgeResearchSessionUpdate(BaseModel):
title: str | None = Field(default=None, min_length=1, max_length=255)
draft_markdown: str | None = Field(default=None, max_length=2_000_000)
class KnowledgeResearchPublicationSave(BaseModel):
title: str = Field(..., min_length=1, max_length=255)
expected_saved_hash: str | None = Field(default=None, min_length=64, max_length=64)
class ConversationDepositCreate(BaseModel):
thread_id: str = Field(..., min_length=1, max_length=128)
question: str = Field(default="", max_length=200_000)
@@ -523,28 +505,6 @@ def _select_embedding_model(models: list[dict[str, Any]], requested_id: str | No
return str(selected["id"])
def _select_knowledge_qa_model(models: list[dict[str, Any]]) -> str:
available = [
item
for item in models
if str(item.get("type") or "").strip().lower() == "knowledgeqa"
and str(item.get("id") or "").strip()
and str(item.get("status") or "active").strip().lower() == "active"
]
available.sort(
key=lambda item: (
0 if item.get("is_default") is True else 1,
str(item.get("id") or "").strip(),
)
)
if not available:
raise WeKnoraError(
"WeKnora has no active KnowledgeQA model; configure one before creating a Wiki",
status_code=400,
)
return str(available[0]["id"]).strip()
def _json_response(payload: Any, *, status_code: int = 200) -> Response:
return Response(
content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
@@ -2366,7 +2326,7 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
})
return {
"id": row["id"],
"remote_status": "cached" if source.get("_local_cache_only") else "available" if remote is not None else "missing",
"remote_status": "available" if remote is not None else "missing",
"name": display_name,
"description": description,
"test_questions": resolved_test_questions(profile_row),
@@ -3506,76 +3466,42 @@ async def list_knowledge_bases(
scope: Literal["personal", "public", "all"] = Query(default="all"),
refresh: bool = False,
) -> dict[str, Any]:
trace = trace_request(request, "llmwiki_knowledge_bases")
client = _client_or_503()
user_id, is_admin = await _actor(request)
try:
stage_started = time.perf_counter()
user_id, is_admin = await _actor(request)
trace_stage(trace, "actor_and_access", stage_started)
stage_started = time.perf_counter()
store = _store(request)
rows = await store.list_visible(user_id, scope=scope, is_admin=False)
trace_stage(trace, "local_knowledge_base_query_db", stage_started)
client: WeKnoraClient | None = None
remote_items: list[dict[str, Any]] = []
remote: dict[str, dict[str, Any]] = {}
if refresh:
stage_started = time.perf_counter()
client = _client_or_503()
try:
remote_items = await client.list_knowledge_bases(refresh=True)
remote = {str(item.get("id")): item for item in remote_items if item.get("id")}
except WeKnoraError as exc:
_raise_upstream(exc)
trace_stage(trace, "weknora_explicit_refresh_wait", stage_started)
deposit_remote = next(
(item for item in remote_items if str(item.get("name") or "").strip() == CONVERSATION_DEPOSIT_KB_NAME),
None,
)
if deposit_remote is not None:
previous_mapping = await store.get_by_weknora_id(str(deposit_remote.get("id") or ""))
deposit_mapping = await ensure_conversation_deposit_mapping(
store,
client,
remote_knowledge_bases=remote_items,
)
description_was_synchronized = previous_mapping is None or not _is_conversation_deposit_mapping(previous_mapping) or str(previous_mapping.get("description") or "") != CONVERSATION_DEPOSIT_KB_DESCRIPTION
if deposit_mapping is not None and description_was_synchronized:
remote_id = str(deposit_mapping["weknora_id"])
remote[remote_id] = {
**remote.get(remote_id, deposit_remote),
"description": CONVERSATION_DEPOSIT_KB_DESCRIPTION,
}
trace_stage(trace, "conversation_deposit_mapping_sync", stage_started)
stage_started = time.perf_counter()
knowledge_bases: list[dict[str, Any]] = []
for row in rows:
if refresh:
remote_item = remote.get(str(row["weknora_id"]))
# An explicit successful refresh is authoritative: do not show
# mappings whose remote knowledge base has been deleted.
if remote_item is None:
continue
assert client is not None
row, remote_item = await _sync_conversation_deposit_view(store, client, row, remote_item)
else:
# Normal page loads are served entirely from the local SQL
# mapping. A remote check happens only after an explicit refresh.
remote_item = {
"id": str(row.get("weknora_id") or ""),
"name": str(row.get("name") or ""),
"description": str(row.get("description") or ""),
"_local_cache_only": True,
}
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
trace_stage(trace, "map_visible_knowledge_bases", stage_started)
trace["details"].update({"scope": scope, "refresh": refresh, "remote_count": len(remote_items) if refresh else None, "local_count": len(rows), "result_count": len(knowledge_bases)})
return {"knowledge_bases": knowledge_bases}
finally:
finish_trace(trace)
remote_items = await client.list_knowledge_bases(refresh=refresh)
remote = {str(item.get("id")): item for item in remote_items if item.get("id")}
except WeKnoraError as exc:
_raise_upstream(exc)
deposit_remote = next(
(item for item in remote_items if str(item.get("name") or "").strip() == CONVERSATION_DEPOSIT_KB_NAME),
None,
)
if deposit_remote is not None:
previous_mapping = await _store(request).get_by_weknora_id(str(deposit_remote.get("id") or ""))
deposit_mapping = await ensure_conversation_deposit_mapping(
_store(request),
client,
remote_knowledge_bases=remote_items,
)
description_was_synchronized = previous_mapping is None or not _is_conversation_deposit_mapping(previous_mapping) or str(previous_mapping.get("description") or "") != CONVERSATION_DEPOSIT_KB_DESCRIPTION
if deposit_mapping is not None and description_was_synchronized:
remote_id = str(deposit_mapping["weknora_id"])
remote[remote_id] = {
**remote.get(remote_id, deposit_remote),
"description": CONVERSATION_DEPOSIT_KB_DESCRIPTION,
}
rows = await _store(request).list_visible(user_id, scope=scope, is_admin=False)
knowledge_bases: list[dict[str, Any]] = []
for row in rows:
remote_item = remote.get(str(row["weknora_id"]))
# A successful remote list is authoritative for every scope. Keep the
# mapping for existing references, but never display a deleted base.
if remote_item is None:
continue
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
return {"knowledge_bases": knowledge_bases}
@router.get("/api/llmwiki/embedding-models")
@@ -3597,62 +3523,37 @@ async def list_embedding_models(request: Request) -> dict[str, Any]:
async def list_agent_knowledge_bases(request: Request, agent_id: str) -> dict[str, Any]:
"""Return the agent-bound bases still visible to the current DeerFlow user."""
trace = trace_request(request, "llmwiki_agent_knowledge_bases")
user_id, is_admin = await _actor(request)
agent_store = get_agent_store(request)
agent = await (agent_store.get_any(agent_id) if is_admin else agent_store.get_visible(agent_id, user_id))
if agent is None:
raise HTTPException(status_code=404, detail="Agent not found")
try:
stage_started = time.perf_counter()
user_id, is_admin = await _actor(request)
agent_store = get_agent_store(request)
agent = await (agent_store.get_any(agent_id) if is_admin else agent_store.get_visible(agent_id, user_id))
if agent is None:
raise HTTPException(status_code=404, detail="Agent not found")
trace_stage(trace, "actor_and_agent_access", stage_started)
stage_started = time.perf_counter()
try:
config = load_agent_config(agent_id)
except (FileNotFoundError, ValueError):
config = None
bound_ids = list(config.llmwiki_knowledge_base_ids or []) if config else []
trace_stage(trace, "load_agent_binding_config", stage_started)
stage_started = time.perf_counter()
store = _store(request)
rows: list[dict[str, Any]] = []
for mapping_id in bound_ids:
row = await store.get_authorized(
mapping_id,
user_id,
write=False,
is_admin=is_admin,
)
if row is not None:
rows.append(row)
trace_stage(trace, "local_knowledge_base_mapping_query", stage_started)
stage_started = time.perf_counter()
knowledge_bases = []
for row in rows:
remote_item = {
"id": str(row.get("weknora_id") or ""),
"name": str(row.get("name") or ""),
"description": str(row.get("description") or ""),
"_local_cache_only": True,
}
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
trace_stage(trace, "map_agent_knowledge_bases", stage_started)
trace["details"].update({"bound_count": len(bound_ids), "remote_count": None, "result_count": len(knowledge_bases)})
return {"knowledge_bases": knowledge_bases}
finally:
finish_trace(trace)
config = load_agent_config(agent_id)
except (FileNotFoundError, ValueError):
config = None
bound_ids = list(config.llmwiki_knowledge_base_ids or []) if config else []
rows: list[dict[str, Any]] = []
for mapping_id in bound_ids:
row = await _store(request).get_authorized(
mapping_id,
user_id,
write=False,
is_admin=is_admin,
)
if row is not None:
rows.append(row)
try:
remote = {str(item.get("id")): item for item in await _client_or_503().list_knowledge_bases() if item.get("id")}
except WeKnoraError as exc:
_raise_upstream(exc)
return {"knowledge_bases": [_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin) for row in rows if (remote_item := remote.get(str(row["weknora_id"]))) is not None]}
@router.post("/api/llmwiki/knowledge-bases", status_code=201)
async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) -> dict[str, Any]:
client = _client_or_503()
trace = trace_request(request, "llmwiki_knowledge_base_create")
stage_started = time.perf_counter()
user_id, is_admin = await _actor(request)
trace_stage(trace, "actor_and_access", stage_started)
if body.name.strip() == CONVERSATION_DEPOSIT_KB_NAME:
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
wiki_config = body.wiki_config if body.wiki_enabled else None
@@ -3694,24 +3595,16 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from None
try:
stage_started = time.perf_counter()
upstream_models = await client.list_models()
embedding_model_id = _select_embedding_model(
[view for item in upstream_models if (view := _embedding_model_view(item)) is not None],
await _embedding_models(client),
body.embedding_model_id,
)
knowledge_qa_model_id = (
_select_knowledge_qa_model(upstream_models) if body.wiki_enabled else None
)
trace_stage(trace, "weknora_model_catalog", stage_started)
stage_started = time.perf_counter()
remote = await client.create_knowledge_base(
name=body.name.strip(),
description=body.description,
kb_type=body.type,
wiki_enabled=body.wiki_enabled,
embedding_model_id=embedding_model_id,
knowledge_qa_model_id=knowledge_qa_model_id,
wiki_content_instructions=(
compiled.content_instructions if body.wiki_enabled else ""
),
@@ -3719,14 +3612,11 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
compiled.extraction_instructions if body.wiki_enabled else ""
),
)
trace_stage(trace, "weknora_create", stage_started)
except WeKnoraError as exc:
finish_trace(trace)
_raise_upstream(exc)
try:
# Directory settings are references for WeKnora's generator. Creating
# folders here would turn the reference into an accidental template.
stage_started = time.perf_counter()
row = await _store(request).create_mapping(
weknora_id=str(remote["id"]),
owner_user_id=user_id,
@@ -3753,9 +3643,7 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
),
wiki_config_apply_error=None,
)
trace_stage(trace, "local_mapping_create", stage_started)
except Exception as exc:
finish_trace(trace)
logger.exception(
"Failed to initialize or persist LLMWiki mapping; compensating remote create"
)
@@ -3767,7 +3655,6 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
_raise_upstream(exc)
raise HTTPException(status_code=500, detail="Failed to register the knowledge base") from None
_schedule_local_wiki_sync(request, row)
finish_trace(trace)
return _mapping_view(row, remote, actor_user_id=user_id, is_admin=is_admin)
@@ -4169,513 +4056,6 @@ async def get_knowledge_base_graph(
return {"nodes": nodes, "edges": edges, "meta": meta}
def _knowledge_research_store(request: Request):
store = getattr(request.app.state, "knowledge_research_store", None)
if store is None:
raise HTTPException(status_code=503, detail="知识库研究存储尚未就绪")
return store
async def _knowledge_research_can_write(
request: Request,
mapping: dict[str, Any],
user_id: str,
is_admin: bool,
) -> bool:
if _is_conversation_deposit_mapping(mapping):
return False
writable = await _store(request).get_authorized(
str(mapping["id"]),
user_id,
write=True,
is_admin=is_admin,
)
return writable is not None
def _knowledge_research_publication_view(
row: dict[str, Any],
*,
user_id: str,
can_write: bool,
include_content: bool = False,
) -> dict[str, Any] | None:
if not row.get("saved_hash"):
return None
owned = row.get("user_id") == user_id
result: dict[str, Any] = {
"id": row["publication_id"],
"title": row.get("saved_title") or row.get("title") or "",
"wiki_slug": row.get("wiki_slug") or "",
"saved_hash": row["saved_hash"],
"saved_at": row.get("saved_at"),
"updated_at": row.get("saved_at") or row.get("updated_at"),
"owned_by_current_user": owned,
"can_continue": owned,
"can_write": owned and can_write,
}
if owned:
# Only the creator receives a bridge from the public book to the
# private research session. The thread id remains session-only.
result["session_id"] = row["id"]
if include_content:
result["content_markdown"] = row.get("saved_markdown") or ""
return result
def _knowledge_research_session_view(
row: dict[str, Any],
*,
user_id: str,
can_write: bool,
) -> dict[str, Any]:
return {
"id": row["id"],
"thread_id": row["thread_id"],
"title": row.get("title") or "",
"draft_markdown": row.get("draft_markdown") or "",
"created_at": row.get("created_at"),
"updated_at": row.get("updated_at"),
"can_write": can_write,
"publication": _knowledge_research_publication_view(
row,
user_id=user_id,
can_write=can_write,
),
}
def _knowledge_research_default_title(question: str) -> str:
for line in question.splitlines():
normalized = " ".join(line.split())
if normalized:
return normalized[:255]
raise HTTPException(status_code=422, detail="问题不能为空")
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/research/sessions")
async def list_knowledge_research_sessions(
request: Request,
mapping_id: str,
) -> dict[str, Any]:
mapping, user_id, is_admin = await _authorized_mapping(
request,
mapping_id,
write=False,
)
can_write = await _knowledge_research_can_write(
request,
mapping,
user_id,
is_admin,
)
store = _knowledge_research_store(request)
sessions, publications = await asyncio.gather(
store.list_owned(mapping_id, user_id),
store.list_published(mapping_id),
)
return {
"sessions": [
_knowledge_research_session_view(
row,
user_id=user_id,
can_write=can_write,
)
for row in sessions
],
"publications": [
view
for row in publications
if (
view := _knowledge_research_publication_view(
row,
user_id=user_id,
can_write=can_write,
)
)
is not None
],
}
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/research/sessions", status_code=201)
async def create_knowledge_research_session(
request: Request,
mapping_id: str,
body: KnowledgeResearchSessionCreate,
) -> dict[str, Any]:
mapping, user_id, is_admin = await _authorized_mapping(
request,
mapping_id,
write=False,
)
can_write = await _knowledge_research_can_write(
request,
mapping,
user_id,
is_admin,
)
question = body.question.strip()
title = _knowledge_research_default_title(question)
store = _knowledge_research_store(request)
session_id = str(uuid4())
publication_id = str(uuid4())
thread_id = str(uuid4())
session = await store.create(
session_id=session_id,
publication_id=publication_id,
thread_id=thread_id,
knowledge_base_id=mapping_id,
user_id=user_id,
title=title,
)
metadata = {
"system": True,
"thread_type": "knowledge_research",
"knowledge_research_session_id": session_id,
"knowledge_research_knowledge_base_id": mapping_id,
"llmwiki_knowledge_base_ids": [mapping_id],
"assistant_knowledge_base_ids": [],
"knowledge_retrieval_mode": "classic",
"knowledge_space_strict": True,
"rag_mode_enabled": True,
"rag_mode": True,
}
try:
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
await create_thread(
ThreadCreateRequest(
thread_id=thread_id,
assistant_id=None,
metadata=metadata,
initialize_checkpoint=True,
),
request,
)
await get_thread_store(request).update_display_name(
thread_id,
title,
user_id=user_id,
)
except Exception:
# Thread creation writes metadata before its initial checkpoint. Undo
# every side on failure so no half-bound private chat remains.
await store.delete(session_id)
try:
checkpointer = get_checkpointer(request)
if hasattr(checkpointer, "adelete_thread"):
await checkpointer.adelete_thread(thread_id)
except Exception:
logger.debug("Could not remove failed research checkpoint", exc_info=True)
try:
await get_thread_store(request).delete(thread_id, user_id=user_id)
except Exception:
logger.debug("Could not remove failed research thread metadata", exc_info=True)
raise
return _knowledge_research_session_view(
session,
user_id=user_id,
can_write=can_write,
)
async def _owned_knowledge_research_session(
request: Request,
mapping_id: str,
session_id: str,
) -> tuple[dict[str, Any], dict[str, Any], str, bool, bool]:
mapping, user_id, is_admin = await _authorized_mapping(
request,
mapping_id,
write=False,
)
session = await _knowledge_research_store(request).get_owned(
session_id,
user_id,
)
if session is None or session["knowledge_base_id"] != mapping_id:
raise HTTPException(status_code=404, detail="研究会话不存在或无权访问")
can_write = await _knowledge_research_can_write(
request,
mapping,
user_id,
is_admin,
)
return session, mapping, user_id, is_admin, can_write
@router.get(
"/api/llmwiki/knowledge-bases/{mapping_id}/research/sessions/{session_id}"
)
async def get_knowledge_research_session(
request: Request,
mapping_id: str,
session_id: str,
) -> dict[str, Any]:
session, _, user_id, _, can_write = await _owned_knowledge_research_session(
request,
mapping_id,
session_id,
)
return _knowledge_research_session_view(
session,
user_id=user_id,
can_write=can_write,
)
@router.patch(
"/api/llmwiki/knowledge-bases/{mapping_id}/research/sessions/{session_id}"
)
async def update_knowledge_research_session(
request: Request,
mapping_id: str,
session_id: str,
body: KnowledgeResearchSessionUpdate,
) -> dict[str, Any]:
current, _, user_id, _, can_write = await _owned_knowledge_research_session(
request,
mapping_id,
session_id,
)
fields = body.model_fields_set
if not fields:
return _knowledge_research_session_view(
current,
user_id=user_id,
can_write=can_write,
)
title = body.title.strip() if "title" in fields and body.title is not None else None
if "title" in fields and not title:
raise HTTPException(status_code=422, detail="标题不能为空")
updated = await _knowledge_research_store(request).update_owned(
session_id,
user_id,
title=title,
draft_markdown=(
body.draft_markdown if "draft_markdown" in fields else None
),
)
if updated is None:
raise HTTPException(status_code=404, detail="研究会话不存在或无权访问")
if title is not None:
try:
await get_thread_store(request).update_display_name(
updated["thread_id"],
title,
user_id=user_id,
)
except Exception:
logger.debug("Could not update research thread title", exc_info=True)
return _knowledge_research_session_view(
updated,
user_id=user_id,
can_write=can_write,
)
async def _upsert_knowledge_research_wiki_page(
*,
client: WeKnoraClient,
knowledge_base_id: str,
slug: str,
payload: dict[str, Any],
previously_saved: bool,
) -> dict[str, Any]:
if previously_saved:
try:
return await client.update_wiki_page(
knowledge_base_id,
slug,
payload,
)
except WeKnoraError as exc:
if exc.status_code != 404:
raise
return await client.create_wiki_page(
knowledge_base_id,
{"slug": slug, **payload},
)
try:
return await client.create_wiki_page(
knowledge_base_id,
{"slug": slug, **payload},
)
except WeKnoraError as create_error:
# A prior request may have committed upstream before its local DB write
# failed. Probe the stable slug and turn the retry into an update.
if create_error.status_code not in {400, 409}:
raise
try:
await client.get_wiki_page(knowledge_base_id, slug)
except WeKnoraError:
raise create_error
return await client.update_wiki_page(knowledge_base_id, slug, payload)
@router.put(
"/api/llmwiki/knowledge-bases/{mapping_id}/research/sessions/{session_id}/publication"
)
async def save_knowledge_research_publication(
request: Request,
mapping_id: str,
session_id: str,
body: KnowledgeResearchPublicationSave,
) -> dict[str, Any]:
# Write authorization and private-session ownership are independent: even
# an administrator may not publish another user's private conversation.
mapping, user_id, _ = await _authorized_mapping(request, mapping_id, write=True)
if _is_conversation_deposit_mapping(mapping):
raise HTTPException(
status_code=403,
detail="Conversation deposit is managed by cmzs",
)
title = body.title.strip()
if not title:
raise HTTPException(status_code=422, detail="标题不能为空")
store = _knowledge_research_store(request)
client = _client_or_503()
remote_id = str(mapping["weknora_id"])
saved: dict[str, Any] | None = None
async with store.locked_publication(session_id, user_id) as publication:
if publication is None:
raise HTTPException(status_code=404, detail="研究会话不存在或无权访问")
session = publication.snapshot()
if session["knowledge_base_id"] != mapping_id:
raise HTTPException(status_code=404, detail="研究会话不存在或无权访问")
markdown = str(session.get("draft_markdown") or "")
if not markdown.strip():
raise HTTPException(status_code=422, detail="请先生成或填写 Markdown 正文")
digest = hashlib.sha256(
json.dumps(
{"title": title, "markdown": markdown},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
).hexdigest()
current_saved_hash = session.get("saved_hash")
if current_saved_hash == digest:
# A prior identical request may have committed both upstream and
# locally while its HTTP response was lost. Treat that retry as
# a no-op even if it carries the pre-save expected hash.
return _knowledge_research_session_view(
session,
user_id=user_id,
can_write=True,
)
if current_saved_hash and body.expected_saved_hash is None:
raise HTTPException(
status_code=409,
detail={
"code": "KNOWLEDGE_RESEARCH_EXPECTED_HASH_REQUIRED",
"message": "覆盖已保存文档时必须携带当前版本,请刷新后再保存。",
"saved_hash": current_saved_hash,
},
)
if (
body.expected_saved_hash is not None
and body.expected_saved_hash != current_saved_hash
):
raise HTTPException(
status_code=409,
detail={
"code": "KNOWLEDGE_RESEARCH_PUBLICATION_CHANGED",
"message": "文档已被更新,请刷新后再保存。",
"saved_hash": current_saved_hash,
},
)
slug = str(
session.get("wiki_slug")
or f"knowledge-research/{session['publication_id']}"
)
try:
folder_ids = await ensure_directory_paths(
client,
remote_id,
[[KNOWLEDGE_RESEARCH_DIRECTORY]],
)
folder_id = folder_ids.get(KNOWLEDGE_RESEARCH_DIRECTORY)
if not folder_id:
raise WeKnoraError("未能创建深度研究 Wiki 目录", status_code=502)
page = await _upsert_knowledge_research_wiki_page(
client=client,
knowledge_base_id=remote_id,
slug=slug,
previously_saved=bool(current_saved_hash),
payload={
"title": title,
"content": markdown,
"summary": "知识库问答沉淀",
"page_type": "summary",
"status": "published",
"category_path": [KNOWLEDGE_RESEARCH_DIRECTORY],
"folder_id": folder_id,
},
)
except WeKnoraError as exc:
if exc.status_code == 409:
raise HTTPException(status_code=409, detail=str(exc)) from None
_raise_upstream(exc)
saved = publication.save(
title=title,
markdown=markdown,
saved_hash=digest,
wiki_slug=slug,
wiki_page_id=str(page.get("id") or page.get("page_id") or "")
or None,
wiki_folder_id=folder_id,
)
assert saved is not None
_schedule_local_wiki_sync(request, mapping)
return _knowledge_research_session_view(
saved,
user_id=user_id,
can_write=True,
)
@router.get(
"/api/llmwiki/knowledge-bases/{mapping_id}/research/publications/{publication_id}"
)
async def get_knowledge_research_publication(
request: Request,
mapping_id: str,
publication_id: str,
) -> dict[str, Any]:
mapping, user_id, is_admin = await _authorized_mapping(
request,
mapping_id,
write=False,
)
publication = await _knowledge_research_store(request).get_publication(
publication_id
)
if (
publication is None
or publication["knowledge_base_id"] != mapping_id
):
raise HTTPException(status_code=404, detail="研究文档不存在")
can_write = await _knowledge_research_can_write(
request,
mapping,
user_id,
is_admin,
)
view = _knowledge_research_publication_view(
publication,
user_id=user_id,
can_write=can_write,
include_content=True,
)
if view is None:
raise HTTPException(status_code=404, detail="研究文档不存在")
return view
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
async def list_wiki_pages(
request: Request,
@@ -29,7 +29,6 @@ from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.auth.disabled_mode import is_auth_disabled
from app.gateway.catalog_cache import cached_catalog, catalog_key, invalidate_catalog
from app.gateway.deps import get_optional_user_from_request
from deerflow.persistence.menu_overrides import MenuOverrideStore
@@ -117,11 +116,9 @@ def _merge_overrides(default_rows: list[dict], user_rows: list[dict] | None) ->
return merged
async def _read_response(
request: Request, store: MenuOverrideStore, *, actor: tuple[str | None, bool] | None = None,
) -> MenuOverrideListResponse:
user_id, is_admin = actor if actor is not None else await _actor(request)
async def _read_response(request: Request, store: MenuOverrideStore) -> MenuOverrideListResponse:
default_rows = await store.list_overrides()
user_id, is_admin = await _actor(request)
if is_admin or user_id is None:
return MenuOverrideListResponse(
overrides=[MenuOverrideResponse(**row) for row in default_rows],
@@ -194,12 +191,8 @@ def _validate_depth(
@router.get("", response_model=MenuOverrideListResponse)
async def list_menu_overrides(request: Request) -> MenuOverrideListResponse:
actor = await _actor(request)
store = _get_store(request)
return await cached_catalog(
request, "menu_overrides", catalog_key(*actor),
lambda: _read_response(request, store, actor=actor), MenuOverrideListResponse,
)
return await _read_response(request, store)
@router.put("", response_model=MenuOverrideListResponse)
@@ -221,7 +214,6 @@ async def replace_menu_overrides(request: Request, body: MenuOverrideReplaceRequ
await store.replace_all(payload, updated_by=user_id)
else:
await store.replace_user_overrides(user_id, payload, updated_by=user_id)
await invalidate_catalog(request, "menu_overrides")
return await _read_response(request, store)
@@ -234,5 +226,4 @@ async def reset_menu_overrides(request: Request) -> MenuOverrideListResponse:
await store.replace_all([], updated_by=user_id)
else:
await store.replace_user_overrides(user_id, [], updated_by=user_id)
await invalidate_catalog(request, "menu_overrides")
return await _read_response(request, store)
@@ -17,17 +17,15 @@ from __future__ import annotations
import asyncio
import copy
import errno
import logging
import os
import re
import shutil
import sys
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from time import perf_counter, sleep
from typing import Any, Literal
from time import perf_counter
from typing import Any
import yaml
from fastapi import APIRouter, HTTPException, Request
@@ -47,56 +45,6 @@ _MODEL_BLOCK_RE = re.compile(r"(?m)^models:[^\r\n]*(?:\r?\n|$)")
_TOP_LEVEL_KEY_RE = re.compile(r"(?m)^[A-Za-z_][A-Za-z0-9_-]*:[^\r\n]*(?:\r?\n|$)")
_BACKUP_DIR_NAME = ".model-config-backups"
_MUTATION_LOCK = asyncio.Lock()
_ATOMIC_REPLACE_RETRIES = 5
_ATOMIC_REPLACE_RETRY_ERRNOS = {
errno.EBUSY,
errno.EAGAIN,
getattr(errno, "ETXTBSY", 26),
}
def _is_file_mountpoint(path: Path) -> bool:
"""Return whether ``path`` itself is mounted (common with Docker file binds)."""
if not sys.platform.startswith("linux"):
return False
try:
target = str(path.resolve())
with Path("/proc/self/mountinfo").open(encoding="utf-8") as mountinfo:
for line in mountinfo:
fields = line.split(" - ", maxsplit=1)[0].split()
if len(fields) < 5:
continue
mount_path = re.sub(
r"\\([0-7]{3})",
lambda match: chr(int(match.group(1), 8)),
fields[4],
)
if mount_path == target:
return True
except OSError:
logger.debug("Unable to inspect Linux mount table for model config path", exc_info=True)
return False
def _write_mounted_config_in_place(path: Path, raw_before: str, raw_after: str) -> None:
"""Update a single-file bind mount, where Linux refuses atomic rename (EBUSY)."""
try:
with path.open("w", encoding="utf-8", newline="") as handle:
handle.write(raw_after)
handle.flush()
os.fsync(handle.fileno())
except Exception:
# The backup was persisted before reaching this point. Restore the
# previous contents best-effort so a failed write does not leave YAML
# truncated or partially replaced.
try:
with path.open("w", encoding="utf-8", newline="") as handle:
handle.write(raw_before)
handle.flush()
os.fsync(handle.fileno())
except OSError:
logger.exception("Failed to restore model config after an in-place bind-mount write failure")
raise
_EDITOR_FIELDS = frozenset(
{
@@ -122,7 +70,6 @@ _EDITOR_FIELDS = frozenset(
"use_responses_api",
"output_version",
"write_file_transport",
"strict_stream_integrity",
}
)
_PRESERVED_FIELDS = frozenset(ModelConfig.model_fields) | _EDITOR_FIELDS
@@ -145,7 +92,6 @@ class ModelAdminItem(BaseModel):
stream_usage: bool | None = None
supports_thinking: bool = False
supports_reasoning_effort: bool = False
default_reasoning_effort: Literal["low", "medium", "xhigh"] = "xhigh"
supports_vision: bool = False
when_thinking_enabled: dict[str, Any] | None = None
when_thinking_disabled: dict[str, Any] | None = None
@@ -153,9 +99,6 @@ class ModelAdminItem(BaseModel):
use_responses_api: bool | None = None
output_version: str | None = None
write_file_transport: WriteFileTransport = WriteFileTransport.NATIVE
# ``None`` means the runtime default: auto-enabled only for the historically
# affected qwen3.8-flash-next model id (see deerflow.models.chat_stream_state).
strict_stream_integrity: bool | None = None
extra_settings: dict[str, Any] = Field(default_factory=dict)
api_key_configured: bool = False
api_key_source: str | None = None
@@ -203,9 +146,6 @@ class ModelUpsertRequest(BaseModel):
# bridge setting during edits. New models still resolve to ModelConfig's
# native default when the field is omitted.
write_file_transport: WriteFileTransport | None = None
# Optional on the wire so edits preserve an existing toggle; the dedicated
# strict-stream-integrity endpoint is the canonical switch surface.
strict_stream_integrity: bool | None = None
extra_settings: dict[str, Any] = Field(default_factory=dict)
@field_validator("name", "use", "model")
@@ -228,20 +168,6 @@ class ModelTestRequest(BaseModel):
class ReasoningEffortCapabilityRequest(BaseModel):
supports_reasoning_effort: bool
default_reasoning_effort: Literal["low", "medium", "xhigh"] | None = None
class WriteFileTransportToggleRequest(BaseModel):
"""Per-model 文件流式兼容 switch: off keeps native, on selects buffered_gateway."""
write_file_transport: Literal["native", "buffered_gateway"]
class StrictStreamIntegrityToggleRequest(BaseModel):
"""Per-model 流完整性检测 switch: detect gateway streams that end without
a terminal frame and retry instead of committing a half answer."""
strict_stream_integrity: bool
class ModelOrderRequest(BaseModel):
@@ -267,7 +193,6 @@ def _to_admin_item(
raw: dict[str, Any],
*,
supports_reasoning_effort: bool = False,
default_reasoning_effort: str = "xhigh",
) -> ModelAdminItem:
api_key = raw.get("api_key")
extra = {key: copy.deepcopy(value) for key, value in raw.items() if key not in _PRESERVED_FIELDS}
@@ -286,7 +211,6 @@ def _to_admin_item(
stream_usage=raw.get("stream_usage"),
supports_thinking=bool(raw.get("supports_thinking", False)),
supports_reasoning_effort=bool(supports_reasoning_effort),
default_reasoning_effort=default_reasoning_effort if default_reasoning_effort in {"low", "medium", "xhigh"} else "xhigh",
supports_vision=bool(raw.get("supports_vision", False)),
when_thinking_enabled=copy.deepcopy(raw.get("when_thinking_enabled")),
when_thinking_disabled=copy.deepcopy(raw.get("when_thinking_disabled")),
@@ -294,7 +218,6 @@ def _to_admin_item(
use_responses_api=raw.get("use_responses_api"),
output_version=raw.get("output_version"),
write_file_transport=raw.get("write_file_transport", WriteFileTransport.NATIVE),
strict_stream_integrity=raw.get("strict_stream_integrity") if isinstance(raw.get("strict_stream_integrity"), bool) else None,
extra_settings=extra,
api_key_configured=bool(api_key),
api_key_source=_mask_env_source(api_key),
@@ -313,12 +236,12 @@ def _get_capability_store(request: Request, *, required: bool = True) -> ModelCa
return store
async def _capabilities_by_name(request: Request) -> dict[str, dict[str, Any]]:
async def _capabilities_by_name(request: Request) -> dict[str, bool]:
store = _get_capability_store(request, required=False)
if store is None:
return {}
rows = await store.list_capabilities()
return {str(row.get("model_name") or ""): row for row in rows if row.get("model_name")}
return {str(row.get("model_name") or ""): bool(row.get("supports_reasoning_effort", False)) for row in rows if row.get("model_name")}
def _admin_user_id(user: Any) -> str | None:
@@ -334,48 +257,10 @@ def _validate_reasoning_capability(model: dict[str, Any], enabled: bool) -> None
)
# ``write_file_transport=buffered_gateway`` and ``strict_stream_integrity`` are
# implemented only by the patched OpenAI-compatible chat-completions classes
# (see deerflow.models.factory); every other driver is rejected there at
# construction time, so mirror the same constraint here with a clearer message.
_PATCHED_OPENAI_COMPATIBLE_USES = frozenset(
{
"langchain_openai:ChatOpenAI",
"deerflow.models.patched_openai:PatchedChatOpenAI",
}
)
def _validate_patched_openai_use(model: dict[str, Any], feature: str) -> None:
if str(model.get("use") or "") not in _PATCHED_OPENAI_COMPATIBLE_USES:
raise HTTPException(
status_code=422,
detail=(
f"{feature} requires an OpenAI-compatible "
"langchain_openai:ChatOpenAI model"
),
)
if bool(model.get("use_responses_api")):
raise HTTPException(
status_code=422,
detail=(
f"{feature} requires the chat-completions API, "
"not use_responses_api"
),
)
def _validate_write_file_transport(model: dict[str, Any], transport: str) -> None:
if transport != WriteFileTransport.BUFFERED_GATEWAY.value:
return
_validate_patched_openai_use(model, "write_file compatibility")
async def _restore_capability(
store: ModelCapabilityStore,
model_name: str,
previous: dict[str, Any] | None,
request: Request,
) -> None:
"""Best-effort semantic rollback for a failed cross-store model update."""
@@ -386,15 +271,10 @@ async def _restore_capability(
await store.set_reasoning_effort_support(
model_name,
supports_reasoning_effort=bool(previous.get("supports_reasoning_effort", False)),
default_reasoning_effort=str(previous.get("default_reasoning_effort") or "xhigh"),
updated_by=previous.get("updated_by"),
)
except Exception:
logger.exception("Failed to roll back model capability for %s", model_name)
finally:
from app.gateway.catalog_cache import invalidate_catalog
await invalidate_catalog(request, "models")
def _read_config_data(path: Path) -> tuple[str, dict[str, Any], list[dict[str, Any]]]:
@@ -448,42 +328,6 @@ def _replace_models_block(raw_text: str, models: list[dict[str, Any]]) -> str:
return f"{raw_text[: block_match.start()]}{rendered}{raw_text[end:]}"
def _replace_config_file_with_retry(
temp_name: str,
path: Path,
raw_before: str,
raw_after: str,
) -> None:
"""Retry short-lived busy errors from shared/container-mounted filesystems.
Keep the write atomic: never fall back to truncating the live config. The
temporary file is unique per mutation, so a bounded retry is safe when
overlay/NFS scanners briefly hold a rename operation.
"""
if _is_file_mountpoint(path):
logger.info("Model config is a single-file mount; updating mounted file in place")
_write_mounted_config_in_place(path, raw_before, raw_after)
os.unlink(temp_name)
return
for attempt in range(_ATOMIC_REPLACE_RETRIES + 1):
try:
os.replace(temp_name, path)
return
except OSError as exc:
if exc.errno not in _ATOMIC_REPLACE_RETRY_ERRNOS or attempt >= _ATOMIC_REPLACE_RETRIES:
raise
delay = min(0.4, 0.025 * (2**attempt))
logger.warning(
"Model config atomic replace is temporarily busy (errno=%s); retry %d/%d after %.3fs",
exc.errno,
attempt + 1,
_ATOMIC_REPLACE_RETRIES,
delay,
)
sleep(delay)
def _backup_and_write_config(path: Path, raw_before: str, raw_after: str) -> Path:
backup_dir = path.parent / _BACKUP_DIR_NAME
backup_dir.mkdir(parents=True, exist_ok=True)
@@ -501,7 +345,7 @@ def _backup_and_write_config(path: Path, raw_before: str, raw_after: str) -> Pat
handle.write(raw_after)
handle.flush()
os.fsync(handle.fileno())
_replace_config_file_with_retry(temp_name, path, raw_before, raw_after)
os.replace(temp_name, path)
except Exception:
try:
os.unlink(temp_name)
@@ -585,8 +429,6 @@ def _strip_database_owned_fields(models: list[dict[str, Any]]) -> list[dict[str,
async def _save_models(request: Request, raw_before: str, models: list[dict[str, Any]]) -> list[dict[str, Any]]:
from app.gateway.catalog_cache import invalidate_catalog
path = _resolve_config_path()
models = _strip_database_owned_fields(models)
raw_after = _replace_models_block(raw_before, models)
@@ -605,7 +447,6 @@ async def _save_models(request: Request, raw_before: str, models: list[dict[str,
status_code=422,
detail="The model configuration could not be activated; the previous config was restored.",
) from exc
await invalidate_catalog(request, "models")
return models
@@ -622,8 +463,7 @@ async def list_model_configurations(request: Request) -> ModelAdminListResponse:
models=[
_to_admin_item(
item,
supports_reasoning_effort=bool(capabilities.get(str(item.get("name") or ""), {}).get("supports_reasoning_effort", False)),
default_reasoning_effort=str(capabilities.get(str(item.get("name") or ""), {}).get("default_reasoning_effort") or "xhigh"),
supports_reasoning_effort=capabilities.get(str(item.get("name") or ""), False),
)
for item in models
],
@@ -650,19 +490,17 @@ async def create_model_configuration(body: ModelUpsertRequest, request: Request)
await capability_store.set_reasoning_effort_support(
body.name,
supports_reasoning_effort=bool(body.supports_reasoning_effort),
default_reasoning_effort="xhigh",
updated_by=_admin_user_id(admin),
)
try:
await _save_models(request, raw_before, [*models, created])
except Exception:
if capability_changed and capability_store is not None:
await _restore_capability(capability_store, body.name, previous_capability, request)
await _restore_capability(capability_store, body.name, previous_capability)
raise
return _to_admin_item(
created,
supports_reasoning_effort=bool(body.supports_reasoning_effort) if capability_changed else False,
default_reasoning_effort="xhigh",
)
@@ -697,8 +535,7 @@ async def reorder_model_configurations(body: ModelOrderRequest, request: Request
return [
_to_admin_item(
item,
supports_reasoning_effort=bool(capabilities.get(str(item.get("name") or ""), {}).get("supports_reasoning_effort", False)),
default_reasoning_effort=str(capabilities.get(str(item.get("name") or ""), {}).get("default_reasoning_effort") or "xhigh"),
supports_reasoning_effort=capabilities.get(str(item.get("name") or ""), False),
)
for item in ordered_models
]
@@ -715,8 +552,6 @@ async def update_reasoning_effort_capability(
) -> ModelAdminItem:
"""Persist the per-model reasoning-effort selector capability in SQL."""
from app.gateway.catalog_cache import invalidate_catalog
admin = await _require_admin(request)
async with _MUTATION_LOCK:
path = _resolve_config_path()
@@ -730,100 +565,11 @@ async def update_reasoning_effort_capability(
await store.set_reasoning_effort_support(
model_name,
supports_reasoning_effort=body.supports_reasoning_effort,
default_reasoning_effort=body.default_reasoning_effort,
updated_by=_admin_user_id(admin),
)
await invalidate_catalog(request, "models")
return _to_admin_item(
model,
supports_reasoning_effort=body.supports_reasoning_effort,
default_reasoning_effort=body.default_reasoning_effort or "xhigh",
)
@router.put(
"/write-file-transport/{model_name:path}",
response_model=ModelAdminItem,
)
async def update_write_file_transport(
model_name: str,
body: WriteFileTransportToggleRequest,
request: Request,
) -> ModelAdminItem:
"""Toggle one model's buffered-gateway write_file compatibility in config.yaml.
The global system-settings switch keeps overriding every compatible model
while it is enabled; this per-model value becomes effective as soon as the
global switch is turned off.
"""
await _require_admin(request)
async with _MUTATION_LOCK:
path = _resolve_config_path()
raw_before, _, models = await asyncio.to_thread(_read_config_data, path)
index = next((i for i, item in enumerate(models) if str(item.get("name")) == model_name), None)
if index is None:
raise HTTPException(status_code=404, detail=f"Model '{model_name}' was not found")
updated = copy.deepcopy(models[index])
_validate_write_file_transport(updated, body.write_file_transport)
updated["write_file_transport"] = body.write_file_transport
try:
ModelConfig.model_validate(updated)
except Exception as exc:
raise HTTPException(status_code=422, detail=f"Invalid model configuration: {exc}") from exc
next_models = [*models]
next_models[index] = updated
await _save_models(request, raw_before, next_models)
capabilities = await _capabilities_by_name(request)
capability = capabilities.get(model_name, {})
return _to_admin_item(
updated,
supports_reasoning_effort=bool(capability.get("supports_reasoning_effort", False)),
default_reasoning_effort=str(capability.get("default_reasoning_effort") or "xhigh"),
)
@router.put(
"/strict-stream-integrity/{model_name:path}",
response_model=ModelAdminItem,
)
async def update_strict_stream_integrity(
model_name: str,
body: StrictStreamIntegrityToggleRequest,
request: Request,
) -> ModelAdminItem:
"""Toggle one model's strict stream integrity check in config.yaml.
When enabled, a provider stream that ends without a terminal frame raises
a retriable ``IncompleteChatStreamError`` instead of committing a half
answer — the LLM error middleware then retries the call. Explicit ``false``
also overrides the auto-enable default for the affected qwen3.8-flash-next
model id; omitting the key entirely keeps that default.
"""
await _require_admin(request)
async with _MUTATION_LOCK:
path = _resolve_config_path()
raw_before, _, models = await asyncio.to_thread(_read_config_data, path)
index = next((i for i, item in enumerate(models) if str(item.get("name")) == model_name), None)
if index is None:
raise HTTPException(status_code=404, detail=f"Model '{model_name}' was not found")
updated = copy.deepcopy(models[index])
_validate_patched_openai_use(updated, "strict stream integrity")
updated["strict_stream_integrity"] = body.strict_stream_integrity
try:
ModelConfig.model_validate(updated)
except Exception as exc:
raise HTTPException(status_code=422, detail=f"Invalid model configuration: {exc}") from exc
next_models = [*models]
next_models[index] = updated
await _save_models(request, raw_before, next_models)
capabilities = await _capabilities_by_name(request)
capability = capabilities.get(model_name, {})
return _to_admin_item(
updated,
supports_reasoning_effort=bool(capability.get("supports_reasoning_effort", False)),
default_reasoning_effort=str(capability.get("default_reasoning_effort") or "xhigh"),
)
@@ -850,7 +596,6 @@ async def update_model_configuration(model_name: str, body: ModelUpsertRequest,
await capability_store.set_reasoning_effort_support(
model_name,
supports_reasoning_effort=desired_capability,
default_reasoning_effort=str((previous_capability or {}).get("default_reasoning_effort") or "xhigh"),
updated_by=_admin_user_id(admin),
)
next_models = [*models]
@@ -859,7 +604,7 @@ async def update_model_configuration(model_name: str, body: ModelUpsertRequest,
await _save_models(request, raw_before, next_models)
except Exception:
if capability_changed and capability_store is not None:
await _restore_capability(capability_store, model_name, previous_capability, request)
await _restore_capability(capability_store, model_name, previous_capability)
raise
if capability_changed:
supports_reasoning_effort = desired_capability
@@ -871,7 +616,6 @@ async def update_model_configuration(model_name: str, body: ModelUpsertRequest,
return _to_admin_item(
updated,
supports_reasoning_effort=supports_reasoning_effort,
default_reasoning_effort=str((previous_capability or {}).get("default_reasoning_effort") or "xhigh"),
)
@@ -894,7 +638,7 @@ async def delete_model_configuration(model_name: str, request: Request) -> None:
await _save_models(request, raw_before, [*models[:index], *models[index + 1 :]])
except Exception:
if capability_store is not None:
await _restore_capability(capability_store, model_name, previous_capability, request)
await _restore_capability(capability_store, model_name, previous_capability)
raise
@@ -1,10 +1,8 @@
import logging
from typing import Literal
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.catalog_cache import cached_catalog
from app.gateway.deps import get_config
from deerflow.config.app_config import AppConfig
from deerflow.config.model_config import ModelConfig
@@ -38,7 +36,6 @@ def _build_model_response(
model: ModelConfig,
*,
supports_reasoning_effort: bool = False,
default_reasoning_effort: str = "xhigh",
) -> "ModelResponse":
extra = model.model_extra or {}
provider = str(extra["provider"]) if "provider" in extra else _derive_provider(model.use)
@@ -51,7 +48,6 @@ def _build_model_response(
description=model.description,
supports_thinking=model.supports_thinking,
supports_reasoning_effort=bool(supports_reasoning_effort and model.supports_thinking),
default_reasoning_effort=default_reasoning_effort if default_reasoning_effort in {"low", "medium", "xhigh"} else "xhigh",
supports_vision=model.supports_vision,
supports_tool_calling=supports_tool_calling,
is_local=is_local,
@@ -68,7 +64,6 @@ class ModelResponse(BaseModel):
description: str | None = Field(None, description="Model description")
supports_thinking: bool = Field(default=False, description="Whether model supports thinking mode")
supports_reasoning_effort: bool = Field(default=False, description="Whether model supports reasoning effort")
default_reasoning_effort: Literal["low", "medium", "xhigh"] = Field(default="xhigh", description="Globally configured default reasoning effort")
supports_vision: bool = Field(default=False, description="Whether model supports image inputs")
supports_tool_calling: bool = Field(default=True, description="Whether model supports tool/function calling")
is_local: bool = Field(default=False, description="Whether this is a locally-hosted model")
@@ -88,22 +83,18 @@ class ModelsListResponse(BaseModel):
token_usage: TokenUsageResponse
async def _reasoning_capabilities(request: Request) -> dict[str, dict[str, object]]:
async def _reasoning_capabilities(request: Request) -> dict[str, bool]:
store = getattr(request.app.state, "model_capability_store", None)
if store is None:
return {}
async def load() -> dict[str, dict[str, object]]:
rows = await store.list_capabilities()
return {str(row.get("model_name") or ""): row for row in rows if row.get("model_name")}
try:
return await cached_catalog(request, "models", "reasoning-capabilities", load)
rows = await store.list_capabilities()
except Exception:
# Capability reads fail closed: the selector disappears, while ordinary
# model use remains available.
logger.warning("Failed to read model capabilities", exc_info=True)
return {}
return {str(row.get("model_name") or ""): bool(row.get("supports_reasoning_effort", False)) for row in rows if row.get("model_name")}
@router.get(
@@ -152,8 +143,7 @@ async def list_models(request: Request, config: AppConfig = Depends(get_config))
models = [
_build_model_response(
model,
supports_reasoning_effort=bool(capabilities.get(model.name, {}).get("supports_reasoning_effort", False)),
default_reasoning_effort=str(capabilities.get(model.name, {}).get("default_reasoning_effort") or "xhigh"),
supports_reasoning_effort=capabilities.get(model.name, False),
)
for model in config.models
]
@@ -198,6 +188,5 @@ async def get_model(model_name: str, request: Request, config: AppConfig = Depen
capabilities = await _reasoning_capabilities(request)
return _build_model_response(
model,
supports_reasoning_effort=bool(capabilities.get(model.name, {}).get("supports_reasoning_effort", False)),
default_reasoning_effort=str(capabilities.get(model.name, {}).get("default_reasoning_effort") or "xhigh"),
supports_reasoning_effort=capabilities.get(model.name, False),
)
@@ -61,6 +61,7 @@ from typing import Any
import httpx
from fastapi import APIRouter, Body, HTTPException, Request
from fastapi.responses import StreamingResponse
from langgraph.checkpoint.base import empty_checkpoint
from app.gateway.roundtable_diag import record_foreground_diag
from app.gateway.roundtable_model_fallback import fallback_model_chain, is_llm_error_message, reason_text
@@ -81,18 +82,11 @@ from deerflow.agents.roundtable_orchestrator.extraction_spec import build_summar
from deerflow.config import get_app_config
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.tools.builtins.clarification_utils import resolve_allow_multiple
from deerflow.utils.time import now_iso
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/multi-agent", tags=["multi-agent"])
_STEP3_AGENT_IDS = frozenset({
ACTION_PLAN_AGENT_ID,
DASHBOARD_AGENT_ID,
REPORT_AGENT_ID,
STRUCTURE_AGENT_ID,
SUMMARY_AGENT_ID,
})
def _stage_for_agent(agent_type: str, agent_name: str) -> str:
"""把一次 /run/stream 的 (agent_type, agent_name) 映射成诊断日志的 stage。
@@ -113,62 +107,12 @@ def _stage_for_agent(agent_type: str, agent_name: str) -> str:
return "step2_leader" if agent_type == "leader" else "step2_seat"
def _is_step2_room_limited(
agent_type: str,
agent_name: str,
*,
finalize_only: bool = False,
) -> bool:
"""Only the actual Step-2 coordinator and seats consume a room.
The Step-3 singleton agents reuse the same ``/run/stream`` endpoint, so
this boundary must live here rather than at the route level.
"""
# 最后总控只把已经完成的席位交付整理成正文,不再占用席位执行资源。允许它在
# 长耗时席位跑过房间 hard deadline 后仍完成收口,避免最后一步被 409 踢回第一步。
if agent_type == "leader" and finalize_only:
return False
return agent_type == "leader" or agent_name not in _STEP3_AGENT_IDS
# 圆桌(intent / recommend / 席位 / 协调)创建的 thread 都是功能型内部线程,
# 不是用户的普通对话。打上这个 metadata,前端「最近的对话」列表(recent-chat-list.tsx)
# 据 `metadata.thread_type` 过滤掉,与 scheduler / bootstrap 系统会话同一套机制。
_ROUNDTABLE_THREAD_METADATA: dict[str, Any] = {"thread_type": "roundtable", "system": True}
def _room_thread_metadata(
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None,
) -> dict[str, Any]:
"""Step-2 threads carry a stable room key so later occupancy can see live runs."""
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
from deerflow.persistence.roundtable_rooms.sql import compute_room_session_key
meta = dict(_ROUNDTABLE_THREAD_METADATA)
clean_session_id = (room_session_id or "").strip() or None
key = compute_room_session_key(clean_session_id) or compute_active_dedupe_key(
draft_id, user_id, task_id
)
if key:
meta["roundtable_room_key"] = key
if clean_session_id:
meta["roundtable_room_session_id"] = clean_session_id
if draft_id:
meta["draft_id"] = draft_id
if task_id:
meta["task_id"] = task_id
return meta
def _step3_thread_metadata() -> dict[str, Any]:
"""Step-3 threads are internal threads, but never Step-2 room occupants."""
return dict(_ROUNDTABLE_THREAD_METADATA)
# ---------------------------------------------------------------------------
# 共享:异常诊断器(intent/recommend/multi-agent 共用)
# ---------------------------------------------------------------------------
@@ -471,17 +415,59 @@ def _resolve_user_id(request: Request) -> str:
async def _create_thread_direct(request: Request, *, metadata: dict[str, Any] | None = None) -> str:
"""同进程版的 ``POST /api/threads``。
只创建业务元数据,首轮图运行负责初始化 checkpoint。
metadata 的任务共享/系统会话规则与原入口保持一致。
"""
from app.gateway.deps import get_thread_store
完全镜像 ``routers/threads.create_thread`` 的写盘动作(thread_meta +
空 checkpoint),但跳过 TCP + Pydantic 反序列化。仅供 init 阶段批量建席位
线程用,失败时抛 HTTPException 由 SSE 帧上报。
``metadata`` 写入 thread 元数据;init 传 ``_ROUNDTABLE_THREAD_METADATA`` 把
席位 / 协调线程标记成系统会话,从前端「最近的对话」里隐藏。
"""
from app.gateway.deps import get_checkpointer, get_thread_store
checkpointer = get_checkpointer(request)
thread_store = get_thread_store(request)
thread_id = str(uuid.uuid4())
now = now_iso()
try:
await get_thread_store(request).create(thread_id, assistant_id=None, metadata=metadata or {})
await thread_store.create(thread_id, assistant_id=None, metadata=metadata or {})
except Exception:
logger.exception("[init-direct] thread_meta create failed for %s", thread_id)
raise HTTPException(status_code=500, detail="Failed to create thread")
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
ckpt_metadata = {
"step": -1,
"source": "input",
"writes": None,
"parents": {},
"created_at": now,
}
async def _cleanup_partial_thread() -> None:
delete_checkpoint = getattr(checkpointer, "adelete_thread", None)
if callable(delete_checkpoint):
try:
await delete_checkpoint(thread_id)
except Exception:
logger.exception("[init-direct] partial checkpoint cleanup failed for %s", thread_id)
try:
await thread_store.delete(thread_id)
except Exception:
logger.exception("[init-direct] partial thread_meta cleanup failed for %s", thread_id)
try:
await checkpointer.aput(config, empty_checkpoint(), ckpt_metadata, {})
except asyncio.CancelledError:
# A sibling seat may fail while this task is between metadata commit and
# checkpoint creation. Compensate before propagating cancellation, under
# shield so a second cancellation cannot abort the cleanup half-way.
await asyncio.shield(_cleanup_partial_thread())
raise
except Exception:
logger.exception("[init-direct] checkpoint create failed for %s", thread_id)
await _cleanup_partial_thread()
raise HTTPException(status_code=500, detail="Failed to create thread")
return thread_id
@@ -732,23 +718,18 @@ def _run_payload(
new_message: str,
excluded_tools: list[str],
skill_stop_names: list[str],
allowed_tools: list[str] | None = None,
recursion_limit: int = 1000,
thinking_enabled: bool = True,
reasoning_effort: str = "medium",
subagent_enabled: bool = False,
stream_custom: bool = False,
multitask_strategy: str = "reject",
cancel_on_disconnect: bool = False,
extra_context: dict[str, Any] | None = None,
files: list[dict[str, Any]] | None = None,
roundtable_file_stream: bool = False,
) -> dict[str, Any]:
"""构造 ``POST /api/threads/{thread_id}/runs/stream`` 的请求体。
- ``context.agent_name``:本次 run 使用的 agent(内置 id 或 init 创建的 coordinator id)。
- ``excluded_tools``:按路由裁剪工具面(intent 仅澄清、special 禁派活等)。
- ``allowed_tools``:安全敏感角色的真正白名单;``None`` 保持兼容,空列表禁用全部。
- ``skill_stop_names``:leader 侧配合 SkillStopMiddleware 在 ``agent_orchestration``
调用后截断图,以便网关从 AIMessage.tool_calls 解析派活列表。
- ``thinking_enabled`` / ``reasoning_effort``:默认开 CoT + medium,适合 Step 1 澄清
@@ -773,7 +754,7 @@ def _run_payload(
]
},
"metadata": {"run_scope": RUN_SCOPE_ORCHESTRATION_CHILD},
"config": {"recursion_limit": recursion_limit},
"config": {"recursion_limit": 1000},
"context": {
"agent_name": agent_name,
"model_name": model_name,
@@ -797,7 +778,6 @@ def _run_payload(
# 席位 → run/stream 报 400 "agent_name ... missing from d_agent_thread_id"。
# 圆桌是一次性研讨,本就不该吃历史会话记忆,这里统一关注入根治该污染。
"memory_injection_enabled": False,
"allowed_tools": allowed_tools,
},
# ultra 席位会派子智能体(task 工具),其 task_started/running/completed 进度走
# **custom** 流(StreamWriter)。仅这种场景加 "custom",让前端任务卡片能显示子智能体
@@ -808,10 +788,13 @@ def _run_payload(
"stream_subgraphs": True,
"stream_resumable": True,
"assistant_id": "lead_agent",
# 第二步实时会商不支持脱离浏览器继续执行。断网、暂停或前端主动 abort 后必须
# 取消后端 run;第三步等既有调用保持 continue,不改变其断线恢复语义。
"on_disconnect": "cancel" if cancel_on_disconnect else "continue",
# rollback 仍保留,负责处理人工干预与新一轮抢占同一 thread 的极短竞态。
"on_disconnect": "continue",
# ⚠️ 圆桌 run 必须能被「下一次同 thread 的 run」抢占。圆桌每个 thread 同一时刻
# 只应有一个活跃 run,但 on_disconnect="continue" 让前端 abort SSE 后**后端 run
# 仍在跑**(为可恢复);此时用户「人工干预」立刻发起新 leader run 打到同一总控
# thread,默认 reject 策略就会 409「already has an active run」→ 被 _stream_upstream
# 包成 502。调用方(leader / special)传 multitask_strategy="rollback",让新 run 先
# 取消该 thread 上没结束的旧 run 再继续 —— 这正是「最新一轮/干预指令优先」的语义。
"multitask_strategy": multitask_strategy,
"excluded_tools": excluded_tools,
"skill_stop_names": skill_stop_names,
@@ -820,20 +803,9 @@ def _run_payload(
# 供 read_peer_delivery 工具按需读取)。只进 context dict,不进 prompt → 不调工具就不耗 token。
if extra_context:
payload["context"].update(extra_context)
if roundtable_file_stream:
payload["context"].update(_roundtable_file_stream_context())
return payload
def _roundtable_file_stream_context() -> dict[str, bool]:
"""会商专用文件流式兼容。关闭时不写 context,本轮仍只看全局开关。"""
from deerflow.config.system_settings import get_roundtable_room_limit_settings
if get_roundtable_room_limit_settings().write_file_compatibility_enabled:
return {"roundtable_write_file_compatibility": True}
return {}
# ---------------------------------------------------------------------------
# POST /api/multi-agent/init — Step 2 席位 + 总控初始化(SSE)
# ---------------------------------------------------------------------------
@@ -854,7 +826,7 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
SSE 事件(每条 ``data: {...}\\n\\n``)::
seat_ready — 某一席位 thread 已创建(``asyncio.as_completed`` 顺序,先完先发)
coordinator_ready — 总控 thread 元数据就绪,席位列表随首次问答注入
coordinator_ready — 总控 thread 就绪 + 已注入本次席位列表
init_done — ``thread_ids`` 全量映射,前端存 ``threadIds`` 并开始编排
error — 失败详情(同步校验失败则直接 HTTP 4xx,不进 SSE)
@@ -865,9 +837,9 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
在首次调用时自动落地。每次 init **不再创建新的 coordinator agent**(避免
``.deer-flow/agents/`` 累积 ``roundtable-coordinator-<ts36>`` 残留)。
初始化只创建业务线程元数据,不访问 checkpoint。每轮 leader run 根据 thread
映射注入真实席位清单,随实际输入一起持久化。SOUL 长期稳定,席位列表随会话
动态注入。前端为兼容旧 client 仍可传
本次圆桌的「可调度席位列表」由 init 在创建好 coordinator thread 之后,以一条
human 消息追加到 thread 历史里,leader run 每轮都能从对话历史里读到。这样
SOUL 长期稳定,席位列表随会话动态注入。前端为兼容旧 client 仍可传
``main_agent_name`` 字段,但后端会忽略并使用 ``COORDINATOR_AGENT_ID``。
"""
# 调用前自检:与 intent/recommend 一样,缺失时就地补建,避免 /run/stream 报 500。
@@ -875,6 +847,7 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
agent_names_in = payload.get("agent_names") or []
agent_names: list[str] = [str(name).lower() for name in agent_names_in]
model: str | None = payload.get("model")
# main_agent_name 是历史字段,2026-05 改造后忽略,统一用单例 id。保留接收但不再用前端值,
# 避免下游再次创建带时间戳的临时 coordinator。
main_agent_name = COORDINATOR_AGENT_ID
@@ -885,26 +858,6 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
for name in agent_names:
_validate_agent_id(name, "agent_names entry")
# The browser already acquired the room before entering Step 2. Init only
# renews that lease; it cannot create a second room or bypass admission.
from app.gateway.roundtable_room_gate import enforce_existing_roundtable_room
room_draft_id = str(payload.get("draft_id") or "").strip() or None
room_task_id = str(payload.get("task_id") or "").strip() or None
room_session_id = str(payload.get("room_session_id") or "").strip() or None
await enforce_existing_roundtable_room(
request,
draft_id=room_draft_id,
task_id=room_task_id,
room_session_id=room_session_id,
)
room_thread_meta = _room_thread_metadata(
draft_id=room_draft_id,
user_id=_resolve_user_id(request),
task_id=room_task_id,
room_session_id=room_session_id,
)
def _frame(payload: dict[str, Any]) -> str:
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
@@ -920,10 +873,10 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
agent_meta = await _get_agents_batch_direct(request, agent_names)
t_meta_done = time.perf_counter()
# 2) 每个席位 thread 元数据并行创建,不预写空 checkpoint,
# 2) 每个席位 thread 并行创建(直调 thread_store + checkpointer),
# 用 asyncio.as_completed 保持"先完先发 seat_ready"的体验。
async def _prepare_sub_agent(sub_id: str) -> tuple[str, str, str, str]:
thread_id = await _create_thread_direct(request, metadata=room_thread_meta)
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
meta = agent_meta.get(sub_id.lower(), {})
return (
sub_id.lower(),
@@ -955,11 +908,50 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
t_seats_done = time.perf_counter()
agent_threads: dict[str, str] = {sid: thread_id for sid, thread_id, _, _ in prepared}
# SOUL 使用固定总控单例。席位清单由 _leader_run 随实际输入注入,
# 初始化不通过 loopback 追加消息,也无需创建任何空 checkpoint。
coord_thread = await _create_thread_direct(request, metadata=room_thread_meta)
seat_lines: list[str] = [
f"- agent_name: {sid}({display_name}):{description}"
for sid, _, display_name, description in prepared
]
# 协调智能体改造为**单例**(id = roundtable-coordinator):
# - SOUL 已经由 ensure_roundtable_functional_agents() 在路由入口落地,
# 通用协调规则常驻磁盘,无需每次重写。
# - 本次"可调度席位清单"以 human 消息形式 append 到 coordinator
# thread 历史里,leader run 每轮都能在对话历史最前面看到。
# 这样席位列表随会话动态变化,但磁盘上不再累积 roundtable-coordinator-<ts36>。
#
# 注意:model 字段在 lead_agent run 时通过 context.model_name 注入,
# 不需要写回到 agent config.yaml(否则会污染单例的配置)。
seat_introduction = (
"【圆桌系统初始化】本次研讨可调度的席位清单如下,请严格按 agent_name "
"通过 agent_orchestration 技能派活,不要派给不在清单内的 agent:\n"
+ "\n".join(seat_lines)
)
coord_thread = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
agent_threads[main_agent_name] = coord_thread
# 把席位列表注入 coordinator thread 的对话历史。HTTP loopback 调用
# 走 /api/threads/{tid}/state(读+写),开销 ~10-30ms,在 init 整体
# 几百 ms 的尺度上微不足道,但比起手撕 langgraph checkpoint 内部结构
# 风险低、与 _broadcast / _append_thread_message 一套语义,长期可维护。
try:
base = _loopback_base(request)
headers = _auth_headers(request)
async with _make_loopback_client(request) as seed_client:
await _append_thread_message(
seed_client,
base,
headers,
coord_thread,
seat_introduction,
)
except Exception:
# 注入失败不阻断 init:leader run 仍能跑,只是看不到席位清单,
# 可能派活给错的 id。会被 _strip_display_suffix / _validate_agent_id
# 兜底,前端能看到错误帧;运维通过 backend 日志找到根因再修复。
logger.exception("[init-direct] failed to seed coordinator thread with seat list")
t_coord_done = time.perf_counter()
logger.info(
"[init-direct] meta=%.3fs seats=%.3fs coord=%.3fs total=%.3fs seats_count=%d",
@@ -1009,10 +1001,7 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
@router.post("/report/init")
async def init_report_thread(
request: Request,
payload: dict | None = Body(default=None),
) -> dict[str, str]:
async def init_report_thread(request: Request) -> dict[str, str]:
"""为 Step 3「结果绘制」创建一个供方案可视化总结智能体(``roundtable-report``)
运行的线程,返回 ``{"agent_name", "thread_id"}``。
@@ -1027,10 +1016,7 @@ async def init_report_thread(
"""
try:
ensure_roundtable_functional_agents()
thread_id = await _create_thread_direct(
request,
metadata=_step3_thread_metadata(),
)
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
except Exception as exc:
await record_foreground_diag(
request, stage="step3_report", level="error", event="thread_init_failed",
@@ -1042,10 +1028,7 @@ async def init_report_thread(
@router.post("/summary/init")
async def init_summary_thread(
request: Request,
payload: dict | None = Body(default=None),
) -> dict[str, str]:
async def init_summary_thread(request: Request) -> dict[str, str]:
"""为 Step 3「总结报告」创建一个供方案总结报告智能体(``roundtable-summary``)
运行的线程,返回 ``{"agent_name", "thread_id"}``。
@@ -1061,10 +1044,7 @@ async def init_summary_thread(
"""
try:
ensure_roundtable_functional_agents()
thread_id = await _create_thread_direct(
request,
metadata=_step3_thread_metadata(),
)
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
except Exception as exc:
await record_foreground_diag(
request, stage="step3_summary", level="error", event="thread_init_failed",
@@ -1102,10 +1082,7 @@ async def init_position_action_plan_thread(request: Request) -> dict[str, str]:
@router.post("/dashboard/init")
async def init_dashboard_thread(
request: Request,
payload: dict | None = Body(default=None),
) -> dict[str, str]:
async def init_dashboard_thread(request: Request) -> dict[str, str]:
"""为 Step 3「大屏多页」创建一个供大屏多页数据智能体(``roundtable-dashboard``)
运行的线程,返回 ``{"agent_name", "thread_id"}``。
@@ -1123,10 +1100,7 @@ async def init_dashboard_thread(
"""
try:
ensure_roundtable_functional_agents()
thread_id = await _create_thread_direct(
request,
metadata=_step3_thread_metadata(),
)
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
except Exception as exc:
await record_foreground_diag(
request, stage="step3_dashboard", level="error", event="thread_init_failed",
@@ -1138,10 +1112,7 @@ async def init_dashboard_thread(
@router.post("/structure/init")
async def init_structure_thread(
request: Request,
payload: dict | None = Body(default=None),
) -> dict[str, str]:
async def init_structure_thread(request: Request) -> dict[str, str]:
"""为 Step 3「入库」创建一个供结构化抽取智能体(``roundtable-structure``)运行的线程,
返回 ``{"agent_name", "thread_id"}``。
@@ -1159,10 +1130,7 @@ async def init_structure_thread(
"""
try:
ensure_roundtable_functional_agents()
thread_id = await _create_thread_direct(
request,
metadata=_step3_thread_metadata(),
)
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
except Exception as exc:
await record_foreground_diag(
request, stage="step3_structure", level="error", event="thread_init_failed",
@@ -1213,6 +1181,7 @@ async def _stream_upstream(
本机往返通常 <10ms),怀疑 HTTP_PROXY 拦了回环——检查 client 是否走 _make_loopback_client
(trust_env=False)。
"""
url = f"{base}/api/threads/{thread_id}/runs/stream"
captured: list[str] = []
agent_for_log = (body.get("context") or {}).get("agent_name") or thread_id
@@ -1220,8 +1189,8 @@ async def _stream_upstream(
# 头阶段非 200 的瞬时重试。
# 背景:网关是**单 worker** + checkpointer 是**单连接 aiosqlite**(全进程一把
# asyncio.Lock 串行 checkpoint 读写)。圆桌第三步进入时,前端把「方案总结/结果绘制/
# 大屏」三个重 run **同一刻并行**打出来,若再叠加尚未完成取消收尾的第二步席位 run,
# 瞬时把单锁/单 worker 打满 —— 此时回环子请求 /runs/stream 的 start_run
# 大屏」三个重 run **同一刻并行**打出来,叠加第二步 on_disconnect=continue 仍在后台
# 跑的席位 run,瞬时把单锁/单 worker 打满 —— 此时回环子请求 /runs/stream 的 start_run
# 里那串 DB / checkpoint 写有概率抛错(或同 thread 抢占竞态),返回非 200,被本函数原样
# 包成 502 漏给前端(用户侧表现为"进第三步概率性 502")。
# 关键安全前提:这个非 200 发生在**响应头阶段**,本函数**尚未 yield 任何 SSE 行**给下游,
@@ -1584,7 +1553,6 @@ async def _leader_run(
model_name: str,
seat_meta: dict[str, dict[str, Any]] | None = None,
synthesis_mode: bool = False,
finalize_only: bool = False,
suppress_pre_broadcast: bool = True,
files: list[dict[str, Any]] | None = None,
) -> AsyncIterator[str | dict[str, Any]]:
@@ -1616,9 +1584,6 @@ async def _leader_run(
# new_message),绝不阻断 leader run。这些只进总控**本轮 run 输入**,不广播给席位。
seat_status_block = ""
full_deliverables_block = ""
# Older clients may omit names/descriptions; the thread mapping still
# supplies the authoritative IDs, including on a checkpoint-free first run.
seat_meta = seat_meta or {sid: {} for sid in agent_threads if sid != agent_name}
if seat_meta:
seat_ids = [name for name in agent_threads if name != agent_name]
if seat_ids:
@@ -1635,9 +1600,10 @@ async def _leader_run(
undelivered = [sid for sid in seat_ids if not delivery.get(sid, (False, ""))[0]]
seats_meta_list = [(sid, seat_meta.get(sid, {})) for sid in seat_ids]
seat_status_block = _build_seat_status_block(seats_meta_list, delivery, undelivered_count=len(undelivered))
# 普通综合轮只有全部席位交付后才注入全文,避免弱模型过早收口;显式强制收口轮
# 即使仍有失败席位,也要注入其余有效交付并在最终结论中如实标明缺口。
if finalize_only or (synthesis_mode and not undelivered):
# 第 2 阶段(仅「综合轮 且 所有席位都已交付」):再**全文**取各已交付席位的交付,拼综合块。
# 还有 ⬜ 未交付席位时不注入 —— 否则"综合输出最终方案"的指令会怂恿弱模型在席位没集齐时
# 就提前收口(实测:3/5 交付就臆称"三方已发言"草草收口)。全文读只在这一轮才付出。
if synthesis_mode and not undelivered:
try:
full_delivery = await _collect_delivery_status(
client, base, headers,
@@ -1655,14 +1621,6 @@ async def _leader_run(
len(undelivered),
bool(full_deliverables_block),
)
if finalize_only:
new_message = (
f"{new_message}\n\n"
"【最终收口·纯正文】本轮所有工具均已关闭。请直接根据上方席位交付输出一份合并后的最终结论正文;"
"不要按席位逐份复述原文,也不要反复改写同一份总结。"
"不要调用或尝试 write_file、present_files、agent_orchestration、skill_view 或任何其它工具,"
"不要再派活,也不要只描述你准备做什么。"
)
leader_input = "\n\n".join(
part for part in (seat_status_block, full_deliverables_block, new_message) if part
)
@@ -1679,14 +1637,12 @@ async def _leader_run(
# 带了上传文件时,给总控解锁「读」类工具(leader policy 默认禁 read_file/ls/view_image),
# 否则文件清单注入了却无工具读取。写/派活类仍按 policy 封锁,不放开。
leader_excluded = policy.excluded_tools
leader_allowed = [] if finalize_only else list(getattr(policy, "allowed_tools", None) or [])
if files and not finalize_only:
if files:
_read_tools = {"read_file", "ls", "grep", "view_image"}
leader_excluded = [t for t in leader_excluded if t not in _read_tools]
leader_allowed.extend(sorted(_read_tools))
# 模型自动容错:按 config.yaml models[] 顺序的候选链。当前模型返回明确的错误标记/
# 兜底错误文案时才切到下一个模型重跑;没有正文或派活不触发换模。
# 模型自动容错:按 config.yaml models[] 顺序的候选链。当前模型出问题(兜底错误文案 /
# 既无正文又无派活)时切到下一个模型重跑(见循环内检测)。``_mk_body`` 用候选模型重建 body。
model_chain = fallback_model_chain(model_name) or [model_name]
model_idx = 0
current_model = model_chain[0]
@@ -1699,17 +1655,13 @@ async def _leader_run(
# 注入了席位清单的输入(leader_input);seat_roster 为空时等同 new_message。
new_message=leader_input,
excluded_tools=leader_excluded,
allowed_tools=list(dict.fromkeys(leader_allowed)),
skill_stop_names=effective_stop_names,
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
# 人工干预/恢复会重发 leader run 到同一总控 thread;抢占掉没结束的旧 run,避免 409。
multitask_strategy="rollback",
cancel_on_disconnect=True,
# 用户随干预上传的参考文件 → 总控可用 read_file 等工具读取后再派活。
files=files,
roundtable_file_stream=True,
)
body = _mk_body(current_model)
@@ -1777,11 +1729,11 @@ async def _leader_run(
leader_text = _flatten_content(dispatch_msg.get("content", ""))
tool_calls = dispatch_msg.get("tool_calls") or []
# 模型自动容错通常只识别真正的模型错误。总控暂时没有正文/派活不代表模型故障,不能据此
# 擅自换掉用户选择的模型(部分模型会长时间 reasoning 或连续调用工具)。唯一例外是:
# 无工具的最终收口请求已经完整结束却仍无正文,此时该次结果客观不可交付,才换下一个模型。
# 模型自动容错:本轮总控产出是「中间件兜底错误文案」或「既无正文又无派活」(模型没干活)
# → 若还有备选模型,切到下一个重跑(不消耗派活纠正预算)。先发 model_switch 帧让前端把
# 这条失败气泡重置,再用新模型重新流式。最后一个模型仍失败则照常往下走(错误文案当共识展示)。
fail_reason = is_llm_error_message(dispatch_msg)
if fail_reason is None and finalize_only and not (leader_text or "").strip():
if fail_reason is None and not (leader_text or "").strip() and not tool_calls:
fail_reason = "empty"
if fail_reason and model_idx + 1 < len(model_chain):
model_idx += 1
@@ -1798,16 +1750,6 @@ async def _leader_run(
current_model = next_model
body = _mk_body(current_model)
continue
if fail_reason:
yield {
"status": "error",
"role": "leader",
"agent_name": agent_name,
"model_used": current_model,
"reason": reason_text(fail_reason),
"content": leader_text,
}
return
# 澄清分支:与 intent.py 相同字段;status 为字符串 "clarification"(非数组),
# 前端据此暂停 runOrchestration 循环,展示 Step 2 澄清条。
@@ -1961,9 +1903,11 @@ async def _special_run(
directive = build_summary_directive(business_code)
if directive:
seat_message = f"{new_message}\n\n---\n\n{directive}"
# 模型自动容错:只在真正的模型错误(中间件错误标记/兜底错误文案)时换下一个模型。
# 「仅有思考 / 暂无可见正文 / 交付为空」不是模型故障,不能触发换模;这些状态仍在本轮
# 结束后按 invalid_delivery 交给既有交付校验处理。
# 模型自动容错:按 config.yaml models[] 顺序的候选链。某模型出问题(兜底错误文案;
# 真正的研讨席位还含「交付为空」)→ 发 model_switch 帧让前端重置该席位气泡,再用下一个
# 模型重跑。Step3 单例(report/summary/dashboard/ingest)的产物是落盘文件,正文可能很短,
# 故**不**把「正文为空」当失败(与后台 run_report/summary 的 validate=None 一致),只在真正
# 的模型错误时换模型。
model_chain = fallback_model_chain(model_name) or [model_name]
empty_is_failure = role in ("seat", "seat_parallel")
@@ -1974,9 +1918,7 @@ async def _special_run(
thread_id=thread_id,
new_message=seat_message,
excluded_tools=seat_excluded,
allowed_tools=getattr(policy, "allowed_tools", None),
skill_stop_names=merge_skill_stop_names(policy.skill_stop_names, skill_stop_names),
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
# 席位执行模式覆盖 policy 的 thinking/reasoning(None → 回退 policy 默认);
# subagent 仅 ultra 档置真。工具裁剪(excluded_tools)仍完全由 policy 决定。
thinking_enabled=(policy.thinking_enabled if thinking_enabled is None else thinking_enabled),
@@ -1986,14 +1928,12 @@ async def _special_run(
stream_custom=subagent_enabled,
# 干预/重新派活会把同一席位 thread 再跑一次;抢占掉没结束的旧 run,避免 409。
multitask_strategy="rollback",
cancel_on_disconnect=role in ("seat", "seat_parallel"),
# 前序席位完整交付 → run context,供 read_peer_delivery 工具按需取全文(§12.9)。
extra_context=(
{"roundtable_peer_deliveries": peer_deliveries} if peer_deliveries else None
),
# 用户随干预上传的参考文件 → 席位可用 read_file 等工具读取。
files=files,
roundtable_file_stream=True,
)
t_payload_built = time.perf_counter()
@@ -2064,8 +2004,10 @@ async def _special_run(
visible = _visible_delivery_from_messages(messages)
delivery_kind = classify_seat_delivery(content) if content.strip() else ("valid" if visible else "empty")
# 模型失败检测:只认中间件标记或兜底错误文案。思考较长/暂无正文绝不触发换模。
# 模型失败检测:兜底错误文案;研讨席位再加「无可见正文」(思考不算)。还有备选模型则换。
fail_reason = is_llm_error_message(last_msg)
if fail_reason is None and empty_is_failure and delivery_kind != "valid":
fail_reason = "thinking_only" if delivery_kind == "thinking_only" else "empty"
if fail_reason and idx + 1 < len(model_chain):
next_model = model_chain[idx + 1]
logger.warning("[special-fallback] %s 模型 %s 失败(%s),切换到 %s", agent_name, candidate, fail_reason, next_model)
@@ -2079,19 +2021,8 @@ async def _special_run(
}
continue
if fail_reason:
yield {
"status": "error",
"role": role,
"agent_name": agent_name,
"model_used": candidate,
"reason": reason_text(fail_reason),
"content": visible or content,
}
return
if empty_is_failure and delivery_kind != "valid":
logger.warning("[special-delivery] %s 本轮未产出可见正文(%s),不触发换模", agent_name, delivery_kind)
logger.warning("[special-fallback] %s 所有模型均未产出可见正文(%s)", agent_name, delivery_kind)
yield {
"status": "invalid_delivery",
"reason": delivery_kind,
@@ -2156,7 +2087,6 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
# 额外把各已交付席位的完整交付正文注入总控输入,供其写最终方案。缺省 False(派活决策轮
# 只给摘要,保持上下文精简)。旧前端不传 → False,行为安全退化。
synthesis_mode: bool = bool(payload.get("synthesis_mode") or False)
finalize_only: bool = bool(payload.get("finalize_only") or False)
# 并行禁写文件标记(仅 special 用):前端 runDagOrchestration 在真并行 stage(≥2 席位)
# 置真,网关据此用 seat_parallel policy 额外禁 write_file/bash/str_replace(§4)。
# 兼容 no_file 别名;缺省 False → 普通 seat,旧前端不传也不破坏。
@@ -2203,27 +2133,6 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
for key in agent_threads:
_validate_agent_id(key, "d_agent_thread_id key")
# Browser-side acquire admits the room exactly once when Step 2 begins.
# Every model-producing Step-2 entry only renews that existing room, so a
# pause, clarification answer or user intervention never competes for a
# second slot. Step 3 is excluded because consensus releases the room.
step2_room_limited = _is_step2_room_limited(
agent_type,
agent_name,
# 只有显式最终收口轮绕过房间时限;普通综合轮仍可继续派活。
finalize_only=finalize_only,
)
if step2_room_limited:
from app.gateway.roundtable_room_gate import enforce_roundtable_run
await enforce_roundtable_run(
request,
draft_id=str(payload.get("draft_id") or "").strip() or None,
task_id=str(payload.get("task_id") or "").strip() or None,
room_session_id=str(payload.get("room_session_id") or "").strip() or None,
thread_ids=list(agent_threads.values()),
)
base = _loopback_base(request)
headers = _auth_headers(request)
@@ -2251,7 +2160,6 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
client, base, headers, agent_threads, agent_name, new_message, skill_stop_names, model_name,
seat_meta=seat_meta,
synthesis_mode=synthesis_mode,
finalize_only=finalize_only,
suppress_pre_broadcast=suppress_pre_broadcast,
files=files,
)
@@ -2303,13 +2211,6 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
"agent_name": agent_name,
}
yield f"data: {json.dumps(err_frame, ensure_ascii=False)}\n\n"
except asyncio.CancelledError:
# Intentional pause and a real network disconnect are both an
# SSE cancellation at this layer. Releasing here made a normal
# pause lose its admitted room. Browser heartbeats now own
# liveness: pauses keep heartbeating; a disconnected browser
# stops heartbeating and the 60-second lease expires.
raise
return StreamingResponse(
stream_generator(),
@@ -6,10 +6,9 @@
``agent_id``,与前端任务深链 ``goPath=3qfx`` 取的是同一个);
2. 按 taskId 拉取任务详情(consumer 的 ``cop-task-detail``,地址 / 鉴权头 / mock 都在
``config.yaml`` 的 ``task_deeplink`` 段),拼出与前端深链一致的开场白
``任务名称:…\n任务内容:…\n任务类型:…\n任务id(task_id)为…``
(``taskType`` 字典码 1/2/3 映射为 打击/塑势/防御,无法识别则省略该行);
3. **新建一条真实会话**(thread 打任务标签并登记到 ``taskId + goPath`` 专用聊天记录),
以开场白作为第一条用户消息
``任务名称:…\n任务内容:…\n任务id(task_id)为…``;
3. **新建一条真实会话**(thread 打 ``metadata.{taskId, agent_id}`` 标签,与前端
``taskThreadMetadata`` / ``useTaskThreads`` 的过滤口径一致),以开场白作为第一条用户消息
**后台启动一次持久化运行**(写 checkpointer,不阻塞);
4. **立即返回** ``{thread_id, run_id, agent_id, status}`` —— 调用方拿到「调用成功」即可,
随后在 3qfx 任务工作区的对话列表里就能看到这条问答并实时观看其进展。
@@ -27,7 +26,7 @@
返回::
{"thread_id": "…", "run_id": "…", "agent_id": "…", "status": "running",
"opening_text": "任务名称:…\n任务内容:…\n任务类型:打击\n任务id(task_id)为2055"}
"opening_text": "任务名称:…\n任务内容:…\n任务id(task_id)为2055"}
"""
from __future__ import annotations
@@ -93,41 +92,6 @@ class CopTaskDetail(BaseModel):
task_direction: str = ""
task_name: str = ""
task_content: str = ""
task_type: str = ""
_TASK_TYPE_LABELS = {"1": "打击", "2": "塑势", "3": "防御"}
def map_task_type_label(value: Any) -> str:
"""把 cop-task-detail 的 taskType 字典码映射成发送文本。无法识别时返回空串。"""
if isinstance(value, bool) or value is None:
return ""
if isinstance(value, int):
key = str(value)
elif isinstance(value, float):
if not value.is_integer():
return ""
key = str(int(value))
elif isinstance(value, str):
key = value.strip()
else:
return ""
if key in _TASK_TYPE_LABELS.values():
return key
return _TASK_TYPE_LABELS.get(key, "")
def _task_type_code(value: Any) -> str:
if isinstance(value, bool) or value is None:
return ""
if isinstance(value, int):
return str(value)
if isinstance(value, float):
return str(int(value)) if value.is_integer() else ""
if isinstance(value, str):
return value.strip()
return ""
async def _resolve_threeq_agent_id(request: Request) -> str:
@@ -153,7 +117,6 @@ def _parse_cop_task_detail(body: Any) -> CopTaskDetail:
task_direction=data.get("taskDirection") if isinstance(data.get("taskDirection"), str) else "",
task_name=data.get("taskName") if isinstance(data.get("taskName"), str) else "",
task_content=data.get("taskContent") if isinstance(data.get("taskContent"), str) else "",
task_type=_task_type_code(data.get("taskType")),
)
@@ -183,11 +146,9 @@ async def _fetch_cop_task_detail(task_id: str) -> CopTaskDetail:
def build_opening_text(detail: CopTaskDetail, task_id: str) -> str:
"""拼 3qfx 开场白,与前端 ``buildOpeningText``(3qfx 分支)逐行对齐。"""
task_type_label = map_task_type_label(detail.task_type)
parts = [
f"任务名称:{detail.task_name}" if detail.task_name else "",
f"任务内容:{detail.task_content}" if detail.task_content else "",
f"任务类型:{task_type_label}" if task_type_label else "",
f"任务id(task_id)为{task_id}" if task_id else "",
]
return "\n".join(p for p in parts if p)
@@ -219,15 +180,9 @@ async def ask_3qfx(body: ThreeQAskRequest, request: Request) -> ThreeQAskRespons
run_body = RunCreateRequest(
assistant_id=agent_id,
input={"messages": [{"role": "user", "content": opening_text}]},
# taskRecordId + taskGoPath 写入任务深链专用聊天记录;taskId 继续触发
# 后端「任务深链对话全局共享 / 按创建者解析文件」语义。
metadata={
"taskId": task_id,
"taskRecordId": task_id,
"taskGoPath": "3qfx",
"thread_type": "task_deeplink",
"agent_id": agent_id,
},
# taskId + agent_id 让该会话被 3qfx 任务工作区的 useTaskThreads 过滤命中(按任务+业务隔离),
# 且 metadata.taskId 触发后端「任务深链对话全局共享 / 按创建者解析文件」语义。
metadata={"taskId": task_id, "agent_id": agent_id},
context=context,
multitask_strategy="reject",
)
@@ -51,7 +51,6 @@ from pydantic import BaseModel, Field
from deerflow.agents.lead_agent.agent import make_lead_agent
from deerflow.config.agents_config import list_custom_agents, validate_agent_id
from deerflow.runtime.runs.worker import build_agent_with_timing
logger = logging.getLogger(__name__)
@@ -82,7 +81,7 @@ class OpenChatRequest(BaseModel):
thinking_enabled: bool = Field(default=False, description="是否开启思考;默认 False = 关闭思考(内网模型也会被强制关闭)")
strip_think: bool = Field(default=True, description="关闭思考时,从答案里剥离残留的 <think>…</think> 块")
thread_id: str | None = Field(default=None, description="可选;仅用于文件隔离,留空自动生成")
recursion_limit: int = Field(default=100, ge=1, le=300, description="兼容旧客户端保留;实际步数上限统一使用管理员的问答步骤设置", deprecated=True)
recursion_limit: int = Field(default=100, ge=1, le=300, description="智能体单次问答的最大步数")
include_files: bool = Field(default=True, description="是否随结果回传智能体在 outputs 目录生成的文件(名称+内容)")
@@ -280,13 +279,7 @@ def _build_run_context(body: OpenChatRequest) -> tuple[dict[str, Any], str | Non
}
if agent_id:
configurable["agent_id"] = agent_id
from deerflow.config.system_settings import get_conversation_context_settings
# Match browser chat: external callers cannot override the server budget.
config: dict[str, Any] = {
"configurable": configurable,
"recursion_limit": get_conversation_context_settings().max_run_steps,
}
config: dict[str, Any] = {"configurable": configurable, "recursion_limit": body.recursion_limit}
return config, agent_id, thread_id
@@ -308,7 +301,7 @@ async def open_chat(body: OpenChatRequest) -> OpenChatResponse:
config, agent_id, thread_id = _build_run_context(body)
try:
agent = await build_agent_with_timing(make_lead_agent, config, None, thread_id)
agent = make_lead_agent(config)
result = await agent.ainvoke({"messages": lc_messages}, config=config)
except Exception as exc: # noqa: BLE001 — 开放接口需把内部错误归一成 502
logger.exception("open_chat 调用智能体失败 (model=%s, agent=%s)", body.model_name, agent_id)
@@ -352,7 +345,7 @@ async def _open_chat_event_stream(body: OpenChatRequest) -> AsyncIterator[str]:
async def produce() -> None:
nonlocal final_state
try:
agent = await build_agent_with_timing(make_lead_agent, config, None, thread_id)
agent = make_lead_agent(config)
async for mode, chunk in agent.astream({"messages": lc_messages}, config=config, stream_mode=["values", "messages"]):
if mode == "values" and isinstance(chunk, dict):
final_state = chunk
@@ -20,7 +20,7 @@ from datetime import datetime
from typing import Any
from uuid import uuid4
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from deerflow.runtime.user_context import get_effective_user_id
@@ -103,13 +103,10 @@ def _get_store(request: Request):
@router.get("", response_model=TaskListResponse)
async def list_tasks(
request: Request,
id_prefix: str | None = Query(default=None, max_length=64),
) -> TaskListResponse:
async def list_tasks(request: Request) -> TaskListResponse:
store = _get_store(request)
user_id = _current_user_id(request)
rows = await store.list_tasks(user_id, id_prefix=id_prefix)
rows = await store.list_tasks(user_id)
return TaskListResponse(tasks=[TaskResponse(**r) for r in rows])
@@ -9,8 +9,8 @@ Each fan-out agent runs as a pure **executing seat**: ``excluded_tools`` =
``_EXCLUDED_TOOLS`` (``write_file`` / ``str_replace`` / ``agent_orchestration`` /
``bash``) so a seat can neither write files (parallel writes would clash) nor
dispatch / run scripts (a weak model fed "总控 / 业务链条" framing would otherwise
mimic the controller and "派活"). Read-only tools stay on. ``ask_clarification``
is available only to strategy intent, the sole stage with a reply surface.
mimic the controller and "派活"). Read-only tools + ``ask_clarification`` stay on,
so the step bar and clarification cards still work.
Unified event protocol (one JSON object per ``data:`` frame)::
@@ -42,10 +42,7 @@ from pydantic import BaseModel, Field
from app.gateway.roundtable_model_fallback import is_llm_error_message, reason_text
from app.gateway.routers import agents as agents_router
from app.gateway.routers._strategy_selection_seed import (
STRATEGY_AGENT_ID_PREFIX,
STRATEGY_INTENT_AGENT_ID,
)
from app.gateway.routers._strategy_selection_seed import STRATEGY_AGENT_ID_PREFIX
from app.gateway.routers.multi_agent import (
_ROUNDTABLE_THREAD_METADATA,
_auth_headers,
@@ -131,7 +128,7 @@ def _has_substantive_content(content: str) -> bool:
# 链条」语气的任务后,会概率性地用 agent_orchestration 真去派活,或绕道 bash 跑脚本
# 调内部接口模仿总控调度(bug:单步单智能体链条里子智能体输出「总控调度指令…」并用
# glob / bash 翻找派活脚本)。物理禁掉这两类即可根治。其余只读工具(ls / read_file /
# 检索)保留;ask_clarification 仅意图阶段保留,因为只有它有可续答的协作卡片。
# 检索)与 ask_clarification(协作卡片)保留:既不互相打架,又能正常出步骤条 / 要澄清。
_EXCLUDED_TOOLS: list[str] = [
"write_file", # 并行不互相写文件打架
"str_replace",
@@ -140,26 +137,6 @@ _EXCLUDED_TOOLS: list[str] = [
]
def _strategy_stage_allows_clarification(agent_id: str) -> bool:
"""Return whether a strategy-selection stage has a user reply surface.
The workbench renders and resumes clarification cards only for the intent
stage. Angle generation, judging and final synthesis are one-shot stages;
letting those agents call ``ask_clarification`` produces an interrupt that
the UI cannot answer and therefore turns a recoverable run into a failure.
"""
return agent_id.strip().casefold() == STRATEGY_INTENT_AGENT_ID.casefold()
def _excluded_tools_for_agent(agent_id: str) -> list[str]:
excluded = list(_EXCLUDED_TOOLS)
normalized = agent_id.strip().casefold()
if normalized.startswith(STRATEGY_AGENT_ID_PREFIX.casefold()) and not _strategy_stage_allows_clarification(agent_id):
excluded.append("ask_clarification")
return excluded
# ── 并行面板兼容登录:复用单点开关、密码、限速与封禁校验 ────────────────────
@@ -173,44 +150,12 @@ class _TokenResponse(BaseModel):
user_id: str
def _force_situation_detail_compat_token(username: str) -> _TokenResponse:
"""Return a non-JWT value accepted only by open force-situation detail APIs.
Existing task-detail skills call this legacy endpoint before every detail
GET/PUT and require a non-empty value even though those endpoints are now
credential-free. Keeping the compatibility behavior generic avoids
coupling backend routing to a particular skill name or service account.
"""
normalized = username.casefold()
return _TokenResponse(
# Deliberately not a signed JWT: AuthMiddleware rejects it everywhere
# except the explicitly credential-free force-situation detail routes.
access_token=f"force-situation-detail-compat-{normalized}",
token_type="bearer",
user_id=f"external:{normalized}",
)
@router.post("/auth/token", response_model=_TokenResponse)
async def issue_token(body: _TokenRequest, request: Request, response: Response, password: str | None = Query(default=None)) -> _TokenResponse:
"""Issue a normal login token, or detail-route compatibility value on policy rejection."""
username = body.username.strip()
from app.gateway.routers.auth import login_by_username
"""Keep the panel response contract while enforcing standard login policy."""
from app.gateway.routers.auth import UsernameLoginRequest, login_by_username
try:
result = await login_by_username(
request,
response,
password=password,
)
except HTTPException as exc:
# The caller needs this endpoint only as a preflight before the open
# force-situation detail routes. Do not make individual skill names or
# current password/SSO policy a prerequisite for that workflow.
if exc.status_code in {401, 403}:
return _force_situation_detail_compat_token(username)
raise
result = await login_by_username(request, response, UsernameLoginRequest(username=body.username.strip()), password=password)
return _TokenResponse(access_token=result.access_token, token_type="bearer", user_id=result.user_id)
@@ -423,7 +368,7 @@ async def _run_one_agent(
model_name=model_name,
thread_id=thread_id,
new_message=new_message,
excluded_tools=_excluded_tools_for_agent(agent_id),
excluded_tools=list(_EXCLUDED_TOOLS),
skill_stop_names=[],
thinking_enabled=True,
reasoning_effort="medium",
@@ -480,7 +425,7 @@ async def _run_one_agent(
}
def _build_task_message(task_name: str, brief: str, *, agent_id: str = "") -> str:
def _build_task_message(task_name: str, brief: str) -> str:
parts: list[str] = []
if task_name.strip():
parts.append(f"任务名称:{task_name.strip()}")
@@ -495,23 +440,13 @@ def _build_task_message(task_name: str, brief: str, *, agent_id: str = "") -> st
"调度对象 / 分配给『…』的任务 / agent_name」之类调度内容,严禁罗列或转派其他席位的任务,严禁调用"
"任何派活 / 调度类工具,也不要用脚本(bash / python)去调用内部接口或翻找派活脚本。"
)
# 只有意图阶段有可交互、可恢复的澄清卡片。其余策略阶段均为一次性交付,必须基于已确认
# 的任务继续完成;否则 ask_clarification 会产生前端无法回答的中断并被误报为生成失败。
if agent_id and agent_id.casefold().startswith(STRATEGY_AGENT_ID_PREFIX.casefold()) and not _strategy_stage_allows_clarification(agent_id):
parts.append(
"【本阶段禁止反问】当前是无需用户中途介入的一次性交付阶段。不得调用 `ask_clarification`,"
"不得在正文里向用户提问、要求补充材料或要求点击重试。信息不足时请采用保守的合理假设,"
"明确降低相应结论的置信度,并把缺口列入“数据限制与待核验”,然后直接完成本阶段要求的全部交付;"
"不要重复输出同一正文或多个版本。"
)
else:
# 澄清护栏:要向用户确认时,必须用 ask_clarification 工具发起(前端据此渲染可点击的
# 协作卡片),不能只把问题写在正文里。能合理假设就先假设并标注。
parts.append(
"【需要向用户确认时】如果确实缺少完成分析所必需的关键信息,请**调用 `ask_clarification` 工具**"
"发起确认(这样用户会看到一张可点击的协作卡片来回答);**不要只把问题写在正文里**,否则用户无法回应。"
"能基于已有信息合理假设的,就先假设并在正文中标注你的假设;只有当不同假设会显著改变结论时,才发起确认。"
)
# 澄清护栏:要向用户确认时,必须用 ask_clarification 工具发起(前端据此渲染可点击的协作
# 卡片),不能只把问题写在正文里 —— 否则用户看不到卡片、无法回应。能合理假设就先假设并标注。
parts.append(
"【需要向用户确认时】如果确实缺少完成分析所必需的关键信息,请**调用 `ask_clarification` 工具**"
"发起确认(这样用户会看到一张可点击的协作卡片来回答);**不要只把问题写在正文里**,否则用户无法回应。"
"能基于已有信息合理假设的,就先假设并在正文中标注你的假设;只有当不同假设会显著改变结论时,才发起确认。"
)
return "\n\n".join(parts)
@@ -579,13 +514,13 @@ async def run_parallel_agents_stream(request: Request, payload: dict = Body(...)
base = _loopback_base(request)
headers = _auth_headers(request)
new_message = _build_task_message(task_name, brief)
async def _consume_agent(
client: Any,
agent_id: str,
queue: asyncio.Queue[dict[str, Any]],
) -> None:
new_message = _build_task_message(task_name, brief, agent_id=agent_id)
await queue.put({"type": "agent_start", "agent_id": agent_id})
last_event = ""
emitted_tools: set[str] = set() # 已发过 step 的 tool_call id(去重)
@@ -197,7 +197,7 @@ class RejectNodeRequest(BaseModel):
class ActionPlanDictionaryOption(BaseModel):
"""One task-system dictionary row for fields still backed by an enum."""
"""One task-system dictionary row, preserving dictValue as the model input."""
label: str = Field(default="", max_length=256)
value: str | int | float = ""
@@ -207,15 +207,10 @@ class ActionPlanDictionaryOption(BaseModel):
class ActionPlanDictionaries(BaseModel):
"""Task-system dictionaries for fields other than the model-derived action type.
``actionType`` is deliberately absent: the action planner must infer a
concise type name from the roundtable material instead of choosing a
frontend-provided dictionary value. Pydantic's default extra-field handling
keeps legacy clients that still submit ``action_types`` compatible.
"""
"""The four action fields that must be selected from task-system dictionaries."""
publish_channels: list[ActionPlanDictionaryOption] = Field(default_factory=list)
action_types: list[ActionPlanDictionaryOption] = Field(default_factory=list)
info_formats: list[ActionPlanDictionaryOption] = Field(default_factory=list)
responsible_positions: list[ActionPlanDictionaryOption] = Field(default_factory=list)
@@ -1243,12 +1238,13 @@ async def _build_builtin_prompt(
summary_content = _truncate_text(summary.get("content"), 12_000)
dictionary_groups = (
("发布渠道(publishChannel)", action_dictionaries.publish_channels if action_dictionaries else []),
("行动类型(actionType)", action_dictionaries.action_types if action_dictionaries else []),
("信息形式(infoFormat)", action_dictionaries.info_formats if action_dictionaries else []),
("责任岗位(responsiblePosition)", action_dictionaries.responsible_positions if action_dictionaries else []),
)
dictionaries_ready = bool(action_task_id is not None and all(options for _, options in dictionary_groups))
dictionary_lines = [
"【前端传入的行动字段字典(名称 → 必须原样使用的 dictValue)】",
"【前端传入的行动字典(名称 → 必须原样使用的 dictValue)】",
f"- 当前 taskId:{action_task_id if action_task_id is not None else '(本地预览)'}",
]
for label, options in dictionary_groups:
@@ -1258,15 +1254,12 @@ async def _build_builtin_prompt(
for option in options
)
dictionary_instruction = (
"actionType 必须由你根据方案总结和岗位交付自行分析归纳,填写简洁、明确的中文类型名称;"
"不要从前端字典选择 actionType,也不要填写 dictValue 或数字编码。"
"publishChannel、infoFormat、responsiblePosition 必须从下方同名字典复制 dictValue 并写成整数;"
"actionType、publishChannel、infoFormat、responsiblePosition 必须从下方同名字典复制 dictValue 并写成整数;"
"taskId 必须原样写为当前 taskId。不要保留 id、title、goal、basis、owner_position、dependencies 等旧字段,"
"也不要用 0 或空值代替已提供的字典项。"
"也不要用 0 或空值代替未选择的字典项。"
if dictionaries_ready
else "当前三类任务系统字典未获取完整,这是本地预览/接口未就绪状态。仍需生成 9 字段 JSON:"
"actionType 仍须由你分析并填写中文类型名称;publishChannel、infoFormat、responsiblePosition 可暂填 0 占位,"
"且不得调用 position-roundtable-action-create Skill 或外部任务系统。"
else "当前四类任务系统字典未获取完整,这是本地预览/接口未就绪状态。仍需生成 9 字段 JSON,"
"但四个枚举字段可暂填 0 占位,且不得调用 position-roundtable-action-create Skill 或外部任务系统。"
)
submit_instruction = (
"若当前 taskId 是正整数,首次生成完整行动清单后按行动规划智能体的 SOUL 调用 position-roundtable-action-create Skill 批量新增行动;"
@@ -1275,7 +1268,7 @@ async def _build_builtin_prompt(
else "本轮字典不完整,只生成报告和 JSON 预览;不得调用 Skill、不得请求外部任务系统,也不能宣称行动已入库。"
)
instructions = [
"请基于以下岗位会商材料,以及已生效的方案总结(如有),生成或按用户要求修订《岗位会商行动规划》。",
"请基于以下方案总结和岗位会商材料,生成或按用户要求修订《岗位会商行动规划》。",
"必须使用 write_file 分别写入两个最终交付:`/mnt/user-data/outputs/行动规划报告.md` 与 `/mnt/user-data/outputs/action-plan-subtasks.json`。",
"Markdown 报告需给出分阶段推进建议、优先级、风险控制与验收安排。"
"JSON 必须是严格合法数组;每个对象只能包含 actionName、actionDescription、actionType、"
@@ -1283,8 +1276,8 @@ async def _build_builtin_prompt(
dictionary_instruction,
submit_instruction,
"",
"【已生效方案总结(如有)】",
summary_content or "(本轮未生成方案总结;请直接依据下方有效岗位交付、任务信息和已确认意图进行归纳。)",
"【已生效方案总结】",
summary_content or "(方案总结正文未写入快照,请依据下方有效岗位交付重新归纳。)",
"",
*dictionary_lines,
]
@@ -1683,6 +1676,9 @@ async def _prepare_builtin_run(
raise HTTPException(status_code=409, detail="所有岗位产物完成后才能生成或更新内置交付")
if kind == "action_plan":
summary = session.get("summary_snapshot")
if not isinstance(summary, dict) or summary.get("status") != "done":
raise HTTPException(status_code=409, detail="请先生成有效的方案总结")
thread_id = session.get("action_plan_thread_id")
agent_id = ACTION_PLAN_AGENT_ID
display_name = "行动规划智能体"
@@ -142,7 +142,14 @@ async def get_my_position(request: Request, store: PositionStore = Depends(get_p
user = await get_optional_user_from_request(request)
if user is None:
return MyPositionResponse(position=None)
record = await store.get_effective_position(str(user.id))
record = None
pid = await store.get_user_position_id(str(user.id))
if pid:
record = await store.get_position(pid)
if record is None:
default_id = await store.get_default_position_id()
if default_id:
record = await store.get_position(default_id)
return MyPositionResponse(position=PositionResponse(**record) if record else None)
@@ -114,8 +114,7 @@ async def public_knowledge_vector_search(
# second read ensures an unpublished or newly reserved result is never
# emitted by this anonymous endpoint.
still_published = {str(row["id"]) for row in await _published_mappings(request)}
wiki_index = request.app.state.config.llmwiki.local_wiki_index
frontend_base_url = wiki_index.resolve_frontend_base_url(request.headers.get("origin"))
frontend_base_url = request.app.state.config.llmwiki.local_wiki_index.frontend_base_url
results: list[dict[str, Any]] = []
for raw_item in payload.get("results") or []:
if str(raw_item.get("knowledge_base_id") or "") not in still_published:
@@ -16,10 +16,10 @@ import logging
from fastapi import APIRouter, HTTPException, Query
from app.gateway.admin_query_window import ADMIN_QUERY_WINDOW_DAYS, resolve_admin_query_window
from app.gateway.routers.admin_users import (
AdminLeaderboardResponse,
_load_or_schedule_daily_leaderboard,
_resolve_range,
)
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.engine import get_session_factory
@@ -68,7 +68,7 @@ def _anonymize(resp: AdminLeaderboardResponse) -> AdminLeaderboardResponse:
@router.get("", response_model=AdminLeaderboardResponse)
@router.get("/", response_model=AdminLeaderboardResponse)
async def get_public_leaderboard(
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
days: int = Query(default=30, ge=1, le=365),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
limit: int = Query(default=20, ge=5, le=100),
@@ -79,11 +79,7 @@ async def get_public_leaderboard(
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
since_dt, until_dt, _range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
since_dt, until_dt, _range_days = _resolve_range(days, since, until)
resp = await _load_or_schedule_daily_leaderboard(
session_factory,
settings,
@@ -9,7 +9,6 @@ from uuid import uuid4
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.api_read_cache import get_cache_version, get_cached_json, set_cached_json
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
@@ -66,16 +65,6 @@ async def list_recommended_questions(
all: bool = Query(default=False, description="Admin management view: return every question, unfiltered by 岗位."),
tag_ids: list[str] | None = Query(default=None, description="Keep questions carrying any of these tags (OR)."),
) -> RecommendedQuestionsListResponse:
user_id = _current_user_id(request)
config = getattr(request.app.state, "config", None)
cache_key = f"user:{user_id}"
normalized_tag_ids = tag_ids if isinstance(tag_ids, list) else None
cacheable = not all and not normalized_tag_ids
version = await get_cache_version(config, "recommended_questions") if cacheable else None
if cacheable:
cached = await get_cached_json(config, "recommended_questions", cache_key, version=version)
if cached is not None:
return RecommendedQuestionsListResponse.model_validate(cached)
store = _get_store(request)
rows = await store.list_all()
responses = [RecommendedQuestionResponse(**r) for r in rows]
@@ -102,7 +91,7 @@ async def list_recommended_questions(
sync = getattr(request.app.state, "position_sync", None)
if sync is not None:
try:
visible = await sync.resolve_visible(user_id, "recommended_question")
visible = await sync.resolve_visible(_current_user_id(request), "recommended_question")
except Exception:
logger.debug("recommended-question visibility resolve failed", exc_info=True)
visible = None
@@ -111,20 +100,11 @@ async def list_recommended_questions(
else:
responses = [r for r in responses if not r.tags]
if normalized_tag_ids:
wanted = set(normalized_tag_ids)
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
result = RecommendedQuestionsListResponse(questions=responses)
if cacheable:
await set_cached_json(
config,
"recommended_questions",
cache_key,
result,
expected_version=version,
)
return result
return RecommendedQuestionsListResponse(questions=responses)
@router.post("/recommended-questions", response_model=RecommendedQuestionResponse, status_code=201)
@@ -41,25 +41,6 @@ router = APIRouter(prefix="/api/roundtable-chains", tags=["roundtable-chains"])
MIN_SEATS = 2
MAX_SEATS = 20
# 功能型内置智能体不可作业务链席位(Step 1 意图理解 / 推荐器 / 总控 / 报告)。
# 与前端 `api/recommend.ts` 的 `filterRecommendCandidates` 排除集逐字对齐:
# 席位富化(_enrich_chain_seats)把这些 id 一律标为 available=False,保证
# 「前端不再实时查 /api/agents」后「功能型智能体不可入席」的语义不变。
_FUNCTIONAL_AGENT_IDS = frozenset(
{
"roundtable-intent",
"roundtable-recommender",
"roundtable-coordinator",
"roundtable-report",
}
)
_FUNCTIONAL_AGENT_PREFIX = "roundtable-coordinator-"
def _is_functional_agent(agent_id: str) -> bool:
lowered = agent_id.lower()
return lowered in _FUNCTIONAL_AGENT_IDS or lowered.startswith(_FUNCTIONAL_AGENT_PREFIX)
# ── schemas ────────────────────────────────────────────────────────────────
@@ -99,30 +80,11 @@ class ChainSeatSchema(BaseModel):
job_responsibilities: str | None = Field(default=None, max_length=4000)
class ChainSeatResponseSchema(ChainSeatSchema):
"""席位响应模型:在落库字段之外附带两个**读时计算**的富化字段。
前端选链(弹窗 / 左栏点选 / 深链自动选链)从此不再实时拉 `/api/agents`,
所需的「席位智能体是否仍可用」与「最新名称」由后端在返回链条时用一次批量
查询(`agent_store.list_by_ids`,与 `/api/agents` 同一套可见性过滤)算好:
- ``available``:该席位智能体当前对请求用户可见(本人 / 内置 / 已发布、
非私享、非管理员隐藏)且不是功能型内置 → True;否则 False。
- ``live_name``:智能体当前名称;查不到时为 None(前端回退 seat.name 快照)。
仅用于响应序列化,**绝不持久化**:create/update 请求沿用 ``ChainSeatSchema``,
富化字段不会进 seats JSON(存量数据零迁移)。
"""
available: bool | None = None
live_name: str | None = None
class ChainResponse(BaseModel):
id: str
title: str = ""
description: str | None = None
seats: list[ChainSeatResponseSchema] = []
seats: list[ChainSeatSchema] = []
# Optional layered orchestration: list of stages, each a list of agent_ids
# run in parallel (stages run serially). None/absent = linear chain.
stages: list[list[str]] | None = None
@@ -141,10 +103,6 @@ class ChainResponse(BaseModel):
# 「待审核」标记:True = owner 已提交审核(等待管理员在「待审核」筛选里发布);
# 管理员发布时随 is_public=True 一起清回 False。缺省 False。
pending_review: bool = False
# 管理员目录开关:True = 从所有普通列表(我的/公共/待审核/选链弹窗)隐藏,仅管理员
# 管理视图(admin_view=true)可见。纯目录开关:get_chain 单条读取不受影响,已配置的
# 业务映射 / 进行中的研讨继续可用。缺省 False。
hidden: bool = False
# 是否当前用户所有(仅本人可编辑 / 删除 / 发布;公共筛选里别人的链条 is_mine=False)。
is_mine: bool = True
# 配置者(owner)的 user_id 及解析后的可读用户名(email)。owner_name 由路由层
@@ -241,60 +199,6 @@ async def _enrich_owner_names(rows: list[dict]) -> None:
row["owner_name"] = cache[uid]
def _apply_seat_enrichment(chains: list[dict], live_records: list[dict]) -> None:
"""把批量查到的实时智能体信息写进每条链每个 seat(就地修改响应 dict)。
纯函数(不触网),便于单测:``live_records`` 是 ``agent_store.list_by_ids``
的返回(dict 列表,含 id/name)。规则:
- 功能型内置智能体(_is_functional_agent)与查不到的 id → available=False、
live_name=None(对齐前端 ``filterRecommendCandidates`` 的排除语义);
- 其余 → available=True、live_name=实时 name(回退 None 由前端用快照名)。
"""
live: dict[str, dict] = {}
for record in live_records:
rid = str(record.get("id") or "").lower()
if rid:
live[rid] = record
for chain in chains:
for seat in chain.get("seats") or []:
aid = str(seat.get("agent_id") or "").lower()
record = live.get(aid)
if record is None or _is_functional_agent(aid):
seat["available"] = False
seat["live_name"] = None
else:
seat["available"] = True
seat["live_name"] = str(record.get("name") or "") or None
async def _enrich_chain_seats(request: Request, rows: list[dict]) -> None:
"""席位富化入口:一次批量 SQL 算好所有 seat 的 available / live_name。
best-effort:agent store 不可用 / 查询异常时不写任何字段(响应缺省 → 前端
按存活处理),与旧前端「agents 列表拉取失败不判删」的降级语义一致。
仅修改响应 dict,落库的 seats JSON 不受影响。
"""
agent_ids = {
str(seat.get("agent_id") or "").lower()
for row in rows
for seat in row.get("seats") or []
if seat.get("agent_id")
}
if not agent_ids:
return
try:
store = get_agent_store(request)
except Exception:
logger.warning("Seat enrichment skipped: agent store unavailable")
return
try:
records = await store.list_by_ids(sorted(agent_ids), _current_user_id(request))
except Exception:
logger.warning("Seat enrichment skipped: batch agent lookup failed", exc_info=True)
return
_apply_seat_enrichment(rows, records)
def _get_store(request: Request):
store = getattr(request.app.state, "roundtable_chain_store", None)
if store is None:
@@ -460,12 +364,7 @@ def _validate_seat_mode(seat_mode: str | None) -> None:
@router.get("", response_model=ChainListResponse)
async def list_chains(
request: Request,
scope: str = "mine",
admin_view: bool = False,
hidden: bool | None = None,
) -> ChainListResponse:
async def list_chains(request: Request, scope: str = "mine") -> ChainListResponse:
"""列出业务链条。
``scope`` 查询参数(筛选):
@@ -475,11 +374,6 @@ async def list_chains(
非管理员请求 ``pending`` 退化为 ``mine``);
- ``all``:所有账号的全部链条(**仅管理员**;非管理员请求 ``all`` 退化为 ``mine``)。
其余值按 ``mine`` 处理。
隐藏开关(管理员目录):默认(含管理员的普通视图,如选链弹窗)隐藏链条
(``hidden=True``)从所有 scope 排除。管理员传 ``admin_view=true`` 切到管理视图:
不排除隐藏链条,并可用 ``hidden=true/false`` 筛选只看隐藏 / 只看显示中(配置页专用)。
非管理员传 ``admin_view`` 一律忽略,绝不越权。
"""
is_admin = _is_admin(request)
normalized_scope = scope if scope in ("mine", "public", "pending", "all") else "mine"
@@ -488,15 +382,8 @@ async def list_chains(
normalized_scope = "mine"
store = _get_store(request)
user_id = _current_user_id(request)
rows = await store.list_chains(
user_id,
scope=normalized_scope,
is_admin=is_admin,
include_hidden=admin_view and is_admin,
hidden=hidden if admin_view and is_admin else None,
)
rows = await store.list_chains(user_id, scope=normalized_scope, is_admin=is_admin)
await _enrich_owner_names(rows)
await _enrich_chain_seats(request, rows)
return ChainListResponse(chains=[ChainResponse(**r) for r in rows])
@@ -524,7 +411,6 @@ async def create_chain(request: Request, body: ChainCreateRequest) -> ChainRespo
},
)
await _enrich_owner_names([row])
await _enrich_chain_seats(request, [row])
return ChainResponse(**row)
@@ -536,7 +422,6 @@ async def get_chain(request: Request, chain_id: str) -> ChainResponse:
if row is None:
raise HTTPException(status_code=404, detail="Chain not found")
await _enrich_owner_names([row])
await _enrich_chain_seats(request, [row])
return ChainResponse(**row)
@@ -566,7 +451,6 @@ async def update_chain(request: Request, chain_id: str, body: ChainUpdateRequest
if row.get("is_public"):
await _publish_seat_agents(request, row, user_id, is_admin=is_admin)
await _enrich_owner_names([row])
await _enrich_chain_seats(request, [row])
return ChainResponse(**row)
@@ -579,35 +463,6 @@ async def delete_chain(request: Request, chain_id: str) -> None:
raise HTTPException(status_code=404, detail="Chain not found")
class ChainHiddenUpdateRequest(BaseModel):
"""Admin-only request body for hiding a chain from every non-admin listing."""
hidden: bool = Field(description="Set true to hide the chain from all non-admin listings (我的/公共/待审核/选链弹窗); false restores it. Purely a catalog switch — get_chain / 业务映射 / 进行中的研讨不受影响。")
@router.put("/{chain_id}/hidden", response_model=ChainResponse)
async def set_chain_hidden(
request: Request, chain_id: str, body: ChainHiddenUpdateRequest
) -> ChainResponse:
"""隐藏 / 取消隐藏一条业务链条(**仅管理员**)。
隐藏后链条从所有普通列表消失(连 owner 的「我的链条」也不显示),但单条读取不过滤:
已配置的业务映射、进行中的研讨、编辑器直接打开(chainId 深链)都继续可用。管理员
管理视图(``admin_view=true``)始终可见,可随时取消隐藏。
"""
if not _is_admin(request):
raise HTTPException(status_code=403, detail="Only system admins can hide or restore chains")
store = _get_store(request)
user_id = _current_user_id(request)
row = await store.update_chain(chain_id, user_id, hidden=body.hidden, is_admin=True)
if row is None:
raise HTTPException(status_code=404, detail="Chain not found")
logger.info("Set chain '%s' hidden=%s by admin", chain_id, body.hidden)
await _enrich_owner_names([row])
await _enrich_chain_seats(request, [row])
return ChainResponse(**row)
# ── 提示词优化(总控提示词 / 阶段目标)──────────────────────────────────────
# 链条编辑器的「AI 优化」按钮:把用户当前写的提示词 + 链条结构发来,单次直连
# LLM 重写后返回全文(同 suggestions 路由的直调模式,不走 agent / loopback)。
@@ -63,9 +63,6 @@ class StartJobRequest(BaseModel):
# 外部任务 id(taskId 深链会商)。非空 = task 作业:按 task_id 共享、不按 user 分权,
# 其 draft_id 指向独立的 roundtable_task_drafts 表(最终报告写回那里)。
taskId: str | None = Field(default=None, max_length=128)
# 当前页面这一次实时会商的隐藏标识。后台接管前用于续占同一个前台名额;
# job 创建后由持久 job 行继续占位。
roomSessionId: str | None = Field(default=None, max_length=128)
intent: str = ""
agents: list[JobAgent] = Field(default_factory=list)
model: str | None = None
@@ -239,24 +236,6 @@ async def start_job(
task_id = (body.taskId or "").strip() or None
draft_id = (body.draftId or "").strip() or None
# Handoff must own the already-admitted foreground room. It only renews
# that lease and never creates capacity during the transition.
from app.gateway.roundtable_room_gate import enforce_existing_roundtable_room
await enforce_existing_roundtable_room(
request,
draft_id=draft_id,
task_id=task_id,
room_session_id=(body.roomSessionId or "").strip() or None,
)
room_store = getattr(request.app.state, "roundtable_room_store", None)
room_started_at = None
read_started_at = getattr(room_store, "started_at", None)
if read_started_at is not None:
room_started_at = await read_started_at(
room_session_id=(body.roomSessionId or "").strip() or None,
holder_user_id=user_id,
)
agents = [SeatRef(a.agentId, a.name or a.agentId) for a in body.agents]
seed_message = (body.seedMessage or body.intent or "").strip()
@@ -301,20 +280,7 @@ async def start_job(
request_id=request_id,
request_hash=_request_hash(body),
input_snapshot=snapshot,
room_started_at=room_started_at,
)
# The durable job row now owns occupancy. Drop the foreground lease so a
# completed background job does not leave a one-minute phantom room.
if room_store is not None:
try:
await room_store.release(
draft_id=draft_id,
user_id=user_id,
task_id=task_id,
room_session_id=(body.roomSessionId or "").strip() or None,
)
except Exception:
logger.warning("failed to release foreground room after job handoff", exc_info=True)
if created:
# 新建(queued):交给 dispatcher 领取执行。
response.status_code = 201
@@ -1,123 +0,0 @@
"""Cross-user roundtable room admission.
Routes (prefix ``/api/roundtable-rooms``)::
GET /summary occupied room count across every account
POST /acquire atomically reserve a room, or 409 when full
POST /heartbeat renew the same durable lease
POST /release release the foreground lease
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.roundtable_room_gate import (
admit_roundtable_room,
backend_occupancy_keys,
release_roundtable_room,
renew_roundtable_room,
)
from deerflow.config.system_settings import get_roundtable_room_limit_settings
router = APIRouter(prefix="/api/roundtable-rooms", tags=["roundtable-rooms"])
class RoomScopeRequest(BaseModel):
draftId: str | None = Field(default=None, max_length=64)
taskId: str | None = Field(default=None, max_length=128)
roomSessionId: str | None = Field(default=None, max_length=128)
class RoomAdmissionResponse(BaseModel):
ok: bool = True
enabled: bool = True
max_rooms: int = 3
active_count: int = 0
renewed: bool = False
code: str = ""
message: str = ""
class RoomSummaryResponse(BaseModel):
enabled: bool = True
max_rooms: int = 3
active_count: int = 0
def _response(decision: dict) -> RoomAdmissionResponse:
return RoomAdmissionResponse(
ok=bool(decision.get("ok")),
enabled=bool(decision.get("enabled")),
max_rooms=int(decision.get("max_rooms") or 0),
active_count=int(decision.get("active_count") or 0),
renewed=bool(decision.get("renewed")),
code=str(decision.get("code") or ""),
message=str(decision.get("message") or ""),
)
def _require_store(request: Request):
store = getattr(request.app.state, "roundtable_room_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Roundtable room store not available")
return store
@router.get("/summary", response_model=RoomSummaryResponse)
async def room_summary(request: Request) -> RoomSummaryResponse:
settings = get_roundtable_room_limit_settings()
store = getattr(request.app.state, "roundtable_room_store", None)
extra = await backend_occupancy_keys(request)
active = (
0
if store is None
else await store.summary(
extra_keys=extra,
max_hold_minutes=getattr(settings, "max_hold_minutes", 30),
)
)
return RoomSummaryResponse(
enabled=settings.enabled,
max_rooms=settings.max_rooms,
active_count=active,
)
@router.post("/acquire", response_model=RoomAdmissionResponse)
async def acquire_room(request: Request, body: RoomScopeRequest) -> RoomAdmissionResponse:
decision = await admit_roundtable_room(
request,
draft_id=body.draftId,
task_id=body.taskId,
room_session_id=body.roomSessionId,
)
if not decision.get("ok", False):
raise HTTPException(status_code=409, detail=decision)
return _response(decision)
@router.post("/heartbeat", response_model=RoomAdmissionResponse)
async def heartbeat_room(request: Request, body: RoomScopeRequest) -> RoomAdmissionResponse:
decision = await renew_roundtable_room(
request,
draft_id=body.draftId,
task_id=body.taskId,
room_session_id=body.roomSessionId,
)
if not decision.get("ok", False):
raise HTTPException(status_code=409, detail=decision)
return _response(decision)
@router.post("/release")
async def release_room(request: Request, body: RoomScopeRequest) -> dict[str, bool]:
_require_store(request)
released = await release_roundtable_room(
request,
draft_id=body.draftId,
task_id=body.taskId,
room_session_id=body.roomSessionId,
)
return {"released": released}
@@ -2,11 +2,10 @@
from __future__ import annotations
import asyncio
import mimetypes
import re
import zipfile
from io import BytesIO
import mimetypes
from pathlib import Path
from typing import Any
from urllib.parse import quote
@@ -211,19 +210,15 @@ async def get_scheduler_thread(request: Request) -> dict:
@router.get("/html-templates")
async def list_html_templates(request: Request) -> dict:
async def list_html_templates() -> dict:
"""可选的 HTML 模板清单(供「模板数据转换器」内置 agent 填充用)。
返回 ``{"templates": [{id, name, description}]}``,前端定时任务表单在选中
「模板数据转换器」后据此渲染「HTML 模板」下拉。开放给任意登录用户。
"""
from app.gateway.catalog_cache import cached_catalog
from deerflow.runtime.scheduler import template_fill
async def load() -> dict:
return {"templates": await asyncio.to_thread(template_fill.list_templates)}
return await cached_catalog(request, "html_templates", "scheduler-list", load)
return {"templates": template_fill.list_templates()}
class TemplateRenderRequest(BaseModel):
@@ -20,15 +20,7 @@ from datetime import datetime
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from starlette.responses import Response
from app.gateway.api_read_cache import (
get_cache_version,
get_cached_json_payload,
release_cache_fill,
set_cached_json,
wait_for_cached_json_payload_or_claim_fill,
)
from app.gateway.deps import get_current_user, get_optional_user_from_request
from deerflow.persistence.sensitive_words import SensitiveWordStore
@@ -97,33 +89,9 @@ async def _require_manager(request: Request) -> None:
@router.get("", response_model=SensitiveWordListResponse)
async def list_sensitive_words(request: Request) -> SensitiveWordListResponse:
config = getattr(request.app.state, "config", None)
cache_key = "global"
version = await get_cache_version(config, "sensitive_words")
cached = await get_cached_json_payload(config, "sensitive_words", cache_key, version=version)
fill_token = None
if cached is None:
cached, fill_token = await wait_for_cached_json_payload_or_claim_fill(
config, "sensitive_words", cache_key, version=version, wait_ms=1000
)
if cached is not None:
return Response(content=cached, media_type="application/json") # type: ignore[return-value]
try:
store = _get_store(request)
rows = await store.list_words()
result = SensitiveWordListResponse(words=[SensitiveWordResponse(**r) for r in rows])
await set_cached_json(
config,
"sensitive_words",
cache_key,
result.model_dump(mode="json"),
expected_version=version,
ttl_seconds=3600,
)
return result
finally:
if fill_token is not None:
await release_cache_fill(config, "sensitive_words", cache_key, fill_token)
store = _get_store(request)
rows = await store.list_words()
return SensitiveWordListResponse(words=[SensitiveWordResponse(**r) for r in rows])
@router.put("", response_model=SensitiveWordListResponse)
@@ -1,61 +0,0 @@
"""Exchange a public service request for a short-lived scoped user token."""
from __future__ import annotations
import hmac
import logging
from datetime import timedelta
from fastapi import APIRouter, HTTPException, Request, status
from pydantic import BaseModel, Field
from app.gateway.auth import create_access_token
from app.gateway.deps import get_local_provider
from app.gateway.routers.auth import enforce_user_approved
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/auth/service", tags=["auth"])
_TOKEN_TTL_SECONDS = 300
_ALLOWED_SCOPES = {"llmwiki:sync", "recommended-questions:read"}
_SERVICE_TOKEN_SECRET = "b4217f191838d12a"
class ServiceExchangeRequest(BaseModel):
username: str = Field(min_length=1, max_length=128)
scope: list[str] = Field(default_factory=lambda: ["llmwiki:sync"], min_length=1, max_length=1)
class ServiceExchangeResponse(BaseModel):
access_token: str
token_type: str = "bearer"
expires_in: int = _TOKEN_TTL_SECONDS
scope: list[str]
@router.post("/exchange", response_model=ServiceExchangeResponse)
async def exchange_service_token(payload: ServiceExchangeRequest, request: Request) -> ServiceExchangeResponse:
supplied = request.headers.get("X-Service-Token", "")
if not supplied or not hmac.compare_digest(supplied, _SERVICE_TOKEN_SECRET):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid service token")
if len(payload.scope) != 1 or payload.scope[0] not in _ALLOWED_SCOPES:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Unsupported service scope")
provider = get_local_provider()
user = await provider.find_user_by_email_local_part(payload.username.strip().lower())
if user is None:
raise HTTPException(status_code=404, detail="Target user not found")
enforce_user_approved(user)
token = create_access_token(
str(user.id),
expires_delta=timedelta(seconds=_TOKEN_TTL_SECONDS),
token_version=user.token_version,
auth_method="service",
scope=payload.scope,
)
logger.info(
"Issued scoped service token target_user_id=%s scope=%s",
user.id,
payload.scope,
)
return ServiceExchangeResponse(access_token=token, scope=payload.scope)
@@ -8,34 +8,18 @@ import shutil
import subprocess
import sys
import tempfile
import time
import zipfile
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, Response, UploadFile
from fastapi import APIRouter, Body, Depends, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from sqlalchemy import select
from app.gateway.api_read_cache import (
get_cache_version,
get_cached_json_payload,
release_cache_fill,
set_cached_json,
wait_for_cached_json_payload_or_claim_fill,
)
from app.gateway.deps import get_agent_store, get_config, get_skill_store, get_tag_store
from app.gateway.path_utils import aresolve_thread_virtual_path
from app.gateway.perf_logging import finish_trace, trace_request, trace_stage
from app.gateway.skill_catalog_cache import (
_build_catalog_payload,
_load_usage_entries,
load_or_warm_skill_catalog_snapshot,
load_skill_catalog_projection,
refresh_skill_catalog_projection,
)
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.config.app_config import AppConfig
from deerflow.config.extensions_config import ExtensionsConfig, SkillDisplayConfig, get_extensions_config, patch_extensions_config
@@ -55,9 +39,6 @@ logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["skills"])
_UNSET = object()
_OWNER_USERNAME_CACHE_SECONDS = 60.0
_MAX_OWNER_USERNAME_CACHE_ENTRIES = 4096
_owner_username_cache: dict[str, tuple[float, str | None]] = {}
# ---------------------------------------------------------------------------
@@ -100,59 +81,21 @@ async def _attach_owner_usernames(responses: list["SkillResponse"]) -> list["Ski
provider = get_local_provider()
except Exception:
return responses
now = time.monotonic()
names: dict[str, str] = {}
missing_ids: set[str] = set()
for user_id in owner_ids:
cached = _owner_username_cache.get(user_id)
if cached is not None and cached[0] > now:
if cached[1]:
names[user_id] = cached[1]
else:
missing_ids.add(user_id)
if missing_ids:
for uid in owner_ids:
try:
users = await provider.get_users_by_ids(sorted(missing_ids))
user = await provider.get_user(uid)
except Exception:
logger.debug("skill owner username batch lookup failed", exc_info=True)
users = None
if users is not None:
loaded = {
str(user.id): _email_local_part(getattr(user, "email", None))
for user in users
}
expires_at = now + _OWNER_USERNAME_CACHE_SECONDS
for user_id in missing_ids:
local = loaded.get(user_id)
_owner_username_cache[user_id] = (expires_at, local)
if local:
names[user_id] = local
if len(_owner_username_cache) > _MAX_OWNER_USERNAME_CACHE_ENTRIES:
expired = [key for key, value in _owner_username_cache.items() if value[0] <= now]
for key in expired:
_owner_username_cache.pop(key, None)
if len(_owner_username_cache) > _MAX_OWNER_USERNAME_CACHE_ENTRIES:
oldest = sorted(_owner_username_cache, key=lambda key: _owner_username_cache[key][0])
excess = len(_owner_username_cache) - _MAX_OWNER_USERNAME_CACHE_ENTRIES
for key in oldest[:excess]:
_owner_username_cache.pop(key, None)
user = None
local = _email_local_part(getattr(user, "email", None)) if user is not None else None
if local:
names[uid] = local
for resp in responses:
if resp.owner_user_id:
resp.owner_username = names.get(resp.owner_user_id)
return responses
async def _refresh_catalog_snapshot(config: AppConfig) -> None:
"""Keep normalized MySQL skill rows current after a skill edit."""
try:
await refresh_skill_catalog_projection(config)
except Exception:
# A failed projection refresh must not turn a successful skill write
# into an API failure. Startup or the next edit can recover it.
logger.exception("Skill catalog projection refresh failed after a mutation")
def _can_view_skill_source(
category: SkillCategory,
owner_user_id: str | None,
@@ -373,7 +316,6 @@ def _skill_to_response(
skill: Skill,
ownership: dict[str, Any] | None = None,
*,
usage_entry: dict[str, Any] | None = None,
display_override: dict[str, Any] | None | object = _UNSET,
display_sample: dict[str, Any] | None = None,
favorited: bool = False,
@@ -385,12 +327,11 @@ def _skill_to_response(
``owner_user_id`` / ``published`` / ``builtin`` fields used for per-user
visibility and the frontend's owner badges.
"""
from deerflow.skills.usage import get_entry
from deerflow.skills.usage import get_entry, latest_activity_at
entry = usage_entry if usage_entry is not None else get_entry(skill.name)
entry = get_entry(skill.name)
created_at_str = entry.get("created_at")
activity_candidates = [entry.get("last_used_at"), entry.get("last_viewed_at"), entry.get("last_patched_at")]
last_at_str = max((value for value in activity_candidates if value), default=created_at_str)
last_at_str = latest_activity_at(skill.name)
created_at: datetime | None = None
last_at: datetime | None = None
if created_at_str:
@@ -464,93 +405,14 @@ def _skill_to_response(
)
def _catalog_item_to_response(
item: dict[str, Any],
ownership: dict[str, Any] | None,
*,
adapter: dict[str, Any] | None,
favorited: bool,
) -> SkillResponse:
"""Build a list response from the MySQL snapshot and MySQL metadata rows."""
try:
category = SkillCategory(str(item.get("category") or SkillCategory.PUBLIC.value))
except ValueError:
category = SkillCategory.PUBLIC
owner_user_id = ownership.get("owner_user_id") if ownership else None
return SkillResponse(
name=str(item.get("name") or ""),
name_zh=ownership.get("name_zh") if ownership else None,
description=str(item.get("description") or ""),
license=item.get("license"),
category=category,
enabled=bool(item.get("enabled", True)),
always_on=bool(ownership.get("always_on", item.get("always_on", False)) if ownership else item.get("always_on", False)),
use_count=int(item.get("use_count") or 0),
view_count=int(item.get("view_count") or 0),
patch_count=int(item.get("patch_count") or 0),
last_activity_at=item.get("last_activity_at"),
created_at=item.get("created_at"),
state=str(item.get("state") or "active"),
pinned=bool(item.get("pinned")),
featured=bool(item.get("featured")),
source=str(item.get("source") or "upload"),
owner_user_id=owner_user_id,
published=bool(ownership.get("published", False)) if ownership else False,
square_id=str(ownership.get("square_id") or "") if ownership else "",
builtin=bool(ownership.get("builtin", owner_user_id is None)) if ownership else category != SkillCategory.CUSTOM,
favorited=favorited,
detail=(ownership.get("detail_preview") or ownership.get("detail")) if ownership else None,
display=adapter.get("display") if adapter else None,
displaySample=adapter.get("sample") if adapter else None,
)
# ``checkfirst=True`` is a check-then-create operation. The in-process lock
# serializes concurrent requests in one worker; the duplicate-table fallback
# covers two Gateway workers racing during first access.
_SKILL_DISPLAY_TABLE_LOCK = asyncio.Lock()
_skill_display_table_engine: object | None = None
def _skill_display_table_already_exists(exc: BaseException) -> bool:
message = str(exc).lower()
return any(
token in message
for token in (
"already exists",
"duplicate table",
"42p07",
"error 1050",
"(1050",
)
)
async def _ensure_skill_display_table() -> bool:
"""Create the display adapter table once when DB persistence is enabled."""
global _skill_display_table_engine
"""Create the display adapter table when DB persistence is enabled."""
engine = get_engine()
sf = get_session_factory()
if engine is None or sf is None:
return False
if _skill_display_table_engine is engine:
return True
async with _SKILL_DISPLAY_TABLE_LOCK:
if _skill_display_table_engine is engine:
return True
try:
async with engine.begin() as conn:
await conn.run_sync(SkillDisplayAdapterRow.__table__.create, checkfirst=True)
except Exception as exc:
# The transaction context rolled back before this exception is
# accepted as the harmless result of another worker winning.
if not _skill_display_table_already_exists(exc):
raise
logger.info("Skill display adapter table was created concurrently; continuing")
_skill_display_table_engine = engine
async with engine.begin() as conn:
await conn.run_sync(SkillDisplayAdapterRow.__table__.create, checkfirst=True)
return True
@@ -599,18 +461,14 @@ async def _list_skill_display_adapters(skill_names: list[str]) -> dict[str, dict
return {}
async with sf() as session:
result = await session.execute(
select(SkillDisplayAdapterRow.skill_name, SkillDisplayAdapterRow.display_config).where(
SkillDisplayAdapterRow.skill_name.in_(skill_names)
)
select(SkillDisplayAdapterRow).where(SkillDisplayAdapterRow.skill_name.in_(skill_names))
)
return {
str(row.skill_name): {
row.skill_name: {
"display": _json_text_loads(row.display_config),
# The editor reads LONGTEXT sample_payload on demand; the list
# API only needs the lightweight display mapping.
"sample": None,
"sample": _json_text_loads(row.sample_payload),
}
for row in result
for row in result.scalars()
}
@@ -821,7 +679,7 @@ async def _generate_skill_display_sample(skill: Skill, app_config: AppConfig) ->
if parsed:
return parsed
logger.warning("Skill display sample model returned non-JSON for %s; falling back to static mock", skill.name)
except TimeoutError:
except asyncio.TimeoutError:
logger.warning("Skill display sample LLM call timed out for %s; falling back to static mock", skill.name)
except Exception:
logger.warning("Skill display sample LLM call failed for %s; falling back to static mock", skill.name, exc_info=True)
@@ -919,51 +777,20 @@ async def _build_ownership_index(skill_store: SkillStore, user_id: str) -> dict[
return {record["name"]: record for record in records}
def _skill_search_tokens(search: str | None) -> list[str]:
"""Split a search query into lowercase AND-tokens (whitespace separated).
``网页 报告`` therefore requires both words to appear, but each may live in
a different field (name vs description) — unlike the old contiguous
substring match which demanded the whole phrase verbatim.
"""
if not search:
return []
return [token for token in search.strip().lower().split() if token]
def _skill_matches_tokens(skill: SkillResponse, tokens: list[str]) -> bool:
"""Every token must appear in at least one searchable skill field.
Fields: English name, Chinese display name, the SKILL.md ``description``,
the long-form ``detail`` annotation, and assigned tag names — so users can
find a skill by what it does, not only by its name.
"""
if not tokens:
return True
haystacks = [
(skill.name or "").lower(),
(skill.name_zh or "").lower(),
(skill.description or "").lower(),
(skill.detail or "").lower(),
" ".join(str(t.get("name") or "").lower() for t in (skill.tags or [])),
]
return all(any(token in field for field in haystacks) for token in tokens)
async def _attach_and_filter_tags(
tag_store: TagStore,
responses: list[SkillResponse],
search: str | None,
tag_ids: list[str] | None,
) -> list[SkillResponse]:
"""Bulk-attach tags to skill responses, then apply search/tag filters."""
"""Bulk-attach tags to skill responses, then apply name/tag filters."""
assignments = await tag_store.list_assignments("skill", [r.name for r in responses])
for resp in responses:
resp.tags = assignments.get(resp.name, [])
if search:
# Tags are attached first so tag names participate in the match.
tokens = _skill_search_tokens(search)
responses = [r for r in responses if _skill_matches_tokens(r, tokens)]
needle = search.strip().lower()
# Match the English name OR the Chinese display name so 中文 search works.
responses = [r for r in responses if needle in r.name.lower() or (r.name_zh or "").lower().find(needle) >= 0]
if tag_ids:
wanted = set(tag_ids)
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
@@ -992,7 +819,7 @@ async def list_skills(
config: AppConfig = Depends(get_config),
skill_store: SkillStore = Depends(get_skill_store),
tag_store: TagStore = Depends(get_tag_store),
search: str | None = Query(default=None, description="Whitespace-separated keywords (AND); case-insensitive substring match against skill name, Chinese name (name_zh), description, detail, and tag names"),
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name or Chinese name (name_zh)"),
tag_ids: list[str] | None = Query(default=None, description="Keep skills carrying any of these tags (OR)"),
square: str | None = Query(default=None, description="Keep only skills filed under this publish square (发布广场). Omit for all squares."),
scope: str | None = Query(default=None, description="``mine`` filters to the skills the caller's 岗位 (position) makes visible."),
@@ -1010,138 +837,68 @@ async def list_skills(
),
),
) -> SkillsListResponse:
trace = trace_request(request, "skills_list")
cache_fill_token: str | None = None
cache_config = None
cache_key = ""
try:
user_id = _current_user_id(request)
if admin and not _is_admin_user(request):
raise HTTPException(status_code=403, detail="Only system admins can list all skills")
cacheable = (
not admin
and not (search or "").strip()
and not tag_ids
and not (square or "").strip()
and sort == "time_desc"
)
query_fingerprint = json.dumps(
{
"search": search,
"tag_ids": sorted(tag_ids or []),
"square": square,
"scope": scope,
"sort": sort,
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
cache_key = f"user:{user_id}:query:{query_fingerprint}" if cacheable else ""
cache_version = None
request_app_state = getattr(getattr(request, "app", None), "state", None)
cache_config = getattr(request_app_state, "config", config)
if cacheable:
stage_started = time.perf_counter()
cache_version = await get_cache_version(cache_config, "skills")
cached_payload = await get_cached_json_payload(
cache_config,
"skills",
cache_key,
version=cache_version,
)
if cached_payload is None:
cached_payload, cache_fill_token = await wait_for_cached_json_payload_or_claim_fill(
cache_config,
"skills",
cache_key,
version=cache_version,
)
trace_stage(trace, "skills_redis_cache", stage_started)
if cached_payload is not None:
trace.setdefault("details", {})["cache"] = "redis_hit"
return Response(content=cached_payload, media_type="application/json")
stage_started = time.perf_counter()
catalog = await load_skill_catalog_projection(user_id=user_id, admin=admin)
if catalog is None:
# Preserve no-database operation. A MySQL-backed request always
# uses the normalized rows and does not scan SKILL.md files.
legacy_catalog = await load_or_warm_skill_catalog_snapshot(config)
if legacy_catalog is not None:
catalog = legacy_catalog
trace_stage(trace, "skills_legacy_projection_fallback", stage_started)
else:
# This request's dependency and the singleton share the active
# app config. Reuse the process instance to retain parsed skill
# metadata between catalogue reads.
storage = get_or_new_skill_storage(app_config=config)
skills_from_disk = await asyncio.to_thread(storage.load_skills, enabled_only=False)
usage_entries = await asyncio.to_thread(_load_usage_entries)
catalog = await asyncio.to_thread(_build_catalog_payload, skills_from_disk, usage_entries)
trace_stage(trace, "skills_memory_mode_catalog_load", stage_started)
else:
trace_stage(trace, "skills_mysql_projection", stage_started)
trace.setdefault("details", {})["skills_discovered"] = len(catalog)
if get_session_factory() is not None:
# Projection rows include the compact ownership fields; never load
# the SkillRow.detail LONGTEXT column on catalog requests.
ownership = {str(record["name"]): record for record in catalog}
trace_stage(trace, "skills_ownership_in_projection", stage_started)
elif admin:
stage_started = time.perf_counter()
all_skills = get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False)
if admin:
ownership_records = await skill_store.list_all()
ownership = {record["name"]: record for record in ownership_records}
trace_stage(trace, "skills_ownership_db", stage_started)
else:
stage_started = time.perf_counter()
ownership = await _build_ownership_index(skill_store, user_id)
trace_stage(trace, "skills_ownership_db", stage_started)
# Skills the caller's 岗位 (position) granted into their "我的技能"
# collection. Admins browse everything, so the badge is moot for them.
favorite_names: set[str] = set()
if not admin:
try:
stage_started = time.perf_counter()
favorite_names = await skill_store.list_favorite_skill_names(user_id)
trace_stage(trace, "skills_favorites_db", stage_started)
except Exception:
logger.debug("skill favorites lookup failed", exc_info=True)
favorite_names = set()
stage_started = time.perf_counter()
skill_names = [str(item["name"]) for item in catalog if item.get("name")]
display_adapters = await _list_skill_display_adapters(skill_names)
trace_stage(trace, "skills_display_adapters_db", stage_started)
stage_started = time.perf_counter()
display_adapters = await _list_skill_display_adapters([skill.name for skill in all_skills])
responses: list[SkillResponse] = []
for item in catalog:
skill_name = str(item.get("name") or "")
if not skill_name:
continue
try:
category = SkillCategory(str(item.get("category") or SkillCategory.PUBLIC.value))
except ValueError:
category = SkillCategory.PUBLIC
adapter = display_adapters.get(skill_name)
record = ownership.get(skill_name)
for skill in all_skills:
adapter = display_adapters.get(skill.name)
# Public/built-in skills (no DB row) are visible to everyone.
# Custom skills must have an ownership record in the visible set.
if category == SkillCategory.CUSTOM and record is None and not admin and skill_name not in favorite_names:
if skill.category != SkillCategory.CUSTOM:
# Built-in skills usually have no DB row, but an admin-authored
# ``detail`` annotation materializes one — surface it when present.
responses.append(
_skill_to_response(
skill,
ownership.get(skill.name),
display_override=_display_override_from_adapter(adapter),
display_sample=adapter.get("sample") if adapter else None,
favorited=skill.name in favorite_names,
)
)
continue
record = ownership.get(skill.name)
if record is None:
if admin:
# Admin sees every custom skill on disk even without an
# ownership row (legacy / orphaned), so they can clean it up.
record = None
elif skill.name in favorite_names:
# A position granted this skill even though it isn't
# otherwise visible to the user (e.g. an unpublished custom
# skill). Surface it so it lands in their 我的技能.
record = await skill_store.get_any(skill.name)
else:
continue
responses.append(
_catalog_item_to_response(
item,
_skill_to_response(
skill,
record,
adapter=adapter,
favorited=skill_name in favorite_names,
display_override=_display_override_from_adapter(adapter),
display_sample=adapter.get("sample") if adapter else None,
favorited=skill.name in favorite_names,
)
)
trace_stage(trace, "skills_response_assembly", stage_started)
stage_started = time.perf_counter()
responses = await _attach_and_filter_tags(tag_store, responses, search, tag_ids)
trace_stage(trace, "skills_tags_db_and_filter", stage_started)
stage_started = time.perf_counter()
responses = await _attach_owner_usernames(responses)
trace_stage(trace, "skills_owner_usernames_db", stage_started)
square_id = square.strip() if square and square.strip() else None
if square_id:
from deerflow.config.system_settings import load_system_settings, resolve_default_square_id
@@ -1158,30 +915,13 @@ async def list_skills(
# mirroring how agents are added to 我的智能体. The browse-all 广场 therefore
# always shows every visible skill — the ``scope`` param is kept for API
# compatibility but intentionally does not filter here.
stage_started = time.perf_counter()
responses = _sort_skill_responses(responses, sort)
trace_stage(trace, "skills_sort", stage_started)
trace.setdefault("details", {})["skills_returned"] = len(responses)
result = SkillsListResponse(skills=responses)
if cacheable:
await set_cached_json(
cache_config,
"skills",
cache_key,
result,
expected_version=cache_version,
)
trace.setdefault("details", {})["cache"] = "redis_miss"
return result
return SkillsListResponse(skills=responses)
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to load skills: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to load skills: {str(e)}")
finally:
if cache_fill_token is not None and cache_config is not None and cache_key:
await release_cache_fill(cache_config, "skills", cache_key, cache_fill_token)
finish_trace(trace)
_MAX_SKILL_UPLOAD_BYTES = 512 * 1024 * 1024
@@ -1360,7 +1100,6 @@ async def install_skill_upload(
await skill_store.ensure_legacy(
{"name": result["skill_name"], "owner_user_id": user_id, "published": False}
)
await _refresh_catalog_snapshot(config)
await refresh_skills_system_prompt_cache_async()
return SkillInstallResponse(**result)
except FileNotFoundError as e:
@@ -1616,7 +1355,6 @@ async def install_skill(
await skill_store.ensure_legacy(
{"name": result.skill_name, "owner_user_id": user_id, "published": False}
)
await _refresh_catalog_snapshot(config)
await refresh_skills_system_prompt_cache_async()
return result
except FileNotFoundError as e:
@@ -1730,7 +1468,6 @@ async def duplicate_skill(
mark_uploaded(new_name)
except Exception:
pass
await _refresh_catalog_snapshot(config)
await refresh_skills_system_prompt_cache_async()
return SkillInstallResponse(
success=True,
@@ -1804,16 +1541,9 @@ async def list_custom_skills(
) -> SkillsListResponse:
try:
user_id = _current_user_id(request)
storage = get_or_new_skill_storage()
all_skills, usage_entries = await asyncio.gather(
asyncio.to_thread(storage.load_skills, enabled_only=False),
asyncio.to_thread(_load_usage_entries),
)
custom = [skill for skill in all_skills if skill.category == SkillCategory.CUSTOM]
ownership, display_adapters = await asyncio.gather(
_build_ownership_index(skill_store, user_id),
_list_skill_display_adapters([skill.name for skill in custom]),
)
custom = [s for s in get_or_new_skill_storage(app_config=config).load_skills(enabled_only=False) if s.category == SkillCategory.CUSTOM]
ownership = await _build_ownership_index(skill_store, user_id)
display_adapters = await _list_skill_display_adapters([skill.name for skill in custom])
responses = []
for skill in custom:
if skill.name not in ownership:
@@ -1823,7 +1553,6 @@ async def list_custom_skills(
_skill_to_response(
skill,
ownership[skill.name],
usage_entry=usage_entries.get(skill.name, {}),
display_override=_display_override_from_adapter(adapter),
display_sample=adapter.get("sample") if adapter else None,
)
@@ -1871,7 +1600,6 @@ async def update_custom_skill(skill_name: str, request: CustomSkillUpdateRequest
raise HTTPException(status_code=400, detail=f"Security scan blocked the edit: {scan.reason}")
prev_content = storage.read_custom_skill(skill_name)
storage.write_custom_skill(skill_name, SKILL_MD_FILE, request.content)
await _refresh_catalog_snapshot(config)
storage.append_history(
skill_name,
{
@@ -1941,7 +1669,6 @@ async def delete_custom_skill(
await skill_store.delete_admin(skill_name)
else:
await skill_store.delete(skill_name, user_id)
await _refresh_catalog_snapshot(config)
try:
await get_tag_store(request).unassign_all("skill", skill_name)
@@ -2044,7 +1771,6 @@ async def delete_builtin_skill(
skill_owner_user_id=None,
agent_store=agent_store,
)
await _refresh_catalog_snapshot(config)
await refresh_skills_system_prompt_cache_async()
await _sync_skill_view(skill.name)
logger.info("Administrator deleted built-in skill %s", skill.name)
@@ -2187,7 +1913,6 @@ async def update_custom_skill_state(
from deerflow.skills.usage import set_state as _usage_set_state
_usage_set_state(skill_name, body.state)
await _refresh_catalog_snapshot(config)
await refresh_skills_system_prompt_cache_async()
return _reload_skill_response(skill_name, record, config)
except HTTPException:
@@ -2219,7 +1944,6 @@ async def update_custom_skill_pinned(
from deerflow.skills.usage import set_pinned as _usage_set_pinned
_usage_set_pinned(skill_name, body.pinned)
await _refresh_catalog_snapshot(config)
return _reload_skill_response(skill_name, record, config)
except HTTPException:
raise
@@ -2310,7 +2034,6 @@ async def rollback_custom_skill(skill_name: str, request: SkillRollbackRequest,
storage.append_history(skill_name, history_entry)
raise HTTPException(status_code=400, detail=f"Rollback blocked by security scanner: {scan.reason}")
storage.write_custom_skill(skill_name, SKILL_MD_FILE, target_content)
await _refresh_catalog_snapshot(config)
storage.append_history(skill_name, history_entry)
await refresh_skills_system_prompt_cache_async()
return await get_custom_skill(skill_name, config)
@@ -3367,7 +3090,7 @@ async def _generate_skill_name_zh(skill: Skill, app_config: AppConfig, model: An
),
timeout=60,
)
except TimeoutError as e:
except asyncio.TimeoutError as e:
logger.warning("Skill name_zh LLM call timed out for %s", skill.name)
raise SkillNameZhModelError("大模型响应超时(60 秒)") from e
except Exception as e:
@@ -272,7 +272,6 @@ class StrategyAgentItem(BaseModel):
name: str
description: str = ""
model: str | None = None
model_prompt_suffix_enabled: bool = True
soul: str | None = None
skills: list[str] | None = None
created_at: datetime | str | None = None
@@ -288,7 +287,6 @@ class StrategyAngleCreateRequest(BaseModel):
description: str = Field(default="", max_length=2000)
soul: str = Field(default="", max_length=200_000)
model: str | None = Field(default=None, max_length=200)
model_prompt_suffix_enabled: bool = True
skills: list[str] = Field(default_factory=list, max_length=100)
sort_order: int | None = Field(default=None, ge=-10_000, le=10_000)
@@ -298,7 +296,6 @@ class StrategyAgentUpdateRequest(BaseModel):
description: str | None = Field(default=None, max_length=2000)
soul: str | None = Field(default=None, max_length=200_000)
model: str | None = Field(default=None, max_length=200)
model_prompt_suffix_enabled: bool | None = None
skills: list[str] | None = Field(default=None, max_length=100)
enabled: bool | None = None
sort_order: int | None = Field(default=None, ge=-10_000, le=10_000)
@@ -397,7 +394,6 @@ async def _to_item(
name=agent.name,
description=agent.description,
model=agent.model,
model_prompt_suffix_enabled=agent.model_prompt_suffix_enabled,
soul=agent.soul,
skills=agent.skills,
created_at=agent.created_at,
@@ -517,7 +513,6 @@ async def create_strategy_angle(
payload=payload,
name=name,
model=model,
model_prompt_suffix_enabled=payload.model_prompt_suffix_enabled,
skills=skills,
agent_store=agent_store,
role_store=role_store,
@@ -551,7 +546,6 @@ async def _create_strategy_angle_locked(
payload: StrategyAngleCreateRequest,
name: str,
model: str | None,
model_prompt_suffix_enabled: bool,
skills: list[str] | None,
agent_store: AgentStore,
role_store: StrategyAgentRoleStore,
@@ -585,7 +579,6 @@ async def _create_strategy_angle_locked(
description=payload.description,
skills=skills,
model=model,
model_prompt_suffix_enabled=model_prompt_suffix_enabled,
tool_groups=_DYNAMIC_ANGLE_TOOL_GROUPS,
)
await asyncio.to_thread(agents_router._write_agent_soul, agent_id, soul)
@@ -670,11 +663,6 @@ async def _update_strategy_agent_locked(
next_description = payload.description or "" if "description" in fields else str(record.get("description") or "")
extras = await asyncio.to_thread(agents_router._load_config_extras, agent_id)
next_model = _normalize_model(request, payload.model) if "model" in fields else extras["model"]
next_model_prompt_suffix_enabled = (
bool(payload.model_prompt_suffix_enabled)
if "model_prompt_suffix_enabled" in fields
else extras["model_prompt_suffix_enabled"]
)
next_skills = (
await agents_router._validate_strategy_agent_skills(
payload.skills,
@@ -698,7 +686,7 @@ async def _update_strategy_agent_locked(
original_role = dict(role)
role_store = get_strategy_agent_role_store(request)
try:
config_fields = {"name", "description", "model", "model_prompt_suffix_enabled", "skills"}
config_fields = {"name", "description", "model", "skills"}
if fields & config_fields:
await asyncio.to_thread(
agents_router._write_agent_config,
@@ -707,13 +695,11 @@ async def _update_strategy_agent_locked(
description=next_description,
skills=next_skills,
model=next_model,
model_prompt_suffix_enabled=next_model_prompt_suffix_enabled,
tool_groups=extras["tool_groups"],
common_questions=extras["common_questions"],
template_name=extras["template_name"],
seat_skill_directive=extras["seat_skill_directive"],
llmwiki_knowledge_base_ids=extras["llmwiki_knowledge_base_ids"],
chat_link=extras["chat_link"],
)
await agent_store.replace_skills(agent_id, next_skills)
await agent_store.set_extras(
@@ -1,181 +0,0 @@
"""Admin-only model-stream truncation event log (模型截断记录).
Companion to the per-model ``strict_stream_integrity`` switch on the model
management page: the patched OpenAI-compatible streaming layer writes one row
per abnormally ended provider stream (silently committed half answers, strict
integrity detections, output-length caps, mid-stream transport errors). This
router serves the admin viewing page; rows are retained for at most one month
by the SQL repository.
"""
from __future__ import annotations
import logging
from datetime import datetime
from typing import Any
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request, get_stream_truncation_store
from deerflow.persistence.stream_truncation.base import StreamTruncationQuery, StreamTruncationStore
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/stream-truncations", tags=["stream-truncations"])
_EVENT_TYPES = (
"silent_no_terminal",
"incomplete_stream_error",
"length_truncated",
"content_filter",
"midstream_error",
)
class StreamTruncationEventResponse(BaseModel):
id: str
created_at: str | None = None
model_name: str | None = None
provider_completion_id: str | None = None
run_id: str | None = None
event_type: str
strict_mode: bool = False
finish_reason: str = "none"
status: str = "success"
error_type: str | None = None
chunk_count: int = 0
duration_ms: int = 0
first_chunk_ms: int = -1
detail: str | None = None
class StreamTruncationListResponse(BaseModel):
items: list[StreamTruncationEventResponse]
total: int = Field(..., description="Total rows matching the filter, ignoring pagination")
limit: int
offset: int
class StreamTruncationBreakdownItem(BaseModel):
model_name: str
total: int = 0
last_at: str | None = None
silent_no_terminal: int = 0
incomplete_stream_error: int = 0
length_truncated: int = 0
content_filter: int = 0
midstream_error: int = 0
class StreamTruncationBreakdownResponse(BaseModel):
items: list[StreamTruncationBreakdownItem]
async def _require_admin(request: Request) -> None:
user = await get_optional_user_from_request(request)
if user is None:
raise HTTPException(status_code=401, detail="Authentication is required")
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="Stream truncation log is admin-only")
def _require_store(request: Request) -> StreamTruncationStore:
store = get_stream_truncation_store(request)
if store is None:
raise HTTPException(status_code=503, detail="Stream truncation store is not configured")
return store
def _parse_dt(value: str | datetime | None) -> datetime | None:
if not value:
return None
if isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise HTTPException(status_code=422, detail=f"Invalid datetime: {value!r} ({exc})") from exc
def _build_query(
*,
model_name: str | None,
event_type: str | None,
since: str | datetime | None,
until: str | datetime | None,
limit: int,
offset: int,
) -> StreamTruncationQuery:
if event_type and event_type not in _EVENT_TYPES:
raise HTTPException(status_code=422, detail=f"Unknown event type: {event_type}")
return StreamTruncationQuery(
model_name=model_name or None,
event_type=event_type or None,
since=_parse_dt(since),
until=_parse_dt(until),
limit=limit,
offset=offset,
)
def _fill_breakdown_zeros(item: dict[str, Any]) -> dict[str, Any]:
for event_type in _EVENT_TYPES:
item.setdefault(event_type, 0)
return item
@router.get("", response_model=StreamTruncationListResponse)
async def list_stream_truncations(
request: Request,
model_name: str | None = Query(default=None, max_length=128),
event_type: str | None = Query(default=None, max_length=32),
since: str | None = Query(default=None),
until: str | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> StreamTruncationListResponse:
"""List truncation events, newest first."""
await _require_admin(request)
store = _require_store(request)
query = _build_query(
model_name=model_name,
event_type=event_type,
since=since,
until=until,
limit=limit,
offset=offset,
)
items = await store.list(query)
total = await store.count(query)
return StreamTruncationListResponse(
items=[StreamTruncationEventResponse(**item) for item in items],
total=total,
limit=limit,
offset=offset,
)
@router.get("/breakdown", response_model=StreamTruncationBreakdownResponse)
async def stream_truncation_breakdown(
request: Request,
since: str | None = Query(default=None),
until: str | None = Query(default=None),
) -> StreamTruncationBreakdownResponse:
"""Per-model aggregate of truncation events over the query window."""
await _require_admin(request)
store = _require_store(request)
query = _build_query(
model_name=None,
event_type=None,
since=since,
until=until,
limit=1,
offset=0,
)
rows = await store.model_breakdown(query)
return StreamTruncationBreakdownResponse(
items=[
StreamTruncationBreakdownItem(**_fill_breakdown_zeros(item))
for item in rows
]
)
@@ -24,14 +24,11 @@ from deerflow.config.system_settings import (
HtmlMapSettings,
LeaderboardSettings,
ModelConcurrencySettings,
ModelPromptSuffix,
PluginDomainAgentMapping,
PluginManagementSettings,
PromptPrefixSettings,
PublishSquare,
PublishSquaresSettings,
RetryKeepPartialSettings,
RoundtableRoomLimitSettings,
SkillCompressionSettings,
StrategyProjectSettings,
WordHeadingNumberingStyle,
@@ -43,138 +40,12 @@ from deerflow.config.system_settings import (
save_system_settings,
set_module_tag_ids,
)
from deerflow.html_runtime.addresses import (
html_map_tile_url,
html_runtime_base_url,
normalize_html_map_tile_url,
normalize_html_runtime_base_url,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/system-settings", tags=["system-settings"])
@router.get("/model-prompt-suffixes", response_model=dict[str, ModelPromptSuffix])
async def get_model_prompt_suffixes(request: Request) -> dict[str, ModelPromptSuffix]:
await _require_model_prompt_admin(request)
return load_system_settings().model_prompt_suffixes
@router.put("/model-prompt-suffixes/{model_name:path}", response_model=ModelPromptSuffix)
async def update_model_prompt_suffix(model_name: str, body: ModelPromptSuffix, request: Request) -> ModelPromptSuffix:
await _require_model_prompt_admin(request)
current = load_system_settings()
# Retain editable entries for temporarily removed models so an administrator
# can still disable them. New entries must target a real configured model.
if get_app_config().get_model_config(model_name) is None and model_name not in current.model_prompt_suffixes:
raise HTTPException(status_code=404, detail="模型不存在")
if body.enabled and not body.prompt.strip():
raise HTTPException(status_code=422, detail="启用时请填写追加提示词")
current.model_prompt_suffixes[model_name] = body
save_system_settings(current)
return body
async def _require_model_prompt_admin(request: Request) -> None:
user = await get_optional_user_from_request(request)
if user is None:
raise HTTPException(status_code=401, detail="请先登录")
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="仅管理员可管理模型追加提示词")
class ApiReadCacheResponse(BaseModel):
enabled: bool = False
redis_configured: bool = False
ttl_seconds: int = 300
max_value_bytes: int = 524288
browser_status_enabled: bool = True
collections: list[dict] = Field(default_factory=list)
class ApiReadCacheUpdateRequest(BaseModel):
enabled: bool | None = None
ttl_seconds: int | None = Field(default=None, ge=1, le=3600)
browser_status_enabled: bool | None = None
collection_overrides: dict[str, bool] | None = None
async def _require_performance_admin(request: Request) -> None:
# Preserve no-auth deployments, but never swallow an invalid/missing login.
user = await get_current_user_from_request(request)
if user is not None and getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="仅管理员可管理接口性能设置")
@router.get("/api-read-cache", response_model=ApiReadCacheResponse)
async def get_api_read_cache_setting(request: Request) -> ApiReadCacheResponse:
"""Read the persisted administrator-controlled Redis cache switch."""
await _require_performance_admin(request)
from app.gateway.api_read_cache import api_read_cache_redis_configured, get_api_cache_controls, get_api_read_cache_settings
config = getattr(request.app.state, "config", None)
cache_config = getattr(config, "api_read_cache", None)
enabled, ttl_seconds = await get_api_read_cache_settings(config, force_refresh=True)
return ApiReadCacheResponse(
enabled=enabled,
redis_configured=api_read_cache_redis_configured(config),
ttl_seconds=ttl_seconds,
max_value_bytes=int(getattr(cache_config, "max_value_bytes", 524288) or 524288),
**await get_api_cache_controls(config),
)
@router.put("/api-read-cache", response_model=ApiReadCacheResponse)
async def update_api_read_cache_setting(
request: Request,
body: ApiReadCacheUpdateRequest,
) -> ApiReadCacheResponse:
"""Save the Redis cache switch to MySQL and apply it immediately."""
await _require_performance_admin(request)
from app.gateway.api_read_cache import (
API_READ_CACHE_COLLECTIONS,
api_read_cache_redis_configured,
get_api_cache_controls,
get_api_read_cache_settings,
invalidate_cached_collection,
save_api_read_cache_settings,
)
config = getattr(request.app.state, "config", None)
unknown = set(body.collection_overrides or {}) - set(API_READ_CACHE_COLLECTIONS)
if unknown:
raise HTTPException(status_code=422, detail=f"未知缓存接口:{', '.join(sorted(unknown))}")
try:
await save_api_read_cache_settings(
body.enabled, body.ttl_seconds, config=config,
collection_overrides=body.collection_overrides,
browser_status_enabled=body.browser_status_enabled,
)
await asyncio.gather(
*(
invalidate_cached_collection(config, collection, force=True)
for collection in API_READ_CACHE_COLLECTIONS
)
)
except Exception as exc:
logger.exception("Could not persist API Redis cache setting")
raise HTTPException(status_code=503, detail="无法将缓存设置保存到 MySQL") from exc
cache_config = getattr(config, "api_read_cache", None)
enabled, ttl_seconds = await get_api_read_cache_settings(config, force_refresh=True)
return ApiReadCacheResponse(
enabled=enabled,
redis_configured=api_read_cache_redis_configured(config),
ttl_seconds=ttl_seconds,
max_value_bytes=int(getattr(cache_config, "max_value_bytes", 524288) or 524288),
**await get_api_cache_controls(config),
)
class PromptPrefixResponse(BaseModel):
image_generation_unavailable_hint_enabled: bool = False
python_dependency_delete_protection_enabled: bool = False
enabled: bool = Field(..., description="Whether the toggle is exposed to users")
switch_label: str = Field(..., description="Toggle label shown in the chat input")
prompt_prefix: str = Field(..., description="Prefix text prepended to the first user message")
@@ -250,8 +121,6 @@ async def update_strategy_project_settings(
class PromptPrefixUpdateRequest(BaseModel):
image_generation_unavailable_hint_enabled: bool | None = None
python_dependency_delete_protection_enabled: bool | None = None
enabled: bool | None = None
switch_label: str | None = None
prompt_prefix: str | None = None
@@ -310,16 +179,6 @@ class HtmlMapSettingsUpdateRequest(BaseModel):
engine: Literal["echarts", "leaflet"] | None = None
class HtmlRuntimeSettingsResponse(BaseModel):
base_url: str
map_tile_url: str
class HtmlRuntimeSettingsUpdateRequest(BaseModel):
base_url: str
map_tile_url: str
class LeaderboardSettingsResponse(BaseModel):
exclude_user_ids: list[str] = Field(default_factory=list)
include_scheduled: bool = True
@@ -338,8 +197,6 @@ class LeaderboardSettingsUpdateRequest(BaseModel):
def _to_response(settings: PromptPrefixSettings) -> PromptPrefixResponse:
return PromptPrefixResponse(
image_generation_unavailable_hint_enabled=settings.image_generation_unavailable_hint_enabled,
python_dependency_delete_protection_enabled=settings.python_dependency_delete_protection_enabled,
enabled=settings.enabled,
switch_label=settings.switch_label,
prompt_prefix=settings.prompt_prefix,
@@ -532,22 +389,7 @@ class WriteFileCompatibilityUpdateRequest(BaseModel):
enabled: bool
class RetryKeepPartialResponse(BaseModel):
enabled: bool = Field(
default=True,
description=(
"Keep the already-streamed partial answer visible when a model "
"call is silently retried"
),
)
class RetryKeepPartialUpdateRequest(BaseModel):
enabled: bool
class ConversationContextResponse(BaseModel):
max_run_steps: int = 1000
recent_turn_window_enabled: bool = Field(
default=False,
description="Whether all lead-agent model requests keep only the latest three dialogue turns",
@@ -556,39 +398,10 @@ class ConversationContextResponse(BaseModel):
default=3,
description="Fixed number of real user turns retained by the sliding window",
)
max_conversation_rounds: int = Field(
default=20,
ge=1,
le=50,
description="Maximum real user turns allowed in one ordinary chat thread",
)
use_question_as_title: bool = Field(
default=False,
description="Whether the first user question is the final title, without post-answer title generation",
)
class ConversationContextUpdateRequest(BaseModel):
max_run_steps: int | None = Field(default=None, ge=20, le=10000, strict=True)
recent_turn_window_enabled: bool | None = None
max_conversation_rounds: int | None = Field(default=None, ge=1, le=50)
use_question_as_title: bool | None = None
class RoundtableRoomLimitResponse(BaseModel):
enabled: bool = True
max_rooms: int = 3
max_hold_minutes: int = 30
max_cycles: int = 8
write_file_compatibility_enabled: bool = False
class RoundtableRoomLimitUpdateRequest(BaseModel):
enabled: bool | None = None
max_rooms: int | None = Field(default=None, ge=1, le=20)
max_hold_minutes: int | None = Field(default=None, ge=1, le=1440)
max_cycles: int | None = Field(default=None, ge=2, le=100)
write_file_compatibility_enabled: bool | None = None
recent_turn_window_enabled: bool
class OrdinaryKnowledgeRetrievalResponse(BaseModel):
@@ -667,36 +480,10 @@ async def update_write_file_compatibility_setting(
)
@router.get("/retry-keep-partial", response_model=RetryKeepPartialResponse)
async def get_retry_keep_partial_setting() -> RetryKeepPartialResponse:
"""Return the retry-keeps-partial display switch (readable by chat clients)."""
settings = load_system_settings().retry_keep_partial
return RetryKeepPartialResponse(enabled=settings.enabled)
@router.put("/retry-keep-partial", response_model=RetryKeepPartialResponse)
async def update_retry_keep_partial_setting(
body: RetryKeepPartialUpdateRequest,
request: Request,
) -> RetryKeepPartialResponse:
await _require_admin(request)
current = load_system_settings()
updated = RetryKeepPartialSettings(enabled=body.enabled)
saved = save_system_settings(current.model_copy(update={"retry_keep_partial": updated}))
logger.info(
"Retry-keeps-partial display switch changed: enabled=%s",
saved.retry_keep_partial.enabled,
)
return RetryKeepPartialResponse(enabled=saved.retry_keep_partial.enabled)
def _conversation_context_response(settings: ConversationContextSettings) -> ConversationContextResponse:
return ConversationContextResponse(
max_run_steps=settings.max_run_steps,
recent_turn_window_enabled=settings.recent_turn_window_enabled,
recent_turns=3,
max_conversation_rounds=settings.max_conversation_rounds,
use_question_as_title=settings.use_question_as_title,
)
@@ -711,72 +498,20 @@ async def update_conversation_context_setting(
body: ConversationContextUpdateRequest,
request: Request,
) -> ConversationContextResponse:
"""Update the recent-turn window or ordinary-chat round cap. Admin only."""
"""Enable or disable the fixed recent-three-turn window. Admin only."""
await _require_admin(request)
current = load_system_settings()
updated_fields = body.model_dump(exclude_unset=True, exclude_none=True)
if not updated_fields:
raise HTTPException(status_code=422, detail="至少需要提供一个上下文设置项")
updated = current.conversation_context.model_copy(update=updated_fields)
updated = current.conversation_context.model_copy(
update={"recent_turn_window_enabled": body.recent_turn_window_enabled}
)
saved = save_system_settings(current.model_copy(update={"conversation_context": updated}))
logger.info(
"Conversation runtime settings changed: recent_turn_window=%s use_question_as_title=%s max_conversation_rounds=%s max_run_steps=%s",
"Conversation recent-turn window changed: enabled=%s retained_turns=3",
saved.conversation_context.recent_turn_window_enabled,
saved.conversation_context.use_question_as_title,
saved.conversation_context.max_conversation_rounds,
saved.conversation_context.max_run_steps,
)
return _conversation_context_response(saved.conversation_context)
def _roundtable_room_limit_response(settings: RoundtableRoomLimitSettings) -> RoundtableRoomLimitResponse:
return RoundtableRoomLimitResponse(
enabled=settings.enabled,
max_rooms=settings.max_rooms,
max_hold_minutes=settings.max_hold_minutes,
max_cycles=settings.max_cycles,
write_file_compatibility_enabled=settings.write_file_compatibility_enabled,
)
@router.get("/roundtable-room-limit", response_model=RoundtableRoomLimitResponse)
async def get_roundtable_room_limit_setting() -> RoundtableRoomLimitResponse:
"""Return the global step-2 room and closeout settings."""
return _roundtable_room_limit_response(load_system_settings().roundtable_room_limit)
@router.put("/roundtable-room-limit", response_model=RoundtableRoomLimitResponse)
async def update_roundtable_room_limit_setting(
body: RoundtableRoomLimitUpdateRequest,
request: Request,
) -> RoundtableRoomLimitResponse:
"""Update global room limits and the coordinator closeout round. Admin only."""
await _require_admin(request)
current = load_system_settings()
updates: dict[str, object] = {}
if body.enabled is not None:
updates["enabled"] = body.enabled
if body.max_rooms is not None:
updates["max_rooms"] = body.max_rooms
if body.max_hold_minutes is not None:
updates["max_hold_minutes"] = body.max_hold_minutes
if body.max_cycles is not None:
updates["max_cycles"] = body.max_cycles
if body.write_file_compatibility_enabled is not None:
updates["write_file_compatibility_enabled"] = body.write_file_compatibility_enabled
updated = current.roundtable_room_limit.model_copy(update=updates)
saved = save_system_settings(current.model_copy(update={"roundtable_room_limit": updated}))
logger.info(
"Roundtable runtime limits changed: enabled=%s max_rooms=%s max_hold_minutes=%s max_cycles=%s write_file_compatibility=%s",
saved.roundtable_room_limit.enabled,
saved.roundtable_room_limit.max_rooms,
saved.roundtable_room_limit.max_hold_minutes,
saved.roundtable_room_limit.max_cycles,
saved.roundtable_room_limit.write_file_compatibility_enabled,
)
return _roundtable_room_limit_response(saved.roundtable_room_limit)
@router.get(
"/ordinary-knowledge-retrieval",
response_model=OrdinaryKnowledgeRetrievalResponse,
@@ -1071,38 +806,6 @@ async def update_html_map_settings(
return HtmlMapSettingsResponse(engine=saved.html_map.engine)
@router.get("/html-runtime", response_model=HtmlRuntimeSettingsResponse)
async def get_html_runtime_settings() -> HtmlRuntimeSettingsResponse:
"""Return the addresses newly generated HTML pages will use."""
return HtmlRuntimeSettingsResponse(
base_url=html_runtime_base_url(),
map_tile_url=html_map_tile_url(),
)
@router.put("/html-runtime", response_model=HtmlRuntimeSettingsResponse)
async def update_html_runtime_settings(
body: HtmlRuntimeSettingsUpdateRequest,
request: Request,
) -> HtmlRuntimeSettingsResponse:
"""Save the browser-reachable runtime and map-tile addresses. Admin only."""
await _require_admin(request)
try:
base_url = normalize_html_runtime_base_url(body.base_url)
map_tile_url = normalize_html_map_tile_url(body.map_tile_url)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
current = load_system_settings()
updated = current.html_runtime.model_copy(
update={"base_url": base_url, "map_tile_url": map_tile_url}
)
saved = save_system_settings(current.model_copy(update={"html_runtime": updated}))
return HtmlRuntimeSettingsResponse(
base_url=saved.html_runtime.base_url,
map_tile_url=saved.html_runtime.map_tile_url,
)
@router.get("/private-square", response_model=PrivateSquareStatusResponse)
async def get_private_square_status() -> PrivateSquareStatusResponse:
"""Return whether the private square is enabled and a password is set."""
@@ -1236,12 +939,6 @@ class AppearanceResponse(BaseModel):
default_skill_view: Literal["card", "list"] = Field(..., description="System default skill list display mode — card or list")
default_agent_view: Literal["card", "list"] = Field(..., description="System default agent list display mode — card or list")
default_layout_mode: Literal["full", "compact"] = Field(..., description="System default interface layout — full (sidebar + header) or compact (hidden)")
default_composer_width_full: int = Field(..., description="System default composer width percent for full layout. 100 keeps the built-in width.")
default_composer_width_compact: int = Field(..., description="System default composer width percent for compact layout. 100 keeps the built-in width.")
default_composer_height_full: int = Field(..., description="System default composer height percent for full layout. 100 keeps the built-in height.")
default_composer_height_compact: int = Field(..., description="System default composer height percent for compact layout. 100 keeps the built-in height.")
default_agent_composer_width: int = Field(..., description="System default agent composer width percent. 100 keeps the built-in width.")
default_agent_composer_height: int = Field(..., description="System default agent composer height percent. 100 keeps the built-in height.")
compact_shortcut_buttons: list[CompactShortcutButton] = Field(default_factory=list, description="Shortcut buttons rendered in top bars")
custom_themes: list[CustomThemePreset] = Field(default_factory=list, description="Administrator-managed runtime theme presets")
@@ -1251,12 +948,6 @@ class AppearanceUpdateRequest(BaseModel):
default_skill_view: Literal["card", "list"] | None = Field(default=None)
default_agent_view: Literal["card", "list"] | None = Field(default=None)
default_layout_mode: Literal["full", "compact"] | None = Field(default=None)
default_composer_width_full: int | None = Field(default=None)
default_composer_width_compact: int | None = Field(default=None)
default_composer_height_full: int | None = Field(default=None)
default_composer_height_compact: int | None = Field(default=None)
default_agent_composer_width: int | None = Field(default=None)
default_agent_composer_height: int | None = Field(default=None)
compact_shortcut_buttons: list[CompactShortcutButton] | None = Field(default=None)
custom_themes: list[CustomThemePreset] | None = Field(default=None)
@@ -1330,12 +1021,6 @@ def _appearance_response(appearance) -> AppearanceResponse:
default_skill_view=appearance.default_skill_view,
default_agent_view=appearance.default_agent_view,
default_layout_mode=appearance.default_layout_mode,
default_composer_width_full=appearance.default_composer_width_full,
default_composer_width_compact=appearance.default_composer_width_compact,
default_composer_height_full=appearance.default_composer_height_full,
default_composer_height_compact=appearance.default_composer_height_compact,
default_agent_composer_width=appearance.default_agent_composer_width,
default_agent_composer_height=appearance.default_agent_composer_height,
compact_shortcut_buttons=sorted(
appearance.compact_shortcut_buttons,
key=lambda item: item.sort_order,
@@ -1394,17 +1079,7 @@ async def update_appearance_settings(body: AppearanceUpdateRequest, request: Req
current = load_system_settings()
fields = body.model_dump(exclude_unset=True)
update: dict[str, object] = {}
for key in (
"default_skill_view",
"default_agent_view",
"default_layout_mode",
"default_composer_width_full",
"default_composer_width_compact",
"default_composer_height_full",
"default_composer_height_compact",
"default_agent_composer_width",
"default_agent_composer_height",
):
for key in ("default_skill_view", "default_agent_view", "default_layout_mode"):
if key in fields and fields[key] is not None:
update[key] = fields[key]
if "compact_shortcut_buttons" in fields and body.compact_shortcut_buttons is not None:
@@ -8,14 +8,12 @@ without asking an administrator to tag every item.
from __future__ import annotations
import asyncio
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from app.gateway.catalog_cache import cached_catalog, catalog_key, invalidate_catalog
from app.gateway.deps import get_tag_store
from deerflow.persistence.tags import TAG_TYPES, TagStore
@@ -24,13 +22,6 @@ logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/tags", tags=["tags"])
async def _invalidate_tag_catalogues(request: Request) -> None:
await asyncio.gather(*(
invalidate_catalog(request, collection)
for collection in ("tags", "light_apps", "agents", "skills", "recommended_questions")
))
def _validate_tag_type(tag_type: str) -> str:
if tag_type not in TAG_TYPES:
raise HTTPException(status_code=422, detail=f"Invalid tag type '{tag_type}'. Allowed: {', '.join(TAG_TYPES)}")
@@ -77,19 +68,14 @@ class TagAssignmentRequest(BaseModel):
@router.get("", response_model=TagListResponse, summary="List Tags")
async def list_tags(
request: Request,
tag_type: str | None = Query(default=None, description="Filter by tag type"),
store: TagStore = Depends(get_tag_store),
) -> TagListResponse:
if tag_type is not None:
_validate_tag_type(tag_type)
async def load() -> TagListResponse:
try:
records = await store.list_tags(tag_type)
return TagListResponse(tags=[TagResponse(**r) for r in records])
try:
return await cached_catalog(request, "tags", catalog_key("list", tag_type), load, TagListResponse)
except Exception as e:
logger.error("Failed to list tags: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to list tags: {e}")
@@ -98,7 +84,6 @@ async def list_tags(
@router.post("", response_model=TagResponse, status_code=201, summary="Create Tag")
async def create_tag(
body: TagCreateRequest,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagResponse:
_validate_tag_type(body.tag_type)
@@ -112,7 +97,6 @@ async def create_tag(
"description": body.description,
}
)
await _invalidate_tag_catalogues(request)
return TagResponse(**record)
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@@ -127,7 +111,6 @@ async def create_tag(
async def update_tag(
tag_id: str,
body: TagUpdateRequest,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagResponse:
update_data = body.model_dump(exclude_unset=True)
@@ -137,7 +120,6 @@ async def update_tag(
record = await store.update_tag(tag_id, update_data)
if record is None:
raise HTTPException(status_code=404, detail=f"Tag '{tag_id}' not found")
await _invalidate_tag_catalogues(request)
return TagResponse(**record)
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@@ -151,14 +133,12 @@ async def update_tag(
@router.delete("/{tag_id}", status_code=204, summary="Delete Tag")
async def delete_tag(
tag_id: str,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> None:
try:
deleted = await store.delete_tag(tag_id)
if not deleted:
raise HTTPException(status_code=404, detail=f"Tag '{tag_id}' not found")
await _invalidate_tag_catalogues(request)
except HTTPException:
raise
except Exception as e:
@@ -177,19 +157,12 @@ async def delete_tag(
async def get_resource_tags(
target_type: str,
target_id: str,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagListResponse:
_validate_tag_type(target_type)
async def load() -> TagListResponse:
try:
records = await store.get_target_tags(target_type, target_id)
return TagListResponse(tags=[TagResponse(**r) for r in records])
try:
return await cached_catalog(
request, "tags", catalog_key("assignment", target_type, target_id), load, TagListResponse,
)
except Exception as e:
logger.error("Failed to get resource tags: %s", e, exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to get resource tags: {e}")
@@ -204,13 +177,11 @@ async def set_resource_tags(
target_type: str,
target_id: str,
body: TagAssignmentRequest,
request: Request,
store: TagStore = Depends(get_tag_store),
) -> TagListResponse:
_validate_tag_type(target_type)
try:
records = await store.set_target_tags(target_type, target_id, body.tag_ids)
await _invalidate_tag_catalogues(request)
return TagListResponse(tags=[TagResponse(**r) for r in records])
except ValueError as e:
raise HTTPException(status_code=422, detail=str(e))
@@ -45,7 +45,6 @@ from app.gateway.routers.multi_agent import (
)
from app.gateway.routers.task_analysis import _store, _task_or_404
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
from deerflow.runtime.task_analysis.thread_metadata import task_analysis_thread_metadata
logger = logging.getLogger(__name__)
@@ -133,14 +132,9 @@ async def qa_init(request: Request, task_id: str) -> QaInitResponse:
agent_meta = await _get_agents_batch_direct(request, agent_ids)
# 并行建席位 thread + 总控 thread。
seat_threads = await _gather_thread_ids(request, task, agent_ids)
seat_threads = await _gather_thread_ids(request, agent_ids)
coord_thread = await _create_thread_direct(
request,
metadata=task_analysis_thread_metadata(
task,
thread_type="task-analysis-qa",
role="coordinator",
),
request, metadata={"thread_type": "task-analysis-qa", "system": True}
)
thread_ids: dict[str, str] = {**seat_threads, COORDINATOR_AGENT_ID: coord_thread}
@@ -185,22 +179,12 @@ async def qa_init(request: Request, task_id: str) -> QaInitResponse:
)
async def _gather_thread_ids(
request: Request,
task: dict[str, Any],
agent_ids: list[str],
) -> dict[str, str]:
async def _gather_thread_ids(request: Request, agent_ids: list[str]) -> dict[str, str]:
import asyncio
async def _one(agent_id: str) -> tuple[str, str]:
thread_id = await _create_thread_direct(
request,
metadata=task_analysis_thread_metadata(
task,
thread_type="task-analysis-qa",
role="seat",
seat_id=agent_id,
),
request, metadata={"thread_type": "task-analysis-qa", "system": True}
)
return agent_id, thread_id
@@ -26,7 +26,6 @@ from datetime import datetime
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.catalog_cache import cached_catalog, invalidate_catalog
from app.gateway.deps import get_current_user, get_optional_user_from_request
from deerflow.persistence.task_buttons import TaskButtonStore
@@ -111,12 +110,8 @@ async def _require_admin(request: Request) -> None:
@router.get("", response_model=TaskButtonListResponse)
async def list_task_buttons(request: Request) -> TaskButtonListResponse:
store = _get_store(request)
async def load() -> TaskButtonListResponse:
rows = await store.list_buttons()
return TaskButtonListResponse(buttons=[TaskButtonResponse(**r) for r in rows])
return await cached_catalog(request, "task_buttons", "all", load, TaskButtonListResponse)
rows = await store.list_buttons()
return TaskButtonListResponse(buttons=[TaskButtonResponse(**r) for r in rows])
@router.put("", response_model=TaskButtonListResponse)
@@ -128,5 +123,4 @@ async def replace_task_buttons(request: Request, body: TaskButtonReplaceRequest)
[b.model_dump() for b in body.buttons],
updated_by=user_id,
)
await invalidate_catalog(request, "task_buttons")
return TaskButtonListResponse(buttons=[TaskButtonResponse(**r) for r in rows])
@@ -1,190 +0,0 @@
"""Task-detail API for externally initiated single-agent deep links.
The detail snapshot is deliberately owned by DeerFlow. The embedding system
writes it once, then the browser only reads it after login; no frontend call is
made back to the embedding system for task content.
"""
from __future__ import annotations
import json
from typing import Any, Literal
from fastapi import APIRouter, HTTPException, Path, Query, Request
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_validator
from app.gateway.deps import get_task_deeplink_detail_store
router = APIRouter(prefix="/api/task-deeplink-details", tags=["task-deeplink-details"])
_B5_DETAIL_TYPE = "b5-1"
_MAX_EXTRA_BYTES = 64 * 1024
_MAX_SUB_TYPE = 64
class TaskDeeplinkDetailUpsertRequest(BaseModel):
"""Complete task snapshot supplied by the embedding system.
``extra`` is intentionally an object, not a bag of top-level fields, so
the stable contract stays small while integrators can attach their own
structured fields. ``type`` is the business type (the deep-link
``?type=``): the same task id may hold one snapshot per type; omitting it
addresses the legacy no-type partition.
"""
model_config = ConfigDict(populate_by_name=True)
task_name: str = Field(
min_length=1,
max_length=512,
validation_alias=AliasChoices("taskName", "task_name"),
)
task_content: str = Field(
min_length=1,
max_length=20_000,
validation_alias=AliasChoices("taskContent", "task_content"),
)
overview: str = Field(default="", max_length=4_000)
detail_sub_type: str = Field(
default="",
max_length=_MAX_SUB_TYPE,
validation_alias=AliasChoices("type", "taskType", "sub_type"),
)
extra: dict[str, Any] = Field(default_factory=dict)
@field_validator("task_name", "task_content", "overview", "detail_sub_type")
@classmethod
def strip_text(cls, value: str) -> str:
return value.strip()
@field_validator("task_name", "task_content")
@classmethod
def require_nonblank_text(cls, value: str) -> str:
if not value:
raise ValueError("text must not be blank")
return value
@field_validator("extra")
@classmethod
def require_bounded_json_object(cls, value: dict[str, Any]) -> dict[str, Any]:
try:
encoded = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
except (TypeError, ValueError) as exc:
raise ValueError("extra must be JSON serializable") from exc
if len(encoded.encode("utf-8")) > _MAX_EXTRA_BYTES:
raise ValueError(f"extra must not exceed {_MAX_EXTRA_BYTES} bytes")
return value
class TaskDeeplinkDetailResponse(BaseModel):
model_config = ConfigDict(populate_by_name=True)
detail_type: Literal["b5-1"] = Field(serialization_alias="type")
detail_id: str = Field(serialization_alias="id")
detail_sub_type: str = Field(default="", serialization_alias="taskType")
task_name: str = Field(serialization_alias="taskName")
task_content: str = Field(serialization_alias="taskContent")
overview: str = ""
extra: dict[str, Any] = Field(default_factory=dict)
version: int
updated_at: str | None = Field(default=None, serialization_alias="updatedAt")
class TaskDeeplinkDetailEnvelope(BaseModel):
data: TaskDeeplinkDetailResponse
class TaskDeeplinkDetailUpsertEnvelope(TaskDeeplinkDetailEnvelope):
created: bool
changed: bool
def _detail_response(row: dict[str, Any]) -> TaskDeeplinkDetailResponse:
return TaskDeeplinkDetailResponse(
detail_type=row["detail_type"],
detail_id=row["detail_id"],
detail_sub_type=str(row.get("sub_type") or ""),
task_name=row["task_name"],
task_content=row["task_content"],
overview=row.get("overview") or "",
extra=row.get("extra") or {},
version=int(row.get("version") or 1),
updated_at=str(row["updated_at"]) if row.get("updated_at") else None,
)
def _normalized_detail_id(detail_id: str) -> str:
normalized = detail_id.strip()
if not normalized:
raise HTTPException(status_code=422, detail="detail id must not be blank")
return normalized
def _writer_id(request: Request) -> str:
"""Audit identity for a snapshot write.
The PUT endpoint is credential-free (integration systems write before the
user's browser is involved), so an unauthenticated caller is recorded as
the external integrator; a logged-in browser write keeps the user id.
"""
user = getattr(request.state, "user", None)
if user is not None:
return str(getattr(user, "id", "system"))
return "external:task-deeplink"
@router.get("/{detail_type}/{detail_id}", response_model=TaskDeeplinkDetailEnvelope)
async def get_task_deeplink_detail(
request: Request,
detail_type: Literal["b5-1"],
detail_id: str = Path(min_length=1, max_length=128),
type: str = Query(default="", max_length=_MAX_SUB_TYPE),
) -> TaskDeeplinkDetailEnvelope:
"""Read one persisted snapshot for a post-login deep-link workspace.
``?type=`` selects the business-type partition written via the PUT body's
``type`` field; omitted means the legacy no-type partition.
"""
store = get_task_deeplink_detail_store(request)
row = await store.get_detail(
detail_type,
_normalized_detail_id(detail_id),
sub_type=type.strip(),
)
if row is None:
raise HTTPException(status_code=404, detail="Task deep-link detail not found")
return TaskDeeplinkDetailEnvelope(data=_detail_response(row))
@router.put("/{detail_type}/{detail_id}", response_model=TaskDeeplinkDetailUpsertEnvelope)
async def upsert_task_deeplink_detail(
request: Request,
body: TaskDeeplinkDetailUpsertRequest,
detail_type: Literal["b5-1"],
detail_id: str = Path(min_length=1, max_length=128),
) -> TaskDeeplinkDetailUpsertEnvelope:
"""Idempotently create or update a snapshot before ``goPath=b5-1``.
Open write endpoint: no login session or integration key is required
(AuthMiddleware/CSRF let the PUT through); the writer is only recorded
for audit. Reads still require a normal authenticated session.
"""
store = get_task_deeplink_detail_store(request)
row, created, changed = await store.upsert_detail(
detail_type,
_normalized_detail_id(detail_id),
sub_type=body.detail_sub_type,
task_name=body.task_name,
task_content=body.task_content,
overview=body.overview,
extra=body.extra,
updated_by=_writer_id(request),
)
return TaskDeeplinkDetailUpsertEnvelope(
data=_detail_response(row),
created=created,
changed=changed,
)
@@ -1,63 +0,0 @@
"""Fast taskId + goPath history lookup for single-agent deep-link workspaces.
Messages and execution state remain canonical LangGraph threads. This API
reads only the dedicated ``task_deeplink_threads`` cards and never joins or
scans the generic ``threads_meta`` catalogue.
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel
from app.gateway.deps import get_task_deeplink_thread_store
from app.gateway.routers.threads import ThreadResponse
from deerflow.persistence.task_deeplink_threads.base import is_valid_task_deeplink_go_path
from deerflow.utils.time import coerce_iso
router = APIRouter(
prefix="/api/task-deeplink-threads",
tags=["task-deeplink-threads"],
)
class TaskDeeplinkThreadListResponse(BaseModel):
threads: list[ThreadResponse]
@router.get("", response_model=TaskDeeplinkThreadListResponse)
async def list_task_deeplink_threads(
request: Request,
task_id: str = Query(min_length=1, max_length=128),
go_path: str = Query(min_length=1, max_length=32),
limit: int = Query(default=100, ge=1, le=500),
# b5-1 ?type= partition. Provided (even empty) → only that sub-type;
# omitted → every sub-type of the task/goPath partition (legacy callers).
sub_type: str | None = Query(default=None, max_length=64),
) -> TaskDeeplinkThreadListResponse:
"""Return one shared deep-link history partition, newest first."""
normalized_go_path = go_path.strip().lower()
if not is_valid_task_deeplink_go_path(normalized_go_path):
raise HTTPException(status_code=422, detail="Invalid task deep-link goPath")
store = get_task_deeplink_thread_store(request)
rows = await store.list_threads(
task_id.strip(),
normalized_go_path,
limit=limit,
sub_type=sub_type.strip() if sub_type is not None else None,
)
return TaskDeeplinkThreadListResponse(
threads=[
ThreadResponse(
thread_id=row["thread_id"],
status=row.get("status", "idle"),
created_at=coerce_iso(row.get("created_at", "")),
updated_at=coerce_iso(row.get("updated_at", "")),
metadata=row.get("metadata", {}),
values=row.get("values", {}),
interrupts=row.get("interrupts", {}),
)
for row in rows
]
)
@@ -177,7 +177,7 @@ async def get_run(thread_id: str, run_id: str, request: Request) -> RunResponse:
if record is None or record.thread_id != thread_id:
# In-memory miss: fall back to the persistent store so a run still
# resolves after a Gateway restart instead of 404ing.
record = await run_mgr.get_persisted(run_id, authorized_thread_id=thread_id)
record = await run_mgr.get_persisted(run_id)
if record is None or record.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return _record_to_response(record)
@@ -242,7 +242,7 @@ async def join_run(thread_id: str, run_id: str, request: Request) -> StreamingRe
# means the Gateway restarted (or the finished run was cleaned up). There
# is nothing to reconnect to, so stream a clean terminal end instead of a
# 404 — the SDK then clears its reconnect key and renders history.
persisted = await run_mgr.get_persisted(run_id, authorized_thread_id=thread_id)
persisted = await run_mgr.get_persisted(run_id)
if persisted is None or persisted.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
@@ -280,7 +280,7 @@ async def stream_existing_run(
# from before a restart (or already finished) — nothing to join or
# cancel, so emit a clean terminal end so the SDK stops reconnecting
# instead of 404ing. A truly unknown run still 404s.
persisted = await run_mgr.get_persisted(run_id, authorized_thread_id=thread_id)
persisted = await run_mgr.get_persisted(run_id)
if persisted is None or persisted.thread_id != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
return StreamingResponse(
@@ -17,7 +17,6 @@ import json
import logging
import os
import uuid
from contextlib import aclosing
from pathlib import Path
from typing import Any
@@ -27,7 +26,7 @@ from pydantic import BaseModel, Field, field_validator
from sqlalchemy import select
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_thread_store
from app.gateway.deps import get_checkpointer
from app.gateway.llmwiki_deposit import delete_conversation_deposit_documents
from app.gateway.operation_timing import OperationTiming
from app.gateway.utils import sanitize_log_param
@@ -62,9 +61,7 @@ _RAG_DEBUG_LOGGER_NAME = "rag_citation_debug"
# owner identity through the API surface. Defense-in-depth — the
# row-level invariant is still ``threads_meta.user_id`` populated from
# the auth contextvar; this list closes the metadata-blob echo gap.
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset(
{"owner_id", "user_id", "consumer_values_save_called_at"}
)
_SERVER_RESERVED_METADATA_KEYS: frozenset[str] = frozenset({"owner_id", "user_id"})
def _strip_reserved_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
@@ -180,7 +177,6 @@ class ThreadCreateRequest(BaseModel):
thread_id: str | None = Field(default=None, description="Optional thread ID (auto-generated if omitted)")
assistant_id: str | None = Field(default=None, description="Associate thread with an assistant")
metadata: dict[str, Any] = Field(default_factory=dict, description="Initial metadata")
initialize_checkpoint: bool = Field(default=False, description="Compatibility option: immediately write an empty checkpoint instead of initializing on first run")
_strip_reserved = field_validator("metadata")(classmethod(lambda cls, v: _strip_reserved_metadata(v)))
@@ -208,19 +204,6 @@ class ThreadStateResponse(BaseModel):
tasks: list[dict[str, Any]] = Field(default_factory=list, description="Interrupted task details")
class ThreadTitleStateResponse(BaseModel):
"""Lightweight title refresh status backed only by thread metadata."""
title: str = ""
title_provisional: bool | None = None
class ConsumerValuesSaveClaimResponse(BaseModel):
"""Whether the caller must backfill Consumer values for this thread."""
should_save: bool
class ThreadMessagesResponse(BaseModel):
"""A page of the latest thread message state.
@@ -877,13 +860,13 @@ async def delete_thread_data(thread_id: str, request: Request) -> ThreadDeleteRe
async def create_thread(body: ThreadCreateRequest, request: Request, response: Response = None) -> ThreadResponse: # type: ignore[assignment]
"""Create a new thread.
Writes thread metadata immediately; the first graph run initializes state.
Legacy callers can explicitly request an immediate empty checkpoint.
Writes a thread_meta record (so the thread appears in /threads/search)
and an empty checkpoint (so state endpoints work immediately).
Idempotent: returns the existing record when ``thread_id`` already exists.
"""
from app.gateway.deps import get_thread_store
checkpointer = get_checkpointer(request) if body.initialize_checkpoint else None
checkpointer = get_checkpointer(request)
thread_store = get_thread_store(request)
thread_id = body.thread_id or str(uuid.uuid4())
now = now_iso()
@@ -896,13 +879,12 @@ async def create_thread(body: ThreadCreateRequest, request: Request, response: R
"thread_create",
thread_id=sanitize_log_param(thread_id),
metadata_store=type(thread_store).__name__,
checkpointer=type(checkpointer).__name__ if checkpointer is not None else "deferred",
checkpointer=type(checkpointer).__name__,
) as timing:
# Idempotency: return existing record when already present
with timing.phase("metadata_lookup"):
existing_record = await thread_store.get(thread_id)
if existing_record is not None:
_mark_primary_only_thread(request, thread_id)
return ThreadResponse(
thread_id=thread_id,
status=existing_record.get("status", "idle"),
@@ -923,12 +905,8 @@ async def create_thread(body: ThreadCreateRequest, request: Request, response: R
logger.exception("Failed to write thread_meta for %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to create thread")
if not body.initialize_checkpoint:
_mark_primary_only_thread(request, thread_id)
return ThreadResponse(thread_id=thread_id, created_at=now, updated_at=now, metadata=body.metadata)
# Explicit compatibility path only. Never schedule this write in the
# background: a late empty checkpoint could replace the first run's head.
# Keep the initial checkpoint: state endpoints must work immediately.
# Its timing includes saver lock/pool waits as well as the DB write.
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
ckpt_metadata = {
@@ -1001,11 +979,9 @@ async def search_threads(body: ThreadSearchRequest, request: Request, response:
# writing setup wizards, …) carry ``metadata.system=true`` and/or a known
# ``thread_type`` — hide them from the default conversation list so they
# don't crowd out real chats. A caller that explicitly filters on
# ``thread_type``/``system`` (e.g. the studio notebook list), or explicitly
# filters one task via ``taskId``, opts out of the exclusion.
explicit_system_filter = bool(body.metadata) and any(
key in body.metadata for key in ("thread_type", "system", "taskId")
)
# ``thread_type``/``system`` (e.g. the studio notebook list) opts out of
# the exclusion.
explicit_system_filter = bool(body.metadata) and ("thread_type" in body.metadata or "system" in body.metadata)
with OperationTiming(response, logger, "thread_search", metadata_store=type(repo).__name__) as timing:
with timing.phase("metadata_search"):
rows = await repo.search(
@@ -1049,9 +1025,7 @@ async def count_threads(body: ThreadSearchRequest, request: Request) -> ThreadCo
from app.gateway.deps import get_thread_store
repo = get_thread_store(request)
explicit_system_filter = bool(body.metadata) and any(
key in body.metadata for key in ("thread_type", "system", "taskId")
)
explicit_system_filter = bool(body.metadata) and ("thread_type" in body.metadata or "system" in body.metadata)
total = await repo.count(
metadata=body.metadata or None,
status=body.status,
@@ -1090,67 +1064,23 @@ async def patch_thread(thread_id: str, body: ThreadPatchRequest, request: Reques
)
@router.post("/{thread_id}/consumer-values-save-claim", response_model=ConsumerValuesSaveClaimResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def claim_consumer_values_save(
thread_id: str,
request: Request,
) -> ConsumerValuesSaveClaimResponse:
"""Claim the one-time Consumer backfill for a pre-existing conversation.
The timestamp records that the external save request was *attempted*, not
that Consumer accepted it. This matches the non-blocking UI contract: a
Consumer outage must never be surfaced to or block the chat user.
"""
thread_store = get_thread_store(request)
record = await thread_store.get(thread_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
metadata = record.get("metadata") or {}
if metadata.get("consumer_values_save_called_at"):
return ConsumerValuesSaveClaimResponse(should_save=False)
try:
await thread_store.update_metadata(
thread_id,
{"consumer_values_save_called_at": now_iso()},
)
except Exception:
logger.exception("Failed to claim Consumer values save for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to record Consumer values save")
return ConsumerValuesSaveClaimResponse(should_save=True)
@router.get("/{thread_id}", response_model=ThreadResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request, include_state: bool = False) -> ThreadResponse:
async def get_thread(thread_id: str, request: Request) -> ThreadResponse:
"""Get thread info.
Metadata reads use only the business database. Clients that need channel
values can opt in with include_state, or use /state. Legacy threads without
a metadata row retain the checkpoint fallback after authorization.
Reads metadata from the ThreadMetaStore and derives the accurate
execution status from the checkpointer. Falls back to the checkpointer
alone for threads that pre-date ThreadMetaStore adoption (backward compat).
"""
from app.gateway.deps import get_thread_store
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
# The owner-check decorator already authorized this read, including shared
# task threads. A metadata-only shared thread has no checkpoint fallback.
record: dict | None = await thread_store.get(thread_id, user_id=None)
if record is not None and not include_state:
return ThreadResponse(
thread_id=thread_id,
status="busy" if record.get("status") == "running" else record.get("status", "idle"),
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
metadata=record.get("metadata", {}),
)
record: dict | None = await thread_store.get(thread_id)
# Derive accurate status from the checkpointer
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
@@ -1187,91 +1117,61 @@ async def get_thread(thread_id: str, request: Request, include_state: bool = Fal
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
metadata=record.get("metadata", {}),
values=serialize_channel_values(channel_values, slim_display=True),
values=serialize_channel_values(channel_values),
)
# ---------------------------------------------------------------------------
@router.get("/{thread_id}/title-state", response_model=ThreadTitleStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_title_state(thread_id: str, request: Request) -> ThreadTitleStateResponse:
"""Read only title metadata; do not load/serialize a checkpoint or messages."""
record = await get_thread_store(request).get(thread_id)
if record is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
metadata = record.get("metadata") or {}
provisional = metadata.get("_deerflow_title_provisional")
return ThreadTitleStateResponse(
title=str(record.get("display_name") or ""),
title_provisional=provisional if isinstance(provisional, bool) else None,
)
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
@require_permission("threads", "read", owner_check=True)
async def get_thread_state(thread_id: str, request: Request, response: Response = None) -> ThreadStateResponse: # type: ignore[assignment]
async def get_thread_state(thread_id: str, request: Request) -> ThreadStateResponse:
"""Get the latest state snapshot for a thread.
Channel values are serialized to ensure LangChain message objects
are converted to JSON-safe dicts. Oversized tool/file bodies are
truncated for this hydrate snapshot; the checkpoint itself is unchanged.
are converted to JSON-safe dicts.
"""
checkpointer = get_checkpointer(request)
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
with OperationTiming(
response,
logger,
"thread_state",
thread_id=sanitize_log_param(thread_id),
checkpointer=type(checkpointer).__name__,
) as timing:
try:
with timing.phase("checkpoint_load"):
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get state for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread state")
try:
checkpoint_tuple = await checkpointer.aget_tuple(config)
except Exception:
logger.exception("Failed to get state for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread state")
if checkpoint_tuple is None:
from app.gateway.deps import get_thread_store
if checkpoint_tuple is None:
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
record = await get_thread_store(request).get(thread_id, user_id=None)
if record is not None:
return ThreadStateResponse(metadata=record.get("metadata", {}))
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint_id = None
ckpt_config = getattr(checkpoint_tuple, "config", {})
if ckpt_config:
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id")
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint_id = None
ckpt_config = getattr(checkpoint_tuple, "config", {})
if ckpt_config:
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id")
channel_values = checkpoint.get("channel_values", {})
channel_values = checkpoint.get("channel_values", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
parent_checkpoint_id = None
if parent_config:
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
parent_config = getattr(checkpoint_tuple, "parent_config", None)
parent_checkpoint_id = None
if parent_config:
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
tasks = [{"id": getattr(t, "id", ""), "name": getattr(t, "name", "")} for t in tasks_raw]
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
tasks = [{"id": getattr(t, "id", ""), "name": getattr(t, "name", "")} for t in tasks_raw]
values = serialize_channel_values(channel_values)
with timing.phase("serialize"):
values = serialize_channel_values(channel_values, slim_display=True)
return ThreadStateResponse(
values=values,
next=next_tasks,
metadata=metadata,
checkpoint={"id": checkpoint_id, "ts": coerce_iso(metadata.get("created_at", ""))},
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
return ThreadStateResponse(
values=values,
next=next_tasks,
metadata=metadata,
checkpoint={"id": checkpoint_id, "ts": coerce_iso(metadata.get("created_at", ""))},
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_checkpoint_id,
created_at=coerce_iso(metadata.get("created_at", "")),
tasks=tasks,
)
@router.get("/{thread_id}/reference-batches", response_model=ReferenceBatchesResponse)
@@ -2008,50 +1908,9 @@ async def edit_thread_message(
}
def _mark_primary_only_thread(request: Request, thread_id: str) -> None:
"""Tell a dual-read checkpointer not to scan legacy SQLite for this thread."""
saver = getattr(request.app.state, "checkpointer", None)
remember = getattr(saver, "remember_primary_thread", None)
if callable(remember):
remember(thread_id)
def _history_entry_from_tuple(checkpoint_tuple: Any, *, include_values: bool) -> HistoryEntry:
"""Build one history card. Only the latest head carries serialized values."""
ckpt_config = getattr(checkpoint_tuple, "config", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id", "")
parent_id = parent_config.get("configurable", {}).get("checkpoint_id") if parent_config else None
channel_values = checkpoint.get("channel_values", {})
values: dict[str, Any] = serialize_channel_values(channel_values, slim_display=True) if include_values else {}
if title := channel_values.get("title"):
values["title"] = title
if thread_data := channel_values.get("thread_data"):
values["thread_data"] = thread_data
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents")}
if "step" in metadata:
user_meta["step"] = metadata["step"]
return HistoryEntry(
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_id,
metadata=user_meta,
values=values,
created_at=coerce_iso(metadata.get("created_at", "")),
next=[t.name for t in tasks_raw if hasattr(t, "name")],
)
@router.post("/{thread_id}/history", response_model=list[HistoryEntry])
@require_permission("threads", "read", owner_check=True)
async def get_thread_history(
thread_id: str,
body: ThreadHistoryRequest,
request: Request,
response: Response = None, # type: ignore[assignment]
) -> list[HistoryEntry]:
async def get_thread_history(thread_id: str, body: ThreadHistoryRequest, request: Request) -> list[HistoryEntry]:
"""Get checkpoint history for a thread.
Messages are read from the checkpointer's channel values (the
@@ -2059,37 +1918,65 @@ async def get_thread_history(
:func:`~deerflow.runtime.serialization.serialize_channel_values`.
Only the latest (first) checkpoint carries the ``messages`` key to
avoid duplicating them across every entry.
The chat UI always asks for ``limit=1``. That path uses ``aget_tuple``
instead of walking the checkpoint iterator.
"""
checkpointer = get_checkpointer(request)
config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
config: dict[str, Any] = {"configurable": {"thread_id": thread_id}}
if body.before:
config["configurable"]["checkpoint_id"] = body.before
with OperationTiming(
response,
logger,
"thread_history",
thread_id=sanitize_log_param(thread_id),
checkpointer=type(checkpointer).__name__,
limit=str(body.limit),
) as timing:
try:
with timing.phase("checkpoint_load"):
if body.limit == 1 and not body.before:
latest = await checkpointer.aget_tuple(config)
checkpoints = [latest] if latest is not None else []
else:
async with aclosing(checkpointer.alist(config, limit=body.limit)) as history:
checkpoints = [checkpoint async for checkpoint in history]
with timing.phase("serialize"):
return [
_history_entry_from_tuple(checkpoint_tuple, include_values=index == 0)
for index, checkpoint_tuple in enumerate(checkpoints)
]
except Exception:
logger.exception("Failed to get history for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread history")
entries: list[HistoryEntry] = []
is_latest_checkpoint = True
try:
async for checkpoint_tuple in checkpointer.alist(config, limit=body.limit):
ckpt_config = getattr(checkpoint_tuple, "config", {})
parent_config = getattr(checkpoint_tuple, "parent_config", None)
metadata = getattr(checkpoint_tuple, "metadata", {}) or {}
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
checkpoint_id = ckpt_config.get("configurable", {}).get("checkpoint_id", "")
parent_id = None
if parent_config:
parent_id = parent_config.get("configurable", {}).get("checkpoint_id")
channel_values = checkpoint.get("channel_values", {})
# Build values from checkpoint channel_values
values: dict[str, Any] = {}
if title := channel_values.get("title"):
values["title"] = title
if thread_data := channel_values.get("thread_data"):
values["thread_data"] = thread_data
# Attach messages only to the latest checkpoint entry.
if is_latest_checkpoint:
messages = channel_values.get("messages")
if messages:
values["messages"] = serialize_channel_values({"messages": messages}).get("messages", [])
is_latest_checkpoint = False
# Derive next tasks
tasks_raw = getattr(checkpoint_tuple, "tasks", []) or []
next_tasks = [t.name for t in tasks_raw if hasattr(t, "name")]
# Strip LangGraph internal keys from metadata
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents")}
# Keep step for ordering context
if "step" in metadata:
user_meta["step"] = metadata["step"]
entries.append(
HistoryEntry(
checkpoint_id=checkpoint_id,
parent_checkpoint_id=parent_id,
metadata=user_meta,
values=values,
created_at=coerce_iso(metadata.get("created_at", "")),
next=next_tasks,
)
)
except Exception:
logger.exception("Failed to get history for thread %s", sanitize_log_param(thread_id))
raise HTTPException(status_code=500, detail="Failed to get thread history")
return entries
@@ -19,7 +19,6 @@ from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.admin_query_window import resolve_admin_query_window
from app.gateway.deps import get_local_provider, get_optional_user_from_request, get_tool_metrics_store
from deerflow.persistence.tool_metrics.base import ToolMetricsQuery, ToolMetricsStore
@@ -86,11 +85,9 @@ def _require_store(request: Request) -> ToolMetricsStore:
return store
def _parse_dt(value: str | datetime | None) -> datetime | None:
def _parse_dt(value: str | None) -> datetime | None:
if not value:
return None
if isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
@@ -150,8 +147,8 @@ def _build_query(
skill_name: str | None = None,
status: str | None,
error_category: str | None,
since: str | datetime | None,
until: str | datetime | None,
since: str | None,
until: str | None,
limit: int,
offset: int,
) -> ToolMetricsQuery:
@@ -193,7 +190,6 @@ async def list_tool_metrics(
if resolved is None:
return ToolMetricsListResponse(items=[], total=0, limit=limit, offset=offset)
user_id = resolved
since_dt, until_dt, _range_days = resolve_admin_query_window(since=since, until=until)
query = _build_query(
user_id=user_id,
tool_name=tool_name,
@@ -201,8 +197,8 @@ async def list_tool_metrics(
skill_name=skill_name,
status=status,
error_category=error_category,
since=since_dt,
until=until_dt,
since=since,
until=until,
limit=limit,
offset=offset,
)
@@ -254,7 +250,6 @@ async def tool_metrics_summary(
return ToolMetricsSummaryResponse(total=0, error_total=0, by_category={}, by_skill=[])
user_id = resolved
skill_only = _kind_is_skill(kind)
since_dt, until_dt, _range_days = resolve_admin_query_window(since=since, until=until)
def _q(*, status: str | None = None, error_category: str | None = None) -> ToolMetricsQuery:
return _build_query(
@@ -264,8 +259,8 @@ async def tool_metrics_summary(
skill_name=skill_name,
status=status,
error_category=error_category,
since=since_dt,
until=until_dt,
since=since,
until=until,
limit=1,
offset=0,
)
@@ -310,7 +305,6 @@ async def export_tool_metrics(
if not user_id and user_name:
resolved = await _resolve_user_name_to_id(user_name)
user_id = resolved if resolved is not None else "__no_such_user__"
since_dt, until_dt, _range_days = resolve_admin_query_window(since=since, until=until)
try:
from openpyxl import Workbook
@@ -324,8 +318,8 @@ async def export_tool_metrics(
skill_name=skill_name,
status=status,
error_category=error_category,
since=since_dt,
until=until_dt,
since=since,
until=until,
limit=1,
offset=0,
)
@@ -1,14 +1,11 @@
"""Upload router for handling file uploads."""
import asyncio
import logging
import os
import shutil
import stat
import tempfile
import threading
from pathlib import Path
from weakref import WeakKeyDictionary, WeakValueDictionary
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from pydantic import BaseModel, Field
@@ -36,46 +33,13 @@ from deerflow.uploads.manager import (
)
from deerflow.uploads.policy import classify_upload_filename
from deerflow.uploads.validation import UploadContentError, validate_upload_content
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown_sync
from deerflow.utils.file_io import FileIOPool, file_io
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads/{thread_id}/uploads", tags=["uploads"])
UPLOAD_CHUNK_SIZE = 256 * 1024
# Upload work must not fill the default executor used by authentication, or the
# artifact/event pools used by active conversations. Waiting batches use no worker.
_upload_io = FileIOPool(2, 2, "uploads")
_upload_locks: WeakKeyDictionary = WeakKeyDictionary()
def _upload_lock(directory: Path) -> asyncio.Lock:
locks = _upload_locks.setdefault(asyncio.get_running_loop(), WeakValueDictionary())
key = os.path.normcase(str(directory))
lock = locks.get(key)
if lock is None:
lock = asyncio.Lock()
locks[key] = lock
return lock
class _UploadCancelled(Exception):
"""Cooperative stop inside a worker; the request propagates CancelledError."""
def _check_upload_cancelled(cancelled: threading.Event | None) -> None:
if cancelled is not None and cancelled.is_set():
raise _UploadCancelled()
async def _resolve_uploads_path(thread_id: str) -> Path:
owner = await aresolve_path_user_id(thread_id)
try:
return get_paths().sandbox_uploads_dir(thread_id, user_id=owner)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
UPLOAD_CHUNK_SIZE = 8192
class UploadResponse(BaseModel):
@@ -155,37 +119,10 @@ async def _write_upload_file_with_limits(
max_total_size: int,
total_size: int,
) -> tuple[os.PathLike[str] | str, int, int]:
cancelled = threading.Event()
return await _upload_io.run_until_complete(
_copy_upload_file_with_limits,
file,
uploads_dir=uploads_dir,
display_filename=display_filename,
max_single_file_size=max_single_file_size,
max_total_size=max_total_size,
total_size=total_size,
cancelled=cancelled,
on_cancel=cancelled.set,
)
def _copy_upload_file_with_limits(
file: UploadFile,
*,
uploads_dir: os.PathLike[str] | str,
display_filename: str,
max_single_file_size: int,
max_total_size: int,
total_size: int,
cancelled: threading.Event | None = None,
) -> tuple[Path, int, int]:
"""Copy the parsed multipart spool entirely in the dedicated upload worker."""
_check_upload_cancelled(cancelled)
file_size = 0
file_path, fh = open_upload_file_no_symlink(uploads_dir, display_filename)
try:
while chunk := file.file.read(UPLOAD_CHUNK_SIZE):
_check_upload_cancelled(cancelled)
while chunk := await file.read(UPLOAD_CHUNK_SIZE):
file_size += len(chunk)
total_size += len(chunk)
if file_size > max_single_file_size:
@@ -193,7 +130,6 @@ def _copy_upload_file_with_limits(
if total_size > max_total_size:
raise HTTPException(status_code=413, detail="Total upload size too large")
fh.write(chunk)
_check_upload_cancelled(cancelled)
except Exception:
fh.close()
try:
@@ -313,56 +249,27 @@ async def upload_files(
category, max_bytes = classification
prepared.append((file, safe_filename, category, max_bytes))
# Resolve the canonical owner asynchronously before any filesystem work.
sandbox_uploads = await _resolve_uploads_path(thread_id)
cancelled = threading.Event()
# Moving work off the event loop introduces real concurrency. Serialize
# batches for the same directory so rollback cannot undo another upload.
async with _upload_lock(sandbox_uploads):
return await _upload_io.run_until_complete(
_process_upload_batch,
thread_id,
sandbox_uploads,
prepared,
claimed_names,
limits,
_auto_convert_documents_enabled(config),
cancelled,
on_cancel=cancelled.set,
)
def _process_upload_batch(
thread_id: str,
sandbox_uploads: Path,
prepared: list[tuple[UploadFile, str, str, int]],
claimed_names: set[str],
limits: UploadLimits,
auto_convert_documents: bool,
cancelled: threading.Event,
) -> UploadResponse:
"""Own the full staging/commit/rollback lifetime in one bounded worker."""
_check_upload_cancelled(cancelled)
try:
uploads_dir = ensure_uploads_dir(thread_id)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
uploaded_files: list[dict[str, str]] = []
sandbox_sync_targets: list[tuple[Path, str]] = []
total_size = 0
auto_convert_documents = _auto_convert_documents_enabled(config)
staging_dir = Path(tempfile.mkdtemp(prefix=".upload-staging-", dir=uploads_dir))
records: list[dict] = []
committed_names: list[str] | None = None
try:
for file, safe_filename, category, max_bytes in prepared:
staged_path, file_size, total_size = _copy_upload_file_with_limits(
staged_path, file_size, total_size = await _write_upload_file_with_limits(
file,
uploads_dir=staging_dir,
display_filename=safe_filename,
max_single_file_size=max_bytes,
max_total_size=limits.max_total_size,
total_size=total_size,
cancelled=cancelled,
)
file_ext = staged_path.suffix.lower()
try:
@@ -377,8 +284,7 @@ def _process_upload_batch(
"detected_type": file_ext.removeprefix("."),
}
if auto_convert_documents and file_ext in CONVERTIBLE_EXTENSIONS:
_check_upload_cancelled(cancelled)
md_path = convert_file_to_markdown_sync(staged_path)
md_path = await convert_file_to_markdown(staged_path)
if md_path:
markdown_folded = md_path.name.casefold()
if markdown_folded in claimed_names:
@@ -390,7 +296,6 @@ def _process_upload_batch(
record["markdown_file"] = md_path.name
records.append(record)
_check_upload_cancelled(cancelled)
sandbox_provider = get_sandbox_provider()
sync_to_sandbox = not _uses_thread_data_mounts(sandbox_provider)
sandbox = None
@@ -402,7 +307,6 @@ def _process_upload_batch(
commit_names = [str(record["filename"]) for record in records]
commit_names.extend(str(record["markdown_file"]) for record in records if record.get("markdown_file"))
_check_upload_cancelled(cancelled)
_commit_staged_files(staging_dir, uploads_dir, commit_names)
committed_names = commit_names
@@ -441,11 +345,9 @@ def _process_upload_batch(
if sync_to_sandbox and sandbox is not None:
for file_path, virtual_path in sandbox_sync_targets:
_check_upload_cancelled(cancelled)
_make_file_sandbox_writable(file_path)
sandbox.update_file(virtual_path, file_path.read_bytes())
_check_upload_cancelled(cancelled)
except (HTTPException, _UploadCancelled):
except HTTPException:
if committed_names is not None:
_rollback_committed_files(staging_dir, uploads_dir, committed_names)
raise
@@ -488,12 +390,6 @@ async def get_upload_limits(
@require_permission("threads", "read", owner_check=True)
async def list_uploaded_files(thread_id: str, request: Request) -> dict:
"""List all files in a thread's uploads directory."""
sandbox_uploads = await _resolve_uploads_path(thread_id)
async with _upload_lock(sandbox_uploads):
return await file_io.run_until_complete(_list_uploaded_files, thread_id, sandbox_uploads)
def _list_uploaded_files(thread_id: str, sandbox_uploads: Path) -> dict:
try:
uploads_dir = get_uploads_dir(thread_id)
except ValueError as e:
@@ -502,6 +398,7 @@ def _list_uploaded_files(thread_id: str, sandbox_uploads: Path) -> dict:
enrich_file_listing(result, thread_id)
# Gateway additionally includes the sandbox-relative path.
sandbox_uploads = get_paths().sandbox_uploads_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
for f in result["files"]:
f["path"] = str(sandbox_uploads / f["filename"])
@@ -512,12 +409,6 @@ def _list_uploaded_files(thread_id: str, sandbox_uploads: Path) -> dict:
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_uploaded_file(thread_id: str, filename: str, request: Request) -> dict:
"""Delete a file from a thread's uploads directory."""
sandbox_uploads = await _resolve_uploads_path(thread_id)
async with _upload_lock(sandbox_uploads):
return await file_io.run_until_complete(_delete_uploaded_file, thread_id, filename)
def _delete_uploaded_file(thread_id: str, filename: str) -> dict:
try:
uploads_dir = get_uploads_dir(thread_id)
except ValueError as e:
@@ -13,7 +13,6 @@ import logging
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.api_read_cache import get_cache_version, get_cached_json, set_cached_json
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
@@ -53,18 +52,10 @@ def _to_response(prefs: dict) -> UserPreferencesResponse:
@router.get("", response_model=UserPreferencesResponse)
async def get_user_preferences(request: Request) -> UserPreferencesResponse:
user_id = _current_user_id(request)
config = getattr(request.app.state, "config", None)
cache_key = f"user:{user_id}"
version = await get_cache_version(config, "user_preferences")
cached = await get_cached_json(config, "user_preferences", cache_key, version=version)
if cached is not None:
return UserPreferencesResponse.model_validate(cached)
store = _get_store(request)
user_id = _current_user_id(request)
prefs = await store.get_for_user(user_id)
response = _to_response(prefs)
await set_cached_json(config, "user_preferences", cache_key, response, expected_version=version)
return response
return _to_response(prefs)
@router.put("", response_model=UserPreferencesResponse)
@@ -13,7 +13,6 @@ from uuid import uuid4
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.api_read_cache import get_cache_version, get_cached_json, set_cached_json
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
@@ -67,18 +66,10 @@ def _get_store(request: Request):
@router.get("", response_model=UserPromptsListResponse)
async def list_user_prompts(request: Request) -> UserPromptsListResponse:
user_id = _current_user_id(request)
config = getattr(request.app.state, "config", None)
cache_key = f"user:{user_id}"
version = await get_cache_version(config, "user_prompts")
cached = await get_cached_json(config, "user_prompts", cache_key, version=version)
if cached is not None:
return UserPromptsListResponse.model_validate(cached)
store = _get_store(request)
user_id = _current_user_id(request)
rows = await store.list_for_user(user_id)
response = UserPromptsListResponse(prompts=[UserPromptResponse(**r) for r in rows])
await set_cached_json(config, "user_prompts", cache_key, response, expected_version=version)
return response
return UserPromptsListResponse(prompts=[UserPromptResponse(**r) for r in rows])
@router.post("", response_model=UserPromptResponse, status_code=201)