diff --git a/.dockerignore b/.dockerignore index e2541f15..20eab30e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,8 @@ dist # through the offline delivery package, never copied into an image layer. **/.env **/.env.* +**/.worker-pool +**/worker-control offline-backend-20260512/backend/.deer-flow offline-backend-20260512/backend/logs offline-backend-20260512/backend/docker-data diff --git a/.gitignore b/.gitignore index 5b487244..8d2eb722 100644 --- a/.gitignore +++ b/.gitignore @@ -85,11 +85,20 @@ offline-backend-20260512/backend/.deer-flow/* # Per-user skill_evolution overrides — runtime data written by the API offline-backend-20260512/backend/skill_evolution.json +# Automatic config.yaml backups written by the admin models page on every save +offline-backend-20260512/backend/.model-config-backups/ skills/ hermes-agent-main/ .gitignore offline-backend-20260512/backend/tests/* !offline-backend-20260512/backend/tests/test_large_file_safety.py +!offline-backend-20260512/backend/tests/test_account_authentication.py +# HTML 结构化模板(no.1)引擎/存储/路由回归 +!offline-backend-20260512/backend/tests/test_structured_templates.py +!offline-backend-20260512/backend/tests/test_html_runtime_addresses.py +!offline-backend-20260512/backend/tests/test_app_identity.py +!offline-backend-20260512/backend/tests/test_roundtable_room_limit.py +!offline-backend-20260512/backend/tests/test_admin_audit_retention_control.py # Seeder-owned Agent Skills are product source; keep them out of the broad # runtime `skills/` ignore above. !offline-backend-20260512/backend/app/gateway/routers/_*_seed_assets/skills/ @@ -97,12 +106,19 @@ offline-backend-20260512/backend/tests/* # Keep startup seed completeness and task-report conversion regressions versioned. !offline-backend-20260512/backend/tests/test_seed_assets.py !offline-backend-20260512/backend/tests/test_latency_diagnosis_seed.py +!offline-backend-20260512/backend/tests/test_query_slowdown_diagnosis_seed.py +!offline-backend-20260512/backend/tests/test_elasticsearch_offline_ops_seed.py !offline-backend-20260512/backend/tests/test_task_report_convert.py !offline-backend-20260512/backend/tests/test_task_deeplink_details.py !offline-backend-20260512/backend/tests/test_offline_docs.py !offline-backend-20260512/backend/tests/test_migration_compatibility.py +# Keep the admin catalog hidden-switch (agent / business chain) store regressions versioned. +!offline-backend-20260512/backend/tests/test_roundtable_chain_hidden.py +# Keep the business-chain seat enrichment (available/live_name) regressions versioned. +!offline-backend-20260512/backend/tests/test_roundtable_chain_seat_enrichment.py # Keep the dedicated taskId + goPath history Gateway contract versioned. !offline-backend-20260512/backend/tests/test_task_deeplink_threads_api.py +!offline-backend-20260512/backend/tests/test_thread_clear.py # Keep project-scoped cross-conversation memory regressions versioned. !offline-backend-20260512/backend/tests/test_analysis_project_memory.py !offline-backend-20260512/backend/tests/test_analysis_project_memory_integration.py @@ -119,6 +135,15 @@ offline-backend-20260512/backend/tests/* !offline-backend-20260512/backend/tests/test_gauss_database.py !offline-backend-20260512/backend/tests/test_llmwiki_weknora.py !offline-backend-20260512/backend/tests/test_llmwiki_knowledge_management.py +!offline-backend-20260512/backend/tests/test_llmwiki_smart_reasoning.py +!offline-backend-20260512/backend/tests/test_smart_reasoning_retrieval_middleware.py +!offline-backend-20260512/backend/tests/test_knowledge_retrieval_settings.py +!offline-backend-20260512/backend/tests/test_wiki_generation_config.py +!offline-backend-20260512/backend/tests/test_wiki_directory_constraints.py +!offline-backend-20260512/backend/tests/test_llmwiki_parent_directory_routing.py +!offline-backend-20260512/backend/tests/test_llmwiki_loading.py +!offline-backend-20260512/backend/tests/test_weknora_embed_script.py +!offline-backend-20260512/backend/tests/test_weknora_static.py !offline-backend-20260512/backend/tests/test_llmwiki_advanced_retrieval.py !offline-backend-20260512/backend/tests/test_knowledge_live_embedding.py !offline-backend-20260512/backend/tests/test_health_auth_bypass.py @@ -131,6 +156,8 @@ offline-backend-20260512/backend/tests/* # Keep the model configuration regression coverage versioned even though # this deployment repository otherwise ignores newly added backend tests. !offline-backend-20260512/backend/tests/test_model_management.py +# Keep the model-stream truncation event (persistence + classification) coverage versioned. +!offline-backend-20260512/backend/tests/test_stream_truncation.py # Keep the forced-research agent runtime gate and sandbox-policy regressions versioned. !offline-backend-20260512/backend/tests/test_forced_research_agent.py # Keep strategy-selection built-in agent seed and safety-contract regressions versioned. @@ -162,6 +189,10 @@ offline-backend-20260512/backend/tests/* # Keep database-backed upload-policy, enforcement and bounded-preview regressions versioned. !offline-backend-20260512/backend/tests/test_upload_policy.py !offline-backend-20260512/backend/tests/test_uploads_policy_enforcement.py +!offline-backend-20260512/backend/tests/test_upload_responsiveness.py +!offline-backend-20260512/backend/tests/test_conversion_responsiveness.py +!offline-backend-20260512/backend/tests/test_async_agent_entrypoints.py +!offline-backend-20260512/backend/tests/test_async_tool_initialization.py !offline-backend-20260512/backend/tests/test_artifact_preview.py # Independent enterprise-research persistence, HTTP and race regressions. !offline-backend-20260512/backend/tests/test_enterprise_research_tasks.py @@ -178,6 +209,8 @@ offline-backend-20260512/backend/tests/* !offline-backend-20260512/backend/tests/test_workflow_runs.py !offline-backend-20260512/backend/tests/test_workflow_concurrency.py !offline-backend-20260512/backend/tests/test_workflow_schema_rebuild.py +# Per-account question recommendation preferences. +!offline-backend-20260512/backend/tests/test_user_preferences.py # Writing-mode first-report pipeline enforcement regressions. !offline-backend-20260512/backend/tests/test_writing_mode_report_middleware.py !offline-backend-20260512/backend/tests/test_writing_rewrite_prompt.py @@ -205,6 +238,7 @@ git-clone/ frontend-web/.offline-pack-build/ frontend-web/.offline-extract-test/ frontend-web-node_modules-offline-*.tar.gz +/deerflow-frontend-dist-*.zip # Local dependency caches and backend scratch archives /.pnpm-store/ @@ -230,12 +264,23 @@ offline-backend-20260512/backend/skills/* offline-backend-20260512/backend/skills/public/* !offline-backend-20260512/backend/skills/public/analysis-mindmap/ !offline-backend-20260512/backend/skills/public/analysis-mindmap/SKILL.md +# Built-in skill-creator ported from bytedance/deer-flow (product source too). +!offline-backend-20260512/backend/skills/public/skill-creator/ +!offline-backend-20260512/backend/skills/public/skill-creator/** # uv lockfile (not tracked per project preference) offline-backend-20260512/backend/uv.lock !offline-backend-20260512/backend/tests/test_wiki_package_roundtrip.py !offline-backend-20260512/backend/tests/test_llmwiki_incremental_sync_scheduler.py +!offline-backend-20260512/backend/tests/test_worker_pool.py +!offline-backend-20260512/backend/tests/test_chat_step_limit.py +!offline-backend-20260512/backend/tests/test_worker_storage.py +!offline-backend-20260512/backend/tests/test_worker_discovery.py +!offline-backend-20260512/backend/tests/test_worker_diagnostics.py +!offline-backend-20260512/backend/tests/test_worker_deferred_admission.py +!offline-backend-20260512/backend/tests/test_db_loop_isolation.py +!offline-backend-20260512/backend/tests/test_chat_archive_postgres_integration.py # Local-only scratch directory (not committed) 0622/ @@ -245,3 +290,4 @@ __pycache__/ # misc .DS_Store +.worker-pool/ diff --git a/.zcode/plans/plan-session_00e04f204fab48e1d48ae6b407c045ab.md b/.zcode/plans/plan-session_00e04f204fab48e1d48ae6b407c045ab.md new file mode 100644 index 00000000..c4c955db --- /dev/null +++ b/.zcode/plans/plan-session_00e04f204fab48e1d48ae6b407c045ab.md @@ -0,0 +1,64 @@ +# 业务链选链去掉每次 `listAgents()` 查询 + +## 背景与思路 + +选链各环节展示只用到 seat 的 `agent_id + name`（快照已随链条落库，编辑器保存时刷新），额外全量拉 `/api/agents` 只为两件事：**席位存活判断**（existingIds）和**最新改名**（nameById）。存活是实时状态、无法落库保鲜，因此方案是： + +- **后端**在返回链条（list/get 等）时，用一次批量 SQL（`agent_store.list_by_ids`，与 `/api/agents` 同一套可见性过滤）给每个 seat 附带 `available: bool` + `live_name: string | null` 两个**只读计算字段**（不落库，seats JSON 存储结构零变化、零迁移）； +- **前端**选链全链路改用这两个字段，删除所有选链路径上的 `listAgents()` 调用；左栏点选一条链变为**零网络请求**（链条数据本就在内存里）；选链弹窗打开只剩 1 个 `listChains` 小请求。 + +「只映射目前展示的这些」：仅 name + 存活状态，其他智能体字段一概不存（description 等快照维持现状，运行期配置由后端按 agent_id 自取，本就不依赖前端传参）。 + +## 后端改动 + +### 1. `offline-backend-20260512/backend/app/gateway/routers/roundtable_chains.py` + +- 新增常量 `_FUNCTIONAL_AGENT_EXCLUSIONS`（`roundtable-intent` / `roundtable-recommender` / `roundtable-coordinator` / `roundtable-report` + 前缀 `roundtable-coordinator-`，与前端 `api/recommend.ts` 的 `filterRecommendCandidates` 排除集逐字对齐，保持"功能型内置智能体不可作席位"语义不变）。 +- 新增纯函数 `_compute_seat_enrichment(chains, live_records) -> None`：对每条链每个 seat 写入 `available`（live 存在且不在排除集 → True）与 `live_name`（无则 None）；配套薄异步封装 `_enrich_chain_seats(request, chains, user_id)`：收集去重小写 agent_id → `get_agent_store(request).list_by_ids(ids, user_id)` 一次查询（router 已 import `get_agent_store`，参照 `multi_agent.py` 的 `_get_agents_batch_direct` 写法）。 +- 新增**响应专用** `ChainSeatResponseSchema(ChainSeatSchema)`，加 `available: bool | None`、`live_name: str | None`；`ChainResponse.seats` 改用之。创建/更新请求 schema 不动 → 富化字段永不被持久化（避免 create/update `model_dump()` 把默认值写进 seats JSON）。 +- 在所有返回 `ChainResponse`（列表/单条/更新等）的出口调用富化。可见性语义与现状完全一致（`_listing_filter`：本人 / 内置 / 已发布、非私享、非管理员隐藏）。 + +### 2. 测试 `tests/test_roundtable_chain_seat_enrichment.py`（新建） + +镜像 `test_roundtable_chain_stages.py` 的风格：纯函数用 dict 直接测（存活/缺失/排除集/改名刷新/混合多条链）；`list_by_ids` 交互用同名小写归一化断言（stub store）。 + +## 前端改动（frontend-web） + +### 3. `src/roundtable-planning/lib/business-chain.ts` + +- `ChainSeat` 增加可选 `available?: boolean`（undefined = 旧后端未标注 → 视为存活，与现有"查询失败不判删"降级语义一致）和 `live_name?: string | null`，附注释说明来源与回退顺序。 +- `buildChainSeatSelection(chain, candidates?)`：`candidates` 改**可选**；不传时存活 = `seat.available !== false`、名称 = `live_name ?? name ?? agent_id`；传入时保持现行为（测试兼容）。 + +### 4. `src/roundtable-planning/components/BusinessChainPicker.tsx` + +加载 effect 只留 `listChains(scope)`；删 `listAgents()`、`filterRecommendCandidates` 及 `existingIds`/`nameById` state——存活判断用 `seat.available !== false`，tag 名称用 `live_name ?? seat.name ?? agent_id`；已删置灰/删除线/「存活不足 2 个」等 UI 保持不变（改由 `available` 驱动）。`handleUse` 的 `nameOf` 同步改。 + +### 5. `src/roundtable-planning/components/BusinessChainListDialog.tsx` + +`BusinessChainListBody` 删除挂载时的 `listAgents()` effect 与 `nameById` map；`ChainSeatsFlow` 的 seatTag 标签改 `seat.live_name ?? seat.name ?? agentId`。 + +### 6. `src/roundtable-planning/pages/RoundtablePlanningPage.tsx`（3 处调用点） + +- **L486 固定链 effect（8BF 等）**：`Promise.all([getChain, listAgents])` → 仅 `getChain`；席位不足提示改为按 `seat.available === false` 计算（文案不变）。 +- **L1356 `handleSelectChainFromSidebar`（左栏点选，本次体验痛点核心）**：删 `listAgents()`，直接 `buildChainSeatSelection(chain)` —— 点选零请求。 +- **L1503 C3 深链自动选链**：删 `listAgents()` 分支。 +- 清理不再使用的 `listAgents` / `filterRecommendCandidates` import（编辑器页与「智能推荐」tab 的 `listAgents` 是选智能体池所必需，保留不动）。 + +### 7. 测试 `src/roundtable-planning/lib/business-chain.test.ts`（新建，node:test 风格同 `chain-stages.test.ts`） + +覆盖无 candidates 分支：`available` 缺省视为存活、`available:false` 过滤、不足 2 席返回 null、`live_name` 优先于快照名、dag 分支不受影响；package.json 加 `test:business-chain` 脚本。 + +### 8. 文档 `frontend-web/docs/multi-agent-business-chain-dev.md` + +补一段：链条响应 seat 附带 `available`/`live_name`（后端读时计算、不落库）；选链前端不再调 `listAgents()`。 + +## 兼容性 + +- 新后端 + 旧前端：响应多两个字段，旧代码忽略，无影响。 +- 旧后端 + 新前端：`available` undefined → 全按存活处理，行为等于现在"agents 查询失败"的降级路径；若智能体真被删，Step 2 init 的后端 404 硬校验仍兜底（现状即有）。 +- 存量链条 seats JSON 无需迁移；8BF / 岗位协同 / official-task-analysis 等消费 seat 快照的模块零改动。 + +## 验证 + +- 后端：`PYTHONPATH=. uv run pytest tests/test_roundtable_chain_seat_enrichment.py tests/test_roundtable_chain_stages.py tests/test_roundtable_chain_hidden.py -v` + `make lint`（ruff）。 +- 前端：`pnpm typecheck` + `node --experimental-strip-types --test src/roundtable-planning/lib/business-chain.test.ts`。 \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 3f22ae09..004d365e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,7 +88,10 @@ The frontend (`frontend-web/`) is a React 19 + Vite SPA: - **Routing**: React Router v7; top-level routes in `src/pages/PageRoutes.tsx`, workspace sub-routes in `src/pages/WorkspaceRoutes.tsx` - **Pages**: `ChatPage` (thread chat), `AgentChatPage` (custom agent chat), `AgentsPage`/`NewAgentPage` (agent management), `ScheduledTasksPage`/`ScheduledTaskRunDetailPage` (scheduled tasks), `LightAppManagementPage`/`LightAppListPage`/`LightAppIframeView` (轻应用中心 — register/list/embed mini-apps) -- **AI 写作·对话版（对话驱动写作台）**: a **parallel new entry** beside the classic AI-writing page (classic page/routes untouched) — route `/page/canvas/ai-writing-chat`, sidebar 「问答管理 → 课题研究(对话版)」. Page `open-canvas/pages/AIWritingChatPage.tsx` reuses the engine (`AIWritingContext` + `useStream` + `AIWritingDraftPanel`); **idle keeps the classic setup layout** (`WritingSetupChat` + collapsible form) and switches to the chat workbench once writing starts; left panel `components/ai-writing-chat/WritingChatPanel.tsx` = **unified chronological feed** (`ChatFlowTimeline.tsx`, chat bubbles anchor-interleaved into the progress timeline so triggered events render below the user's message) + button-only intervention cards (`ChatInterventionCards.tsx`, inline textareas removed) + bottom unified smart composer. Intent resolution: frontend fast path + `POST /api/ai-writing/sessions/{id}/intent` → `deerflow/agents/ai_writing/intent_router.py` (per-pause action whitelist, high-cost confidence gates, payload cleaning); supports QA bypass, clarify follow-ups, control intents, and material-stage auto re-search + pending-intent resume. Tests `tests/test_ai_writing_intent.py`; plan doc `frontend-web/docs/ai-writing-对话驱动写作台-实现方案.md`. +- **智能体对话跳转**: 创建页、智能体管理编辑框和问答页详情面板都可以给智能体配置「对话跳转」链接、打开方式（新窗口 / 当前窗口）和路由参数（登录参数、主题参数，解析方式与应用管理相同）。配置保存在该智能体 `config.yaml` 的 `chat_link`，并镜像到 `agent_extras.chat_link` 供列表读取。留空时智能体管理的「对话」按钮仍进入本系统问答；填了地址后，按钮按解析后的链接跳转。 +- **模型流完整性检测 + 截断记录 (stream truncation)**: 模型配置页（`/page/workspace/admin/models`，`ModelManagementPage.tsx`）每张模型卡片新增「流完整性检测」开关 — `PUT /api/admin/models/strict-stream-integrity/{name}` 写 config.yaml 的 `strict_stream_integrity: true/false`（`app/gateway/routers/model_management.py`，仅 OpenAI 兼容 Chat Completions 模型可开；显式 `false` 也会覆盖 qwen3.8-flash-next 的自动开启；卡片显示值 = 显式配置 ?? qwen3.8-flash-next 自动规则，前端 `isAutoStrictIntegrityModel` 复刻后端判定）。开启后上游流未发终止帧就结束时抛可重试 `IncompleteChatStreamError`（`chat_stream_state.py`），LLM 错误中间件按 upstream 重试，避免半截回答静默入库。同时每次流异常结束都由 `patched_openai.py` 的 `_ToolCallStreamDiagnostics._record_truncation_event` 落库 `stream_truncation_events`（分类：`silent_no_terminal` 静默截断 / `incomplete_stream_error` 完整性拦截 / `length_truncated` / `content_filter` / `midstream_error`；write_file 兼容桥故意提前收流与用户取消不计；经 `fire_and_forget_db_write` 从任意 loop/thread 安全写入，store 在 `deerflow/persistence/stream_truncation/`，**每次写入顺手删除 30 天前的行，硬上限一个月**）。查看页 `pages/StreamTruncationLogPage.tsx`（路由 `/page/workspace/admin/stream-truncations`，admin gate 同模型配置页；简洁模式「设置和更多」下拉的「模型截断记录」入口在 `workspace-nav-menu.tsx`），client `core/stream-truncation-log/api.ts`；后端 API `GET /api/admin/stream-truncations`（分页 + 模型/类型过滤）与 `/breakdown`（按模型汇总，`app/gateway/routers/stream_truncations.py`）。测试 `tests/test_stream_truncation.py`、`tests/test_model_management.py`。配套「保留中间正文」（管理员设置弹窗同名分区，`GET/PUT /api/system-settings/retry-keep-partial`，默认开启，关闭即恢复旧行为）：①正常路径——原先「消息带可见工具调用就不给正文气泡」的分组规则会把步骤条执行中已流出的正文清掉；现 `groupMessages` 新增 `keepIntermediateText` 选项（MessageList 按开关传入）：本轮全部过程正文**合并成一个气泡**挂在步骤条正下方（合成消息剥思考/工具调用，组 id 加 `:keep-text` 后缀避免 React key 冲突），步骤条保持一条不被拆开（正文气泡对工具消息/后续轮次的累积「透明」，`lastOpenGroup`/`lastNonKeepBubbleGroup` 向前跳过），分组完成后把合并气泡**重排到 present-files/subagent/document-rewrite 附属组之后**（正文块与最终回答相邻、不被文件卡片隔开），仅本轮「最终正文消息」（回扫到的最后一条有正文且无可见工具调用的 ai 消息）保留独立气泡；②重试路径——LLM 错误中间件静默重试时，上一次尝试已流出的半截正文从未进 checkpoint、会被前端 values 对账清掉，`core/threads/hooks.ts` 在收到 `llm_retry` 事件时快照在途消息，被丢弃后按原位插回（纯函数 `core/threads/retry-kept-messages.ts`，快照只在本轮流式期间有效，刷新即失）。测试 `tests/test_strategy_project_settings.py` 尾部、`src/core/messages/utils-group.test.ts`、`src/core/threads/retry-kept-messages.test.ts`（`pnpm test:retry-kept`）。 +- **知识库深度研究（普通知识库限定问答）**: 每个可读知识库从卡片或详情页进入 `/page/workspace/knowledge/:noteId/research`。它不复用原“深度研究”流程，而是把普通 Lead Agent 问答在服务端强制绑定到单个知识库、经典检索并关闭工具/记忆；私有会话与草稿仅创建者可见。首个完整回答自动成为 Markdown 草稿，后续回答须显式“应用为文档”。可写用户保存时在该知识库 Wiki 下创建保留根目录「深度研究」，用独立 publication id 的稳定 slug 创建/覆盖文章；文件名独立于问答标题。书架文章随知识库读权限共享，但“更多”及原问答仅创建者可见；只读用户可问答和整理草稿，不能发布。前端核心页 `pages/KnowledgeResearchPage.tsx`，client `strategy-components/api/knowledge-research.ts`；后端 REST 在 `app/gateway/routers/llmwiki.py`，运行时约束 `app/gateway/knowledge_research.py`，持久化 `deerflow/persistence/knowledge_research.py`。 +- **AI 写作·对话版（对话驱动写作台）**: a **parallel new entry** beside the classic AI-writing page (classic page/routes untouched) — route `/page/canvas/ai-writing-chat`, sidebar 「问答管理 → 课题研究(对话版)」. Page `open-canvas/pages/AIWritingChatPage.tsx` reuses the engine (`AIWritingContext` + `useStream` + `AIWritingDraftPanel`); **idle keeps the classic setup layout** (`WritingSetupChat` + collapsible form) and switches to the chat workbench once writing starts; left panel `components/ai-writing-chat/WritingChatPanel.tsx` = **unified chronological feed** (`ChatFlowTimeline.tsx`, chat bubbles anchor-interleaved into the progress timeline so triggered events render below the user's message) + button-only intervention cards (`ChatInterventionCards.tsx`, inline textareas removed) + bottom unified smart composer. Intent resolution: frontend fast path + `POST /api/ai-writing/sessions/{id}/intent` → `deerflow/agents/ai_writing/intent_router.py` (per-pause action whitelist, high-cost confidence gates, payload cleaning); supports QA bypass, clarify follow-ups, control intents, and material-stage auto re-search + pending-intent resume. Tests `tests/test_ai_writing_intent.py`; plan doc `frontend-web/docs/ai-writing-对话驱动写作台-实现方案.md`. **素材检索技能池限定白名单**：`researcher_node` 通用检索的候选池 = **ai-writing-researcher 智能体 config.yaml `skills` 白名单里的启用技能**（管理员在智能体管理页增删，内置 `knowledge-base-search` 保底且始终排最前），不再列全部启用技能——300+ 技能部署把选择提示词撑爆导致素材检索不可用；容错兜底 `_fallback_candidate_skills` 同样限定白名单；白名单未配置/加载失败回退仅内置知识库检索。实现 `deerflow/agents/ai_writing/nodes/researcher.py::_load_candidate_skills/_researcher_skill_whitelist`，测试 `tests/test_ai_writing_knowledge_base.py`。 - **轻应用中心 (Light App Center)**: a sibling of 任务管理 in the sidebar with two pages — 应用管理 (admin registers `window_open` or `iframe` mini-apps with a route-param builder: login params resolved from `localStorage.userInfo`, theme params literal) and 应用列表 (card grid of `window_open` apps, `window.open` launch). `iframe` apps are mounted under a chosen parent menu (XK/PG/TY/综合管理) and **dynamically injected into the sidebar** by `core/page-layout/use-sidebar-menu.ts` (`useSidebarMenuItems`), opening in the `LightAppIframeView` host. Client + React Query hooks in `strategy-components/api/light-apps.ts`; URL builder/constants in `strategy-components/lib/light-app.ts`. Backend: `/api/light-apps` (global/shared, admin-gated writes). - **菜单管理 (Menu Management, admin defaults + personal layouts)**: `MenuManagementPage` (`pages/MenuManagementPage.tsx`) is available from「设置和更多」and the global Header account dropdown. Administrators edit the shared default menu; ordinary users edit only their own full/compact menu snapshot. For users with a saved snapshot, known node ids keep the personal `label`/`disabled`/`parentId`/`sortOrder` (or compact placement/order), while administrator defaults for node ids absent from that snapshot flow through—so newly-added code menus and iframe light apps follow the current default without overwriting older personal choices. Temporarily absent dynamic nodes are preserved when an unrelated personal edit is saved;「恢复默认」deletes only that user's snapshot. The static tree (`core/page-layout/sidebar-menu.ts`) remains the source of icons/routes and administrator defaults remain sparse per-node deltas. Merge/snapshot helpers live in `core/page-layout/menu-overrides.ts` and `compact-menu-overrides.ts`; full-menu writes carry the complete effective parent hierarchy for cycle/strict three-level validation while persisting only the sparse admin delta or full personal snapshot. Backend `/api/menu-overrides` uses `menu_overrides` (default) + `user_menu_override_configs` (per-user snapshot); `/api/compact-menu-overrides` uses `compact_menu_overrides` (type `A` / `B` defaults) + `user_compact_menu_override_configs` (per-user/type snapshots). Each website selects its compact type via `runtime-config.js` `VITE_COMPACT_MENU_TYPE`; `CompactToolbox` renders `toolbox` items. - **舆情分析智能体（前端虚拟智能体）**: 智能体管理页新增「舆情分析」类型 tab，固定一张虚拟智能体卡片（名称/简介可编辑，存 localStorage）。对话页 `pages/SentimentAgentChatPage.tsx`（路由 `agents/sentiment-analysis/chats/:thread_id`）复用普通问答布局 + `MessageList`，**输入框只有发送按钮**；问答经 Gateway `POST /api/sentiment-agent/stream` 转发外部 AG-UI 网关（不经过 LangGraph），会话仅存 sessionStorage。接口地址 + Basic Auth 在 `runtime-config.js` 配置（`VITE_SENTIMENT_AGENT_API_URL` / `_AUTH_USERNAME` / `_AUTH_PASSWORD`）并随请求传给后台；后台按传入地址转发、关闭 TLS 校验，失败时把完整错误回传。核心模块 `src/core/sentiment-agent/`。 @@ -98,12 +101,15 @@ The frontend (`frontend-web/`) is a React 19 + Vite SPA: - **技能上下线 (Skill Availability, owner/admin)**: the global `Skill.enabled` flag is the online/offline switch. Taking a skill offline persists `skills..enabled=false` in `extensions_config.json` but does **not** delete its directory or remove it from existing agent/workflow configurations; the row remains visible and bringing it online restores those references. Administrators may toggle every skill from 「技能管理」; ordinary users may toggle only their own non-built-in `custom` skills from 「技能 → 技能列表 → 我的」. Offline skills must be absent from every model-facing skill catalogue (`lead` prompt, `search_skills`, `skill_list`, `skill_view`, subagents, AI writing/deep research/roundtable and Workflow run-start validation), from agent skill pickers, and from direct sandbox access: `read_file`/`ls`/`glob`/`grep`/`bash` reject offline, unknown, root-wide and category-wide `/mnt/skills` paths. The backend mutation is `PUT /api/skills/{name}`, authorizes either an administrator or the persisted owner of the target custom skill, patches the raw document so MCP placeholders and unrelated extension fields survive, uses atomic replacement when supported plus a Docker single-file-bind fallback, and refreshes the skill prompt cache. Launchers keep the runtime extensions file in a writable persisted directory. Existing agent/strategy-agent references stay visible as 「已下线」 and may be removed but not newly selected. - **技能地址管理 (Skill Address Management, admin-only)**: 扫描运行中的技能（`skills/{public,custom}` 下所有 `.md`/`.py`）里的 **http(s) URL 与 IP/IP:端口**，按唯一地址聚合全部出现位置（跨 .md/.py），支持**批量替换**（如把 UAT 地址一次性改成生产）。主体抽成可复用组件 `SkillAddressesPanel`（`pages/SkillAddressesPage.tsx` 导出，默认导出 `SkillAddressesPage` 仍保留旧路由 `/page/strategy/admin/skill-addresses` 作直链回退），**作为「技能地址管理」标签页嵌入管理员「技能管理」页 `pages/AdminSkillsPage.tsx`**（该页加 Tabs：「技能列表」+「技能地址管理」；入口在「设置和更多」下拉的「技能管理」`workspace-nav-menu.tsx` → `/page/workspace/admin/skills`，不再单列于 `components/page-sidebar.tsx` 左下角设置下拉）：打开自动扫描并带**进度条**（扫描逐技能 / 应用逐文件 X/N，应对 150+ 技能），URL/IP 类型过滤 + 搜索 + 「片段批量套用」（如所有 `uat.4.cn` → `prod.4.cn`）、每地址「替换为」输入框（可展开看 `file:line` 片段）、内联**预览**、**应用**、**多关键词检索**（空格分隔全部命中），以及**修改记录**面板——每次应用生成一条记录（显示时间/改了哪些地址 from→to/文件数/处数），可**针对单次记录回退**（二次确认）。client + NDJSON 流式解析在 `strategy-components/api/skill-addresses.ts`。后端 `/api/skill-addresses`（**仅管理员**）：所有文件 IO 走 `asyncio.to_thread` 不阻塞事件循环；`GET /scan`(+`/scan/stream`)、`POST /preview`、`POST /apply`(+`/apply/stream`)、`GET /backups`、`POST /rollback/{id}`；**子串安全替换**（基于 token span，不会误伤子串）、`expected_total` 乐观锁（409）、写前自动磁盘备份到 `{skills_root}/.address-edits/`（同时写一份 `{id}.manifest.json` 记录本次替换明细，供修改记录展示）、应用后刷新技能系统提示缓存。纯引擎 `deerflow/skills/address_scan.py`，router `app/gateway/routers/skill_addresses.py`，tests `tests/test_skill_addresses.py`。范围仅运行中技能，不动 `deploy/minimal/skills/`。 - **按钮管理 (Task Button Management, admin-only)**: configures the **可配置跳转按钮** rendered in the left task panel of the task deep-link workspaces (rwfx / 3qfx / xdfx, see `TaskFxWorkspace`). Each button = `{business, label, linkType, target, appendTaskId, idParam, idKind, enabled, sortOrder}`; `linkType` is `url` (外链 `window.open`, optional append of `?{idParam}={id}` — **`idParam` is the editable query-param name**, e.g. `id`/`task_id`), `business` (内部切换到另一业务, reuses `resolveTaskDeeplinkTarget`, always task id), or `purpose` (弹出当前任务的「任务目的」表格 via the shared `TaskPurposeDialog`, no target — configurable for any page incl. xdfx). **`host`（宿主路由）** asks the embedding parent to navigate: `runTaskButton` sends `window.parent.postMessage({ type:'HOST_NAVIGATE', path, queryParams:{ [idParam]: Number(taskId) } }, hostOrigin)` — `target` is the host route, `appendTaskId`/`idParam` become numeric `queryParams` (task id, or action id when `idKind:"action"` — same selector as url buttons, xdfx only), origin from `runtime-config.js` `VITE_HOST_NAVIGATE_ORIGIN` (else ancestorOrigins/referrer; never `*`). **`idKind` (`task`/`action`) only matters for xdfx**: its deep-link `?taskId=` is an 行动id and the 任务id is resolved at login (`fetchTaskIdFromAction`) — `TaskContext` now also stashes the raw `actionId`, so a url **or host** button can append either (default `task`). Page `TaskButtonsPage.tsx` (route `/page/strategy/admin/task-buttons`, **admin-gated**, entry「按钮管理」in the bottom-left settings dropdown of `components/page-sidebar.tsx`, both full + collapsed modes) has per-business tabs, edits name/type/target/启停/排序/增删, saves the **whole set atomically** (replace-all), plus 「导入默认」 to merge code-seed defaults. **Seed/fallback** (like 敏感词管理): only when the whole table is empty (fresh deploy, never saved) do the workspace + page fall back to `defaultTaskButtons(cfg)` — rwfx's url buttons (编辑/下一步/3q详情) built from the **live** `/api/public/config/task-deeplink` plus cross-business switch buttons for **all three** businesses (each switches to the other two — rwfx→三情/行动, 3qfx→任务/行动, xdfx→任务/三情). Once saved, the store is the sole source of truth. The old hardcoded jump buttons were migrated out: `TaskFxWorkspace` no longer hardcodes 切换到三情分析, `TaskActionsPanel` keeps only the non-jump 清除记录, and `TaskPurposePanel` dropped its 3q详情 button (now configurable) **and defaults the 目的树 table to collapsed** (chevron toggle). Helpers `core/auth/task-buttons.ts` (`TaskButton`, `defaultTaskButtons`, `resolveTaskButtons`, `buttonsForBusiness`, `runTaskButton`), client+hooks `strategy-components/api/task-buttons.ts`. Backend: `/api/task-buttons` (global/shared; open read, admin replace-all), store `deerflow.persistence.task_buttons` (table `task_buttons`, auto-created by `create_all`), wired `app.state.task_button_store`. Tests: `tests/test_task_buttons.py`. +- **HTML 模板管理 (HTML Template Management, admin-only)**: 定时任务「HTML 模板」库的可视化管理——列表/预览（iframe srcDoc sandbox）/在线编辑/上传 .html 覆盖/新建/删除/重置内置，改完**即时生效**（调度器每次执行直接读文件，无需重启）。模板从纯代码注册表升级为**文件型存储**：harness 纯引擎 `deerflow/runtime/scheduler/template_store.py`（`{base_dir}/html-templates//{template.html,meta.json}`，gitignored），app.py lifespan 把随包 `scheduler/templates/` 的全部内置模板（目前 27 个）seed 进存储——**目录缺失才复制，绝不覆盖管理页/线上修改**；`template_fill.py` 的 `get_template/list_templates/read_template_text` 改为存储优先、随包注册表兜底，写操作后 `invalidate_cache()` 清引擎 lru_cache。**convertible 自动探测**：上传/编辑正文时检测单行 `const DATA={…};` 注入行（当前引擎唯一注入点，`const MANIFEST=` 同理），带注入行即「可转换填充」（定时任务 JSON 转换管线零代码接入），否则「只展示」；数据存 ` + + diff --git a/diagnose_deerflow_latency.py b/diagnose_deerflow_latency.py new file mode 100644 index 00000000..e205af11 --- /dev/null +++ b/diagnose_deerflow_latency.py @@ -0,0 +1,4782 @@ +#!/usr/bin/env python3 +"""Standalone evidence-first latency diagnostics for DeerFlow HTTP deployments. + +The script is intentionally standard-library-only so it can be copied to a +Linux deployment host and run with the system Python. It combines: + +* fresh-connection HTTP timing (DNS, TCP, TLS, response wait, first body byte, + body read, total), including bounded concurrency and direct-backend + comparisons; +* Linux host, process, socket/backlog, file-descriptor, pressure and Docker + snapshots; +* read-only Nginx capacity/config inspection and log-pattern aggregation; +* optional read-only MySQL status/process/lock checks through the ``mysql`` + command-line client; and +* optional DeerFlow administrator concurrency endpoints. + +Security invariants: + +* URL query values and userinfo are always redacted from console/JSON output. +* Request/response bodies and secret header values are never printed or saved. +* Successful login and agent POSTs can mutate session/run state, so unsafe + methods are never replayed unless ``--allow-unsafe-replay`` is explicit. +* TLS verification is enabled by default. ``--insecure`` is explicit and is + recorded as a high-severity caveat in the report. + +This tool reports evidence and cautious localization. A high response wait is +not, by itself, proof of a slow database, Nginx, or an LLM provider. +""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import contextlib +import datetime as dt +import getpass +import glob +import hashlib +import http.client +import io +import ipaddress +import json +import math +import os +import platform +import re +import shutil +import socket +import ssl +import statistics +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +REPORT_VERSION = 1 +USER_AGENT = "deerflow-latency-diagnosis/1" +SAFE_METHODS = {"GET", "HEAD", "OPTIONS"} +SECRET_HEADER_RE = re.compile( + r"(?:authorization|proxy-authorization|cookie|set-cookie|token|api[-_]?key|secret|password|csrf|session|credential)", + re.IGNORECASE, +) +SECRET_QUERY_RE = re.compile( + r"(?:token|auth|password|passwd|secret|key|ticket|session|cookie|credential)", + re.IGNORECASE, +) +MAX_HEADER_BYTES = 1024 * 1024 +MAX_REQUEST_HEADER_BYTES = 64 * 1024 +MAX_COMMAND_OUTPUT = 4 * 1024 * 1024 +UTC = getattr(dt, "UTC", dt.timezone.utc) # noqa: UP017 - keep the standalone tool usable on Python 3.9 +HEADER_NAME_RE = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") + + +@dataclass +class ProbeOptions: + method: str = "GET" + headers: dict[str, str] = field(default_factory=dict) + body: bytes = b"" + connect_timeout: float = 5.0 + first_byte_timeout: float = 30.0 + read_timeout: float = 5.0 + total_timeout: float = 120.0 + max_response_bytes: int = 1024 * 1024 + read_mode: str = "auto" + insecure: bool = False + ca_file: str | None = None + follow_redirects: bool = False + max_redirects: int = 3 + proxy: str | None = None + use_env_proxy: bool = False + ip_family: str = "auto" + + +class ProbeFailure(RuntimeError): + """Transport failure with the phase that failed.""" + + def __init__(self, phase: str, message: str): + super().__init__(message) + self.phase = phase + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Return 30x to the caller instead of forwarding credentials implicitly.""" + + def redirect_request( + self, + req: urllib.request.Request, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> None: + return None + + +class _SocketBodyReader: + """Bounded socket reader used by HTTP body framing parsers.""" + + def __init__( + self, + sock: socket.socket, + initial: bytes, + *, + idle_timeout: float, + deadline: float, + idle_timeout_phase: str = "body_idle_timeout", + ): + self.sock = sock + self.buffer = bytearray(initial) + self.idle_timeout = max(0.1, idle_timeout) + self.deadline = deadline + self.idle_timeout_phase = idle_timeout_phase + self.eof = False + + def _fill(self) -> None: + if self.eof: + return + remaining = self.deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("body_total_timeout", "overall request timeout expired while reading response body") + self.sock.settimeout(min(self.idle_timeout, remaining)) + try: + chunk = self.sock.recv(65536) + except (TimeoutError, socket.timeout) as exc: # noqa: UP041 - Python 3.9/Windows may not alias these + phase = "body_total_timeout" if time.perf_counter() >= self.deadline else self.idle_timeout_phase + raise ProbeFailure(phase, f"response body made no progress for {min(self.idle_timeout, remaining):.3f}s") from exc + except OSError as exc: + raise ProbeFailure("body_read", _safe_error(exc)) from exc + if not chunk: + self.eof = True + return + self.buffer.extend(chunk) + + def read_some(self, maximum: int) -> bytes: + while not self.buffer and not self.eof: + self._fill() + if not self.buffer: + return b"" + size = min(maximum, len(self.buffer)) + output = bytes(self.buffer[:size]) + del self.buffer[:size] + return output + + def read_exact(self, size: int) -> bytes: + while len(self.buffer) < size and not self.eof: + self._fill() + if len(self.buffer) < size: + raise ProbeFailure("body_framing", "response ended before the declared body framing completed") + output = bytes(self.buffer[:size]) + del self.buffer[:size] + return output + + def read_line(self, limit: int = 8192) -> bytes: + while True: + marker = self.buffer.find(b"\r\n") + if marker >= 0: + if marker > limit: + raise ProbeFailure("body_framing", "HTTP chunk framing line exceeded 8192 bytes") + output = bytes(self.buffer[:marker]) + del self.buffer[: marker + 2] + return output + if len(self.buffer) > limit: + raise ProbeFailure("body_framing", "HTTP chunk framing line exceeded 8192 bytes") + if self.eof: + raise ProbeFailure("body_framing", "response ended inside HTTP chunk framing") + self._fill() + + +def utc_now_iso() -> str: + return dt.datetime.now(UTC).isoformat() + + +def monotonic_ms(start: float, end: float | None = None) -> float: + return round(((end if end is not None else time.perf_counter()) - start) * 1000.0, 3) + + +def _safe_error(exc: BaseException, limit: int = 300) -> str: + """Return a bounded error string with obvious credentials redacted.""" + text = f"{type(exc).__name__}: {exc}" + text = re.sub( + r"https?://[^\s\]\[(){}<>'\"]+", + lambda match: sanitize_url(match.group(0)), + text, + flags=re.IGNORECASE, + ) + text = re.sub( + r"(?i)(authorization|cookie|token|password|secret|api[-_]?key|csrf|session|credential)\s*[:=]\s*[^\s,;]+", + r"\1=", + text, + ) + text = re.sub(r"(?i)://[^/@\s]+@", "://@", text) + return text[:limit] + + +def sanitize_url(url: str) -> str: + """Redact userinfo and every query value while retaining routing shape.""" + try: + parsed = urllib.parse.urlsplit(url) + except ValueError: + return "" + if not parsed.scheme or not parsed.netloc: + return "" + hostname = parsed.hostname or "" + try: + ip = ipaddress.ip_address(hostname) + rendered_host = f"[{ip}]" if ip.version == 6 else str(ip) + except ValueError: + rendered_host = hostname + try: + parsed_port = parsed.port + except ValueError: + return "" + if parsed_port is not None: + rendered_host = f"{rendered_host}:{parsed_port}" + safe_query: list[tuple[str, str]] = [] + for index, query_field in enumerate(parsed.query.split("&")): + if not query_field: + continue + raw_key, separator, _raw_value = query_field.partition("=") + key = urllib.parse.unquote_plus(raw_key) + key_is_safe = bool(separator and len(key) <= 64 and re.fullmatch(r"[A-Za-z0-9_.~-]+", key)) + safe_key = key if key_is_safe else f"param_{index + 1}" + marker = "" if SECRET_QUERY_RE.search(key) else "" + safe_query.append((safe_key, marker)) + query = urllib.parse.urlencode(safe_query) + return urllib.parse.urlunsplit((parsed.scheme, rendered_host, parsed.path or "/", query, "")) + + +def sanitize_origin(url: str) -> str: + try: + parsed = urllib.parse.urlsplit(url) + parsed_port = parsed.port + except ValueError: + return "" + hostname = parsed.hostname or "" + try: + ip = ipaddress.ip_address(hostname) + rendered_host = f"[{ip}]" if ip.version == 6 else str(ip) + except ValueError: + rendered_host = hostname + default_port = 443 if parsed.scheme == "https" else 80 + if parsed_port and parsed_port != default_port: + rendered_host = f"{rendered_host}:{parsed_port}" + return f"{parsed.scheme}://{rendered_host}" + + +def _origin_tuple(url: str) -> tuple[str, str, int]: + parsed = urllib.parse.urlsplit(url) + scheme = parsed.scheme.lower() + if scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("URL must use http:// or https:// and include a host") + return scheme, parsed.hostname.lower(), parsed.port or (443 if scheme == "https" else 80) + + +def _host_is_loopback(hostname: str) -> bool: + normalized = hostname.rstrip(".").lower() + if normalized == "localhost": + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False + + +def _url_has_secret_query(url: str) -> bool: + return any( + bool(value) and bool(SECRET_QUERY_RE.search(key)) + for key, value in urllib.parse.parse_qsl( + urllib.parse.urlsplit(url).query, + keep_blank_values=True, + ) + ) + + +def _header_is_secret(name: str) -> bool: + return bool(SECRET_HEADER_RE.search(name)) + + +def _validate_header(name: str, value: str) -> None: + if not HEADER_NAME_RE.fullmatch(name): + raise ValueError(f"Invalid header name: {name!r}") + if any((ord(ch) < 32 and ch != "\t") or ord(ch) == 127 or ord(ch) > 255 for ch in value): + raise ValueError(f"Header {name!r} contains an illegal control or non-Latin-1 character") + if len(value.encode("iso-8859-1")) > 16 * 1024: + raise ValueError(f"Header {name!r} exceeds 16 KiB") + + +def parse_header_argument(raw: str) -> tuple[str, str]: + name, separator, value = raw.partition(":") + if not separator: + raise ValueError("Header must use NAME:VALUE") + name = name.strip() + value = value.strip() + _validate_header(name, value) + return name, value + + +def parse_header_env_argument(raw: str) -> tuple[str, str]: + name, separator, env_name = raw.partition("=") + if not separator or not name.strip() or not env_name.strip(): + raise ValueError("Environment header must use HEADER=ENV_NAME") + value = os.environ.get(env_name.strip()) + if value is None: + raise ValueError(f"Environment variable {env_name.strip()} is not set") + name = name.strip() + _validate_header(name, value) + return name, value + + +def parse_server_timing(raw: str | None) -> list[dict[str, Any]]: + """Parse only metric name/duration; deliberately discard descriptions.""" + if not raw: + return [] + metrics: list[dict[str, Any]] = [] + for part in raw.split(","): + segments = [segment.strip() for segment in part.split(";") if segment.strip()] + if not segments: + continue + name = re.sub(r"[^A-Za-z0-9_.:-]", "", segments[0])[:80] + if not name: + continue + item: dict[str, Any] = {"name": name} + for segment in segments[1:]: + key, separator, value = segment.partition("=") + if separator and key.strip().lower() == "dur": + try: + duration = float(value.strip().strip('"')) + except ValueError: + continue + if math.isfinite(duration) and duration >= 0: + item["duration_ms"] = round(duration, 3) + metrics.append(item) + return metrics + + +def percentile(values: Sequence[float], percentile_value: float) -> float | None: + if not values: + return None + ordered = sorted(float(value) for value in values) + if len(ordered) == 1: + return round(ordered[0], 3) + rank = (len(ordered) - 1) * percentile_value + lower = math.floor(rank) + upper = math.ceil(rank) + if lower == upper: + return round(ordered[lower], 3) + weight = rank - lower + return round(ordered[lower] * (1.0 - weight) + ordered[upper] * weight, 3) + + +def summarize_numbers(values: Iterable[float | int | None]) -> dict[str, float | int | None]: + clean = [float(value) for value in values if value is not None and math.isfinite(float(value))] + if not clean: + return {"samples": 0, "min": None, "p50": None, "p95": None, "max": None, "avg": None} + return { + "samples": len(clean), + "min": round(min(clean), 3), + "p50": percentile(clean, 0.50), + "p95": percentile(clean, 0.95), + "max": round(max(clean), 3), + "avg": round(statistics.fmean(clean), 3), + } + + +def _select_proxy(url: str, options: ProbeOptions) -> str | None: + if options.proxy: + return options.proxy + if not options.use_env_proxy: + return None + parsed = urllib.parse.urlsplit(url) + hostname = parsed.hostname or "" + bypass_targets = [hostname] if hostname else [] + if parsed.port is not None: + try: + rendered_bypass_host = f"[{hostname}]" if ipaddress.ip_address(hostname).version == 6 else hostname + except ValueError: + rendered_bypass_host = hostname + bypass_targets.append(f"{rendered_bypass_host}:{parsed.port}") + if any(target and urllib.request.proxy_bypass(target) for target in dict.fromkeys(bypass_targets)): + return None + proxies = urllib.request.getproxies() + return proxies.get(parsed.scheme.lower()) + + +def _address_family(value: str) -> int: + if value == "ipv4": + return socket.AF_INET + if value == "ipv6": + return socket.AF_INET6 + return socket.AF_UNSPEC + + +def _connect_socket( + host: str, + port: int, + *, + timeout: float, + family: str, +) -> tuple[socket.socket, dict[str, Any]]: + deadline = time.perf_counter() + timeout + dns_started = time.perf_counter() + resolution: dict[str, Any] = {} + resolved = threading.Event() + + def resolve() -> None: + try: + resolution["infos"] = socket.getaddrinfo( + host, + port, + family=_address_family(family), + type=socket.SOCK_STREAM, + ) + except Exception as exc: # captured and re-raised in the caller thread + resolution["error"] = exc + finally: + resolved.set() + + resolver = threading.Thread(target=resolve, name="deerflow-dns-probe", daemon=True) + resolver.start() + if not resolved.wait(timeout): + raise ProbeFailure("dns", f"timed out after {timeout:.3f}s") + error = resolution.get("error") + if error is not None: + if isinstance(error, socket.gaierror): + raise ProbeFailure("dns", _safe_error(error)) from error + raise ProbeFailure("dns", _safe_error(error)) + infos = resolution.get("infos") or [] + dns_ms = monotonic_ms(dns_started) + if not infos: + raise ProbeFailure("dns", "DNS returned no usable address") + + attempts: list[dict[str, Any]] = [] + last_error: BaseException | None = None + tcp_started = time.perf_counter() + for af, socktype, protocol, _canonname, sockaddr in infos: + remaining = deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("connect", f"DNS and TCP connection exceeded {timeout:.3f}s") + sock = socket.socket(af, socktype, protocol) + sock.settimeout(remaining) + started = time.perf_counter() + try: + sock.connect(sockaddr) + except OSError as exc: + last_error = exc + attempts.append( + { + "family": "ipv6" if af == socket.AF_INET6 else "ipv4", + "address": str(sockaddr[0]), + "connect_ms": monotonic_ms(started), + "outcome": type(exc).__name__, + } + ) + sock.close() + continue + connect_ms = monotonic_ms(started) + attempts.append( + { + "family": "ipv6" if af == socket.AF_INET6 else "ipv4", + "address": str(sockaddr[0]), + "connect_ms": connect_ms, + "outcome": "connected", + } + ) + return sock, { + "dns_ms": dns_ms, + "connect_ms": monotonic_ms(tcp_started), + "chosen_address_connect_ms": connect_ms, + "peer_ip": str(sockaddr[0]), + "connect_attempts": attempts, + } + raise ProbeFailure("connect", _safe_error(last_error or OSError("all addresses failed"))) + + +def _recv_header_block( + sock: socket.socket, + timeout: float, + *, + initial: bytes = b"", +) -> tuple[bytes, bytes, float]: + deadline = time.perf_counter() + timeout + buffer = bytearray(initial) + first_byte_at: float | None = time.perf_counter() if initial else None + while b"\r\n\r\n" not in buffer: + remaining = deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("first_byte", f"response headers exceeded {timeout:.3f}s total") + sock.settimeout(remaining) + try: + chunk = sock.recv(65536) + except (TimeoutError, socket.timeout) as exc: # noqa: UP041 - standalone Python compatibility + raise ProbeFailure("first_byte", f"timed out after {timeout:.3f}s") from exc + except OSError as exc: + raise ProbeFailure("first_byte", _safe_error(exc)) from exc + if not chunk: + raise ProbeFailure("http_headers", "connection closed before response headers completed") + if first_byte_at is None: + first_byte_at = time.perf_counter() + buffer.extend(chunk) + if len(buffer) > MAX_HEADER_BYTES and b"\r\n\r\n" not in buffer: + raise ProbeFailure("http_headers", "response headers exceeded 1 MiB") + head, body = bytes(buffer).split(b"\r\n\r\n", 1) + if len(head) > MAX_HEADER_BYTES: + raise ProbeFailure("http_headers", "response headers exceeded 1 MiB") + return head, body, first_byte_at or time.perf_counter() + + +def _parse_header_block(head: bytes) -> tuple[int, str, dict[str, str]]: + first_line, separator, remainder = head.partition(b"\r\n") + if not separator: + raise ProbeFailure("http_headers", "malformed HTTP status line") + try: + status_text = first_line.decode("iso-8859-1") + status_parts = status_text.split(" ", 2) + if len(status_parts) < 2: + raise ValueError("invalid status line") + version, status_raw = status_parts[:2] + reason = status_parts[2] if len(status_parts) == 3 else "" + if not re.fullmatch(r"HTTP/\d+\.\d+", version): + raise ValueError("invalid protocol") + if not re.fullmatch(r"\d{3}", status_raw): + raise ValueError("invalid status code") + status = int(status_raw) + parsed_headers = http.client.parse_headers(io.BytesIO(remainder + b"\r\n\r\n")) + except (ValueError, http.client.HTTPException) as exc: + raise ProbeFailure("http_headers", _safe_error(exc)) from exc + headers: dict[str, str] = {} + for name in parsed_headers.keys(): + values = parsed_headers.get_all(name) or [] + headers[name.lower()] = ", ".join(values) + return status, reason[:100], headers + + +def _safe_response_headers(headers: Mapping[str, str]) -> dict[str, str]: + allowed = { + "server", + "via", + "content-type", + "content-length", + "transfer-encoding", + "connection", + "x-accel-buffering", + "x-cache", + "retry-after", + } + return {name: str(value)[:300] for name, value in headers.items() if name.lower() in allowed and not _header_is_secret(name)} + + +def _proxy_authorization(parsed_proxy: urllib.parse.SplitResult) -> str | None: + if parsed_proxy.username is None: + return None + username = urllib.parse.unquote(parsed_proxy.username) + password = urllib.parse.unquote(parsed_proxy.password or "") + encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + return f"Basic {encoded}" + + +def _host_header(parsed: urllib.parse.SplitResult) -> str: + hostname = parsed.hostname or "" + try: + if ipaddress.ip_address(hostname).version == 6: + hostname = f"[{hostname}]" + except ValueError: + hostname = hostname.encode("idna").decode("ascii") + default = 443 if parsed.scheme == "https" else 80 + return hostname if (parsed.port or default) == default else f"{hostname}:{parsed.port}" + + +def _build_ssl_context(options: ProbeOptions) -> ssl.SSLContext: + if options.insecure: + context = ssl._create_unverified_context() # noqa: SLF001 - explicit diagnostic flag + else: + context = ssl.create_default_context(cafile=options.ca_file) + with contextlib.suppress(NotImplementedError): + context.set_alpn_protocols(["http/1.1"]) + return context + + +def _do_proxy_connect( + sock: socket.socket, + target_host: str, + target_port: int, + proxy_auth: str | None, + timeout: float, +) -> float: + try: + rendered_host = f"[{target_host}]" if ipaddress.ip_address(target_host).version == 6 else target_host + except ValueError: + rendered_host = target_host.encode("idna").decode("ascii") + authority = f"{rendered_host}:{target_port}" + lines = [ + f"CONNECT {authority} HTTP/1.1", + f"Host: {authority}", + "Proxy-Connection: keep-alive", + ] + if proxy_auth: + lines.append(f"Proxy-Authorization: {proxy_auth}") + request = ("\r\n".join(lines) + "\r\n\r\n").encode("iso-8859-1") + started = time.perf_counter() + try: + sock.sendall(request) + head, _body, _first = _recv_header_block(sock, timeout) + status, _reason, _headers = _parse_header_block(head) + except ProbeFailure as exc: + raise ProbeFailure("proxy_connect", f"{exc.phase}: {exc}") from exc + except OSError as exc: + raise ProbeFailure("proxy_connect", _safe_error(exc)) from exc + if not 200 <= status < 300: + raise ProbeFailure("proxy_connect", f"HTTP proxy CONNECT returned {status}") + return monotonic_ms(started) + + +def _request_target(parsed: urllib.parse.SplitResult, use_http_proxy: bool) -> str: + safe_path = urllib.parse.quote(parsed.path or "/", safe="/%:@-._~!$&'()*+,;=") + safe_query = urllib.parse.quote(parsed.query, safe="=&%/:@-._~!$'()*+,;?") + path = urllib.parse.urlunsplit(("", "", safe_path, safe_query, "")) + if not use_http_proxy: + return path + netloc = _host_header(parsed) + return urllib.parse.urlunsplit((parsed.scheme, netloc, safe_path, safe_query, "")) + + +def _content_length(headers: Mapping[str, str], *, transfer_encoded: bool) -> int | None: + raw = headers.get("content-length") + if raw is None or transfer_encoded: + return None + values = [part.strip() for part in raw.split(",")] + if not values or any(not re.fullmatch(r"\d+", value) for value in values) or len(set(values)) != 1: + raise ProbeFailure("body_framing", "invalid or conflicting Content-Length response header") + return int(values[0]) + + +def _next_chunk_size(reader: _SocketBodyReader) -> int: + raw = reader.read_line() + size_text = raw.split(b";", 1)[0].strip() + if not size_text or not re.fullmatch(rb"[0-9A-Fa-f]+", size_text): + raise ProbeFailure("body_framing", "invalid HTTP chunk size") + size = int(size_text, 16) + if size > 2**63 - 1: + raise ProbeFailure("body_framing", "HTTP chunk size is unreasonably large") + return size + + +def _finish_chunk_trailers(reader: _SocketBodyReader) -> None: + total = 0 + for _index in range(100): + line = reader.read_line() + total += len(line) + 2 + if total > 65536: + raise ProbeFailure("body_framing", "HTTP chunk trailers exceeded 64 KiB") + if not line: + return + raise ProbeFailure("body_framing", "HTTP chunk trailers exceeded 100 fields") + + +def _read_response_body( + sock: socket.socket, + initial: bytes, + headers: Mapping[str, str], + *, + method_has_no_body: bool, + read_mode: str, + max_response_bytes: int, + idle_timeout: float, + first_body_timeout: float, + deadline: float, +) -> dict[str, Any]: + """Read a response entity with Content-Length/chunked framing and hard bounds.""" + transfer_tokens = [item.strip().lower() for item in headers.get("transfer-encoding", "").split(",") if item.strip()] + chunked = bool(transfer_tokens and transfer_tokens[-1] == "chunked") + content_length = _content_length(headers, transfer_encoded=bool(transfer_tokens)) + waiting_for_first_entity_byte = read_mode == "first-byte" + reader = _SocketBodyReader( + sock, + initial, + idle_timeout=first_body_timeout if waiting_for_first_entity_byte else idle_timeout, + deadline=deadline, + idle_timeout_phase="first_body" if waiting_for_first_entity_byte else "body_idle_timeout", + ) + result: dict[str, Any] = { + "body_bytes_read": 0, + "body_complete": False, + "body_truncated_reason": None, + "first_body_at": None, + "error_phase": None, + "error": None, + } + if transfer_tokens and not chunked: + result["framing_note"] = "non_chunked_transfer_encoding_delimited_by_connection_close" + elif transfer_tokens and headers.get("content-length") is not None: + result["framing_note"] = "transfer_encoding_overrides_content_length" + + if method_has_no_body or content_length == 0: + result["body_complete"] = True + return result + if read_mode == "headers": + result["body_truncated_reason"] = "headers_only" + return result + + try: + if chunked: + if read_mode == "first-byte": + while True: + size = _next_chunk_size(reader) + if size == 0: + _finish_chunk_trailers(reader) + result["body_complete"] = True + return result + first = reader.read_exact(1) + if first: + result["body_bytes_read"] = 1 + result["first_body_at"] = time.perf_counter() + result["body_truncated_reason"] = "first_byte_mode" + return result + + while True: + size = _next_chunk_size(reader) + if size == 0: + _finish_chunk_trailers(reader) + result["body_complete"] = True + return result + remaining_chunk = size + while remaining_chunk: + remaining_cap = max_response_bytes - int(result["body_bytes_read"]) + if remaining_cap <= 0: + result["body_truncated_reason"] = "max_response_bytes" + return result + part = reader.read_some(min(65536, remaining_chunk, remaining_cap)) + if not part: + raise ProbeFailure("body_framing", "response ended inside an HTTP chunk") + if result["first_body_at"] is None: + result["first_body_at"] = time.perf_counter() + result["body_bytes_read"] += len(part) + remaining_chunk -= len(part) + if reader.read_exact(2) != b"\r\n": + raise ProbeFailure("body_framing", "HTTP chunk data was not followed by CRLF") + if int(result["body_bytes_read"]) >= max_response_bytes: + # Do not wait for another chunk-size line once the configured + # entity-byte cap is reached. A streaming server is allowed + # to pause forever here; treating that pause as an idle + # timeout would falsely turn an intentional truncation into a + # transport failure. We can still recognize the common + # already-buffered empty-trailer terminator without blocking. + if reader.buffer.startswith(b"0\r\n\r\n"): + del reader.buffer[:5] + result["body_complete"] = True + else: + result["body_truncated_reason"] = "max_response_bytes" + return result + result["body_truncated_reason"] = "max_response_bytes" + return result + + if read_mode == "first-byte": + first = reader.read_some(1) + if first: + result["body_bytes_read"] = 1 + result["first_body_at"] = time.perf_counter() + if content_length == 1: + result["body_complete"] = True + else: + result["body_truncated_reason"] = "first_byte_mode" + elif content_length is None: + result["body_complete"] = True + else: + raise ProbeFailure("body_framing", "response ended before Content-Length bytes arrived") + return result + + expected = content_length + while result["body_bytes_read"] < max_response_bytes: + if expected is not None: + remaining = expected - int(result["body_bytes_read"]) + if remaining <= 0: + result["body_complete"] = True + return result + part = reader.read_some(min(65536, remaining, max_response_bytes - int(result["body_bytes_read"]))) + if not part: + raise ProbeFailure("body_framing", "response ended before Content-Length bytes arrived") + else: + part = reader.read_some(min(65536, max_response_bytes - int(result["body_bytes_read"]))) + if not part: + result["body_complete"] = True + return result + if result["first_body_at"] is None: + result["first_body_at"] = time.perf_counter() + result["body_bytes_read"] += len(part) + if expected is not None and result["body_bytes_read"] >= expected: + result["body_complete"] = True + else: + result["body_truncated_reason"] = "max_response_bytes" + except ProbeFailure as exc: + result["body_truncated_reason"] = exc.phase + result["error_phase"] = exc.phase + result["error"] = str(exc)[:300] + return result + + +def _probe_hop(url: str, options: ProbeOptions) -> tuple[dict[str, Any], str | None]: + hop_started = time.perf_counter() + hop_deadline = hop_started + options.total_timeout + parsed = urllib.parse.urlsplit(url) + scheme, target_host, target_port = _origin_tuple(url) + proxy_url: str | None = None + proxy_parsed: urllib.parse.SplitResult | None = None + connect_host, connect_port = target_host, target_port + proxy_auth: str | None = None + + record: dict[str, Any] = { + "url": sanitize_url(url), + "method": options.method, + "proxy_used": False, + "proxy_origin": None, + "success": False, + "status_code": None, + "error_phase": None, + "error": None, + "timings_ms": {}, + "response": {}, + } + sock: socket.socket | ssl.SSLSocket | None = None + location: str | None = None + try: + proxy_url = _select_proxy(url, options) + if proxy_url: + proxy_parsed = urllib.parse.urlsplit(proxy_url) + if proxy_parsed.scheme.lower() != "http" or not proxy_parsed.hostname: + raise ProbeFailure("proxy", "only an http:// forward proxy is supported") + connect_host = proxy_parsed.hostname + connect_port = proxy_parsed.port or 80 + proxy_auth = _proxy_authorization(proxy_parsed) + if proxy_auth is None: + proxy_auth = next( + (value for name, value in options.headers.items() if name.lower() == "proxy-authorization"), + None, + ) + record["proxy_used"] = True + record["proxy_origin"] = sanitize_origin(proxy_url) + + remaining = hop_deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("total_timeout", "overall request timeout expired before DNS") + sock, connect_details = _connect_socket( + connect_host, + connect_port, + timeout=min(options.connect_timeout, remaining), + family=options.ip_family, + ) + record.update(connect_details) + record["dns_subject"] = "proxy" if proxy_parsed else "origin" + timings = record["timings_ms"] + timings["dns"] = connect_details["dns_ms"] + timings["tcp_connect"] = connect_details["connect_ms"] + + if proxy_parsed and scheme == "https": + remaining = hop_deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("total_timeout", "overall request timeout expired before proxy CONNECT") + timings["proxy_connect"] = _do_proxy_connect( + sock, + target_host, + target_port, + proxy_auth, + min(options.first_byte_timeout, remaining), + ) + + if scheme == "https": + tls_started = time.perf_counter() + try: + remaining = hop_deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("total_timeout", "overall request timeout expired before TLS") + sock.settimeout(min(options.connect_timeout, remaining)) + sock = _build_ssl_context(options).wrap_socket(sock, server_hostname=target_host) + except ssl.SSLCertVerificationError as exc: + raise ProbeFailure("tls_verify", _safe_error(exc)) from exc + except (TimeoutError, ssl.SSLError, OSError) as exc: + raise ProbeFailure("tls_handshake", _safe_error(exc)) from exc + timings["tls_handshake"] = monotonic_ms(tls_started) + cipher = sock.cipher() + cert = sock.getpeercert() or {} + record["tls"] = { + "verified": not options.insecure, + "version": sock.version(), + "cipher": cipher[0] if cipher else None, + "alpn": sock.selected_alpn_protocol(), + "certificate_not_after": cert.get("notAfter"), + } + + request_headers = {name: value for name, value in options.headers.items() if name.lower() != "proxy-authorization"} + lower_names: dict[str, str] = {} + for name in request_headers: + lowered = name.lower() + if lowered in lower_names: + raise ProbeFailure( + "request_validation", + f"duplicate request header names differ only by case: {lower_names[lowered]} and {name}", + ) + lower_names[lowered] = name + for forbidden in ("transfer-encoding", "expect"): + if forbidden in lower_names: + raise ProbeFailure("request_validation", f"request header {forbidden} is not supported") + supplied_length: int | None = None + if "content-length" in lower_names: + raw_length = request_headers.pop(lower_names["content-length"]) + if not re.fullmatch(r"\d+", raw_length.strip()): + raise ProbeFailure("request_validation", "Content-Length must be a non-negative integer") + supplied_length = int(raw_length) + if supplied_length != len(options.body): + raise ProbeFailure("request_validation", "Content-Length does not match the request body") + if "connection" in lower_names: + request_headers.pop(lower_names["connection"]) + defaults = { + "Host": _host_header(parsed), + "User-Agent": USER_AGENT, + "Accept": "*/*", + "Accept-Encoding": "identity", + "Connection": "close", + } + lower_names = {name.lower(): name for name in request_headers} + for name, value in defaults.items(): + if name.lower() not in lower_names: + request_headers[name] = value + if options.body or supplied_length is not None or options.method not in SAFE_METHODS: + request_headers["Content-Length"] = str(len(options.body)) + if proxy_parsed and scheme == "http" and proxy_auth: + request_headers["Proxy-Authorization"] = proxy_auth + for name, value in request_headers.items(): + try: + _validate_header(name, value) + except ValueError as exc: + raise ProbeFailure("request_validation", str(exc)) from exc + + target = _request_target(parsed, bool(proxy_parsed and scheme == "http")) + request_lines = [f"{options.method} {target} HTTP/1.1"] + request_lines.extend(f"{name}: {value}" for name, value in request_headers.items()) + header_bytes = ("\r\n".join(request_lines) + "\r\n\r\n").encode("iso-8859-1") + if len(header_bytes) > MAX_REQUEST_HEADER_BYTES: + raise ProbeFailure("request_validation", "request headers exceed 64 KiB") + request_bytes = header_bytes + options.body + + upload_started = time.perf_counter() + try: + remaining = hop_deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("total_timeout", "overall request timeout expired before upload") + sock.settimeout(min(options.connect_timeout, remaining)) + sock.sendall(request_bytes) + except (TimeoutError, OSError) as exc: + raise ProbeFailure("request_upload", _safe_error(exc)) from exc + uploaded_at = time.perf_counter() + timings["request_upload"] = monotonic_ms(upload_started, uploaded_at) + + remaining = hop_deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("total_timeout", "overall request timeout expired before response") + head, initial_body, first_response_at = _recv_header_block( + sock, + min(options.first_byte_timeout, remaining), + ) + status, _reason, response_headers = _parse_header_block(head) + interim_statuses: list[int] = [] + while 100 <= status < 200 and status != 101: + interim_statuses.append(status) + remaining = hop_deadline - time.perf_counter() + if remaining <= 0: + raise ProbeFailure("first_byte", "overall request timeout expired after interim response") + head, initial_body, _interim_first = _recv_header_block( + sock, + min(options.first_byte_timeout, remaining), + initial=initial_body, + ) + status, _reason, response_headers = _parse_header_block(head) + headers_complete_at = time.perf_counter() + timings["response_wait"] = monotonic_ms(uploaded_at, first_response_at) + timings["headers_complete_after_upload"] = monotonic_ms(uploaded_at, headers_complete_at) + + record["status_code"] = status + if interim_statuses: + record["interim_status_codes"] = interim_statuses + record["response"] = { + "headers": _safe_response_headers(response_headers), + "server_timing": parse_server_timing(response_headers.get("server-timing")), + "body_bytes_read": 0, + "body_complete": False, + "body_truncated_reason": None, + } + location = response_headers.get("location") + if location: + record["response"]["redirect_location"] = sanitize_url(urllib.parse.urljoin(url, location)) + + method_has_no_body = options.method == "HEAD" or status in {204, 304} or 100 <= status < 200 + content_type = response_headers.get("content-type", "").lower() + read_mode = options.read_mode + if read_mode == "auto": + read_mode = "first-byte" if "text/event-stream" in content_type else "full" + + body_started = time.perf_counter() + body_result = _read_response_body( + sock, + initial_body, + response_headers, + method_has_no_body=method_has_no_body, + read_mode=read_mode, + max_response_bytes=options.max_response_bytes, + idle_timeout=options.read_timeout, + first_body_timeout=options.first_byte_timeout, + deadline=hop_deadline, + ) + body_finished_at = time.perf_counter() + first_body_at = body_result.pop("first_body_at") + body_error_phase = body_result.pop("error_phase") + body_error = body_result.pop("error") + record["response"].update(body_result) + if first_body_at is not None: + timings["first_body_after_upload"] = monotonic_ms(uploaded_at, first_body_at) + else: + timings["first_body_after_upload"] = None + timings["body_read"] = monotonic_ms(body_started, body_finished_at) + timings["total"] = monotonic_ms(hop_started, body_finished_at) + if body_error_phase: + record["error_phase"] = body_error_phase + record["error"] = body_error + else: + record["success"] = True + except ProbeFailure as exc: + record["error_phase"] = exc.phase + record["error"] = str(exc)[:300] + record["timings_ms"]["total"] = monotonic_ms(hop_started) + except Exception as exc: # pragma: no cover - last-resort containment + record["error_phase"] = "unexpected" + record["error"] = _safe_error(exc) + record["timings_ms"]["total"] = monotonic_ms(hop_started) + finally: + if sock is not None: + with contextlib.suppress(OSError): + sock.close() + return record, location + + +def probe_url(url: str, options: ProbeOptions) -> dict[str, Any]: + """Probe one logical request, following redirects only when requested.""" + started = time.perf_counter() + current_url = url + current_method = options.method + current_body = options.body + current_headers = dict(options.headers) + hops: list[dict[str, Any]] = [] + visited: set[str] = set() + + for hop_index in range(options.max_redirects + 1): + remaining_total = options.total_timeout - (time.perf_counter() - started) + if remaining_total <= 0: + hops.append( + { + "url": sanitize_url(current_url), + "method": current_method, + "success": False, + "status_code": None, + "error_phase": "total_timeout", + "error": "overall logical request timeout expired", + "timings_ms": {"total": monotonic_ms(started)}, + "response": {}, + "hop": hop_index, + } + ) + break + hop_options = ProbeOptions( + method=current_method, + headers=current_headers, + body=current_body, + connect_timeout=options.connect_timeout, + first_byte_timeout=options.first_byte_timeout, + read_timeout=options.read_timeout, + total_timeout=remaining_total, + max_response_bytes=options.max_response_bytes, + read_mode=options.read_mode, + insecure=options.insecure, + ca_file=options.ca_file, + follow_redirects=options.follow_redirects, + max_redirects=options.max_redirects, + proxy=options.proxy, + use_env_proxy=options.use_env_proxy, + ip_family=options.ip_family, + ) + hop, raw_location = _probe_hop(current_url, hop_options) + hop["hop"] = hop_index + hops.append(hop) + status = hop.get("status_code") + if not hop.get("success") or not options.follow_redirects or status not in {301, 302, 303, 307, 308}: + break + if not raw_location: + break + next_url = urllib.parse.urljoin(current_url, raw_location) + try: + next_origin = _origin_tuple(next_url) + except ValueError as exc: + hop["success"] = False + hop["error_phase"] = "redirect" + hop["error"] = f"invalid redirect target: {_safe_error(exc)}" + break + if next_url in visited: + hop["success"] = False + hop["error_phase"] = "redirect" + hop["error"] = "redirect loop detected" + break + visited.add(current_url) + if hop_index >= options.max_redirects: + hop["success"] = False + hop["error_phase"] = "redirect" + hop["error"] = "maximum redirects exceeded" + break + if _origin_tuple(current_url) != next_origin: + hop["success"] = False + hop["error_phase"] = "redirect" + hop["error"] = "cross-origin redirect refused to prevent credential or request-body disclosure" + break + if status == 303 or (status in {301, 302} and current_method == "POST"): + current_method = "GET" + current_body = b"" + current_headers = {name: value for name, value in current_headers.items() if name.lower() not in {"content-length", "content-type"}} + current_url = next_url + + final = hops[-1] + aggregate_timings: dict[str, float | None] = {} + for key in { + "dns", + "tcp_connect", + "proxy_connect", + "tls_handshake", + "request_upload", + "response_wait", + "headers_complete_after_upload", + "body_read", + }: + values = [hop.get("timings_ms", {}).get(key) for hop in hops] + clean = [float(value) for value in values if value is not None] + aggregate_timings[key] = round(sum(clean), 3) if clean else None + aggregate_timings["total"] = monotonic_ms(started) + return { + "url": sanitize_url(url), + "success": bool(final.get("success")), + "status_code": final.get("status_code"), + "error_phase": final.get("error_phase"), + "error": final.get("error"), + "timings_ms": aggregate_timings, + "hops": hops, + } + + +def summarize_probe_results(results: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + successful = [result for result in results if result.get("success")] + status_counts = Counter(str(result.get("status_code")) for result in successful) + error_counts = Counter(str(result.get("error_phase") or "unknown") for result in results if not result.get("success")) + timing_keys = [ + "dns", + "tcp_connect", + "proxy_connect", + "tls_handshake", + "request_upload", + "response_wait", + "headers_complete_after_upload", + "body_read", + "total", + ] + timings = {key: summarize_numbers(result.get("timings_ms", {}).get(key) for result in successful) for key in timing_keys} + return { + "requests": len(results), + "successful_transport": len(successful), + "transport_errors": len(results) - len(successful), + "status_counts": dict(status_counts), + "error_phase_counts": dict(error_counts), + "timings_ms": timings, + } + + +def run_probe_batch( + url: str, + options: ProbeOptions, + *, + count: int, + concurrency: int, + interval: float = 0.0, +) -> tuple[list[dict[str, Any]], float]: + """Run a bounded number of fresh-connection probes.""" + count = max(1, int(count)) + concurrency = max(1, min(int(concurrency), count)) + started = time.perf_counter() + results: list[dict[str, Any]] = [] + if concurrency == 1: + for index in range(count): + results.append(probe_url(url, options)) + if interval > 0 and index + 1 < count: + time.sleep(interval) + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(probe_url, url, options) for _ in range(count)] + for future in concurrent.futures.as_completed(futures): + results.append(future.result()) + return results, monotonic_ms(started) + + +def validate_replay_safety( + *, + method: str, + url_count: int, + count: int, + concurrency: int, + follow_redirects: bool, + allow_unsafe_replay: bool, +) -> None: + method = method.upper() + if method in SAFE_METHODS: + return + total = max(1, url_count) * max(1, count) + replayed = total > 1 or concurrency > 1 or follow_redirects + if replayed and not allow_unsafe_replay: + raise ValueError( + "Unsafe method would be replayed. Login can invalidate an old session and failed login " + "can trigger IP lockout; agent POSTs can create duplicate runs. Use exactly one URL, " + "--count 1, --concurrency 1 and no redirect following, or explicitly add " + "--allow-unsafe-replay after reviewing the impact." + ) + + +def _parse_scalar(value: str) -> Any: + raw = value.strip().strip("\"'") + lowered = raw.lower() + if lowered in {"true", "yes", "on"}: + return True + if lowered in {"false", "no", "off"}: + return False + if lowered in {"null", "none", "~", ""}: + return None + try: + return int(raw) + except ValueError: + try: + return float(raw) + except ValueError: + return raw + + +def read_safe_config(path: Path) -> dict[str, Any]: + """Read a small allowlist of non-secret scalar YAML settings.""" + wanted = { + "gateway.port", + "database.backend", + "database.pool_size", + "database.max_overflow", + "database.pool_timeout", + "database.pool_recycle", + "database.sqlite_dir", + "runtime_performance.skill_view_reuse_enabled", + "runtime_performance.skill_view_cache_enabled", + "runtime_performance.model_timing_nonblocking_enabled", + "runtime_performance.asyncio_thread_pool_enabled", + "runtime_performance.asyncio_thread_pool_size", + "concurrency.redis.enabled", + "concurrency.redis.socket_connect_timeout_ms", + "concurrency.redis.socket_timeout_ms", + "concurrency.redis.acquire_timeout_ms", + "concurrency.redis.fail_open", + } + output: dict[str, Any] = {} + stack: list[tuple[int, str]] = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError as exc: + return {"path": str(path), "error": _safe_error(exc)} + for raw_line in lines: + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip() or line.lstrip().startswith("-") or ":" not in line: + continue + indent = len(line) - len(line.lstrip(" \t")) + key, _separator, value = line.strip().partition(":") + while stack and stack[-1][0] >= indent: + stack.pop() + dotted = ".".join([item[1] for item in stack] + [key]) + if value.strip(): + if dotted in wanted: + output[dotted] = _parse_scalar(value) + else: + stack.append((indent, key)) + output["path"] = str(path) + return output + + +def find_config_path(explicit: str | None = None) -> Path | None: + candidates: list[Path] = [] + if explicit: + candidates.append(Path(explicit)) + for env_name in ("DEER_FLOW_CONFIG_PATH", "DEER_FLOW_CONFIG", "DEERFLOW_CONFIG"): + if os.environ.get(env_name): + candidates.append(Path(os.environ[env_name])) + script_backend = Path(__file__).resolve().parent.parent + candidates.extend( + [ + Path.cwd() / "config.yaml", + script_backend / "config.yaml", + Path("/app/backend/config.yaml"), + ] + ) + seen: set[Path] = set() + for candidate in candidates: + with contextlib.suppress(OSError): + resolved = candidate.expanduser().resolve() + if resolved in seen: + continue + seen.add(resolved) + if resolved.is_file(): + return resolved + return None + + +def _runtime_home_candidates(config_path: Path | None) -> list[Path]: + candidates: list[Path] = [] + if os.environ.get("DEER_FLOW_HOME"): + candidates.append(Path(os.environ["DEER_FLOW_HOME"])) + if config_path: + candidates.append(config_path.parent / ".deer-flow") + candidates.extend([Path.cwd() / ".deer-flow", Path("/app/backend/.deer-flow")]) + return candidates + + +def read_safe_system_settings(config_path: Path | None) -> dict[str, Any]: + path = next( + (candidate / "system_settings.json" for candidate in _runtime_home_candidates(config_path) if (candidate / "system_settings.json").is_file()), + None, + ) + if path is None: + return {"found": False} + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {"found": True, "path": str(path), "error": _safe_error(exc)} + model = raw.get("model_concurrency") if isinstance(raw, dict) else {} + monitor = raw.get("concurrency_monitor") if isinstance(raw, dict) else {} + if not isinstance(model, dict): + model = {} + if not isinstance(monitor, dict): + monitor = {} + safe_limits = {str(name)[:120]: int(value) for name, value in (model.get("model_limits") or {}).items() if isinstance(value, int) and value > 0} + safe_user_limits = {str(name)[:120]: int(value) for name, value in (model.get("user_model_limits") or {}).items() if isinstance(value, int) and value > 0} + return { + "found": True, + "path": str(path), + "concurrency_monitor_enabled": bool(monitor.get("enabled", False)), + "model_concurrency": { + "enabled": bool(model.get("enabled", False)), + "default_limit": model.get("default_limit"), + "user_default_limit": model.get("user_default_limit"), + "model_limits": safe_limits, + "user_model_limits": safe_user_limits, + }, + } + + +def _run_command( + args: Sequence[str], + *, + timeout: float = 8.0, + env: Mapping[str, str] | None = None, +) -> dict[str, Any]: + started = time.perf_counter() + try: + completed = subprocess.run( + list(args), + check=False, + capture_output=True, + text=False, + timeout=timeout, + env=dict(env) if env is not None else None, + ) + except FileNotFoundError: + return {"available": False, "returncode": None, "elapsed_ms": monotonic_ms(started), "output": b""} + except subprocess.TimeoutExpired as exc: + output = (exc.stdout or b"") + (exc.stderr or b"") + return { + "available": True, + "returncode": None, + "timed_out": True, + "elapsed_ms": monotonic_ms(started), + "output": output[:MAX_COMMAND_OUTPUT], + } + return { + "available": True, + "returncode": completed.returncode, + "timed_out": False, + "elapsed_ms": monotonic_ms(started), + "output": (completed.stdout + completed.stderr)[:MAX_COMMAND_OUTPUT], + } + + +def _read_proc_stat() -> dict[str, int]: + try: + first = Path("/proc/stat").read_text(encoding="ascii").splitlines()[0].split() + except (OSError, IndexError): + return {} + if not first or first[0] != "cpu": + return {} + names = ["user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice"] + values: dict[str, int] = {} + for name, raw in zip(names, first[1:]): + with contextlib.suppress(ValueError): + values[name] = int(raw) + return values + + +def _cpu_delta(before: Mapping[str, int], after: Mapping[str, int]) -> dict[str, float | None]: + keys = set(before) | set(after) + deltas = {key: max(0, int(after.get(key, 0)) - int(before.get(key, 0))) for key in keys} + total = sum(deltas.values()) - deltas.get("guest", 0) - deltas.get("guest_nice", 0) + if total <= 0: + return {"busy_pct": None, "iowait_pct": None, "steal_pct": None} + idle = deltas.get("idle", 0) + return { + "busy_pct": round(100.0 * (total - idle) / total, 2), + "iowait_pct": round(100.0 * deltas.get("iowait", 0) / total, 2), + "steal_pct": round(100.0 * deltas.get("steal", 0) / total, 2), + } + + +def _read_meminfo() -> dict[str, int]: + values: dict[str, int] = {} + try: + lines = Path("/proc/meminfo").read_text(encoding="ascii").splitlines() + except OSError: + return values + for line in lines: + key, separator, rest = line.partition(":") + if not separator: + continue + match = re.search(r"\d+", rest) + if match: + values[key] = int(match.group()) * 1024 + return values + + +def _read_vmstat() -> dict[str, int]: + output: dict[str, int] = {} + try: + lines = Path("/proc/vmstat").read_text(encoding="ascii").splitlines() + except OSError: + return output + wanted = {"pswpin", "pswpout", "pgmajfault", "oom_kill"} + for line in lines: + key, _separator, raw = line.partition(" ") + if key not in wanted: + continue + with contextlib.suppress(ValueError): + output[key] = int(raw.strip()) + return output + + +def _read_pressure() -> dict[str, Any]: + output: dict[str, Any] = {} + for resource in ("cpu", "memory", "io"): + path = Path("/proc/pressure") / resource + if not path.is_file(): + continue + resource_values: dict[str, dict[str, float | int]] = {} + with contextlib.suppress(OSError): + for line in path.read_text(encoding="ascii").splitlines(): + parts = line.split() + if not parts: + continue + metrics: dict[str, float | int] = {} + for part in parts[1:]: + key, separator, raw = part.partition("=") + if not separator: + continue + try: + metrics[key] = int(raw) if key == "total" else float(raw) + except ValueError: + pass + resource_values[parts[0]] = metrics + if resource_values: + output[resource] = resource_values + return output + + +def _read_diskstats() -> dict[str, dict[str, int]]: + output: dict[str, dict[str, int]] = {} + try: + lines = Path("/proc/diskstats").read_text(encoding="ascii").splitlines() + except OSError: + return output + for line in lines: + parts = line.split() + if len(parts) < 14: + continue + name = parts[2] + if not (Path("/sys/block") / name).exists(): + continue + try: + output[name] = { + "reads": int(parts[3]), + "sectors_read": int(parts[5]), + "writes": int(parts[7]), + "sectors_written": int(parts[9]), + "io_ms": int(parts[12]), + } + except ValueError: + continue + return output + + +def _disk_delta( + before: Mapping[str, Mapping[str, int]], + after: Mapping[str, Mapping[str, int]], + elapsed_seconds: float, +) -> dict[str, Any]: + devices: list[dict[str, Any]] = [] + for name in sorted(set(before) & set(after)): + old = before[name] + new = after[name] + read_bytes = max(0, new["sectors_read"] - old["sectors_read"]) * 512 + write_bytes = max(0, new["sectors_written"] - old["sectors_written"]) * 512 + io_ms = max(0, new["io_ms"] - old["io_ms"]) + devices.append( + { + "device": name, + "read_bytes_per_sec": round(read_bytes / max(elapsed_seconds, 0.001), 1), + "write_bytes_per_sec": round(write_bytes / max(elapsed_seconds, 0.001), 1), + "util_pct": round(min(100.0, io_ms / (elapsed_seconds * 10.0)), 2), + "reads": max(0, new["reads"] - old["reads"]), + "writes": max(0, new["writes"] - old["writes"]), + } + ) + devices.sort(key=lambda item: item["util_pct"], reverse=True) + return { + "devices": devices[:12], + "max_util_pct": devices[0]["util_pct"] if devices else None, + "busiest_device": devices[0]["device"] if devices else None, + } + + +def _read_netdev() -> dict[str, dict[str, int]]: + output: dict[str, dict[str, int]] = {} + try: + lines = Path("/proc/net/dev").read_text(encoding="ascii").splitlines()[2:] + except OSError: + return output + for line in lines: + name, separator, raw = line.partition(":") + if not separator: + continue + interface = name.strip() + fields = raw.split() + if len(fields) < 16: + continue + with contextlib.suppress(ValueError): + output[interface] = { + "rx_bytes": int(fields[0]), + "rx_errors": int(fields[2]), + "rx_dropped": int(fields[3]), + "tx_bytes": int(fields[8]), + "tx_errors": int(fields[10]), + "tx_dropped": int(fields[11]), + } + return output + + +def _net_delta( + before: Mapping[str, Mapping[str, int]], + after: Mapping[str, Mapping[str, int]], + elapsed_seconds: float, +) -> dict[str, Any]: + totals = Counter() + for name in set(before) & set(after): + if name == "lo": + continue + for key in before[name]: + totals[key] += max(0, int(after[name].get(key, 0)) - int(before[name].get(key, 0))) + return { + "rx_bytes_per_sec": round(totals["rx_bytes"] / max(elapsed_seconds, 0.001), 1), + "tx_bytes_per_sec": round(totals["tx_bytes"] / max(elapsed_seconds, 0.001), 1), + "rx_errors_delta": totals["rx_errors"], + "tx_errors_delta": totals["tx_errors"], + "rx_dropped_delta": totals["rx_dropped"], + "tx_dropped_delta": totals["tx_dropped"], + } + + +def _read_cgroup_number(path: str) -> int | None: + try: + raw = Path(path).read_text(encoding="ascii").strip() + except OSError: + return None + if raw == "max": + return None + with contextlib.suppress(ValueError): + return int(raw) + return None + + +def _read_cgroup_limits() -> dict[str, Any]: + output: dict[str, Any] = {} + if Path("/sys/fs/cgroup/cgroup.controllers").exists(): + memory_current = _read_cgroup_number("/sys/fs/cgroup/memory.current") + memory_max = _read_cgroup_number("/sys/fs/cgroup/memory.max") + pids_current = _read_cgroup_number("/sys/fs/cgroup/pids.current") + pids_max = _read_cgroup_number("/sys/fs/cgroup/pids.max") + cpu_quota: float | None = None + with contextlib.suppress(OSError, ValueError): + quota_raw, period_raw = Path("/sys/fs/cgroup/cpu.max").read_text(encoding="ascii").split()[:2] + if quota_raw != "max": + cpu_quota = round(int(quota_raw) / int(period_raw), 3) + output = { + "version": 2, + "memory_current_bytes": memory_current, + "memory_limit_bytes": memory_max, + "memory_usage_pct": (round(100.0 * memory_current / memory_max, 2) if memory_current is not None and memory_max and memory_max > 0 else None), + "cpu_limit_cores": cpu_quota, + "pids_current": pids_current, + "pids_limit": pids_max, + } + return output + + +def _read_process_stat(pid: int) -> tuple[int, int] | None: + try: + raw = (Path("/proc") / str(pid) / "stat").read_text(encoding="ascii") + after_name = raw.rsplit(")", 1)[1].split() + return int(after_name[11]), int(after_name[12]) + except (OSError, IndexError, ValueError): + return None + + +def _classify_process(cmdline: str, name: str) -> str | None: + lowered = f"{name} {cmdline}".lower() + if "nginx" in lowered: + return "nginx" + if any(token in lowered for token in ("mysqld", "mariadbd")): + return "mysql" + if any(token in lowered for token in ("uvicorn", "gunicorn", "deerflow", "app.gateway")): + return "deerflow_gateway" + return None + + +def _scan_relevant_processes() -> dict[int, dict[str, Any]]: + output: dict[int, dict[str, Any]] = {} + for path in Path("/proc").iterdir(): + if not path.name.isdigit(): + continue + pid = int(path.name) + try: + cmdline = (path / "cmdline").read_bytes().replace(b"\x00", b" ").decode(errors="replace") + status_lines = (path / "status").read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + continue + status: dict[str, str] = {} + for line in status_lines: + key, separator, value = line.partition(":") + if separator: + status[key] = value.strip() + role = _classify_process(cmdline, status.get("Name", "")) + if role is None: + continue + fd_count: int | None = None + with contextlib.suppress(OSError): + fd_count = len(list((path / "fd").iterdir())) + fd_limit: int | None = None + with contextlib.suppress(OSError): + for line in (path / "limits").read_text(encoding="utf-8").splitlines(): + if line.startswith("Max open files"): + fields = line.split() + if len(fields) >= 4 and fields[3].isdigit(): + fd_limit = int(fields[3]) + rss_kib = 0 + match = re.search(r"\d+", status.get("VmRSS", "")) + if match: + rss_kib = int(match.group()) + output[pid] = { + "pid": pid, + "role": role, + "rss_bytes": rss_kib * 1024, + "threads": int(status.get("Threads", "0") or 0), + "fd_count": fd_count, + "fd_limit": fd_limit, + "cpu_ticks": _read_process_stat(pid), + } + return output + + +def _process_delta( + before: Mapping[int, Mapping[str, Any]], + after: Mapping[int, Mapping[str, Any]], + elapsed_seconds: float, +) -> dict[str, Any]: + tick_rate = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100 + grouped: dict[str, dict[str, Any]] = defaultdict( + lambda: { + "processes": 0, + "rss_bytes": 0, + "threads": 0, + "fd_count": 0, + "fd_limit_min": None, + "cpu_pct_sum": 0.0, + "max_fd_usage_pct": None, + } + ) + for pid, item in after.items(): + role = str(item["role"]) + group = grouped[role] + group["processes"] += 1 + group["rss_bytes"] += int(item.get("rss_bytes") or 0) + group["threads"] += int(item.get("threads") or 0) + fd_count = item.get("fd_count") + fd_limit = item.get("fd_limit") + if isinstance(fd_count, int): + group["fd_count"] += fd_count + if isinstance(fd_limit, int) and fd_limit > 0: + current_min = group["fd_limit_min"] + group["fd_limit_min"] = fd_limit if current_min is None else min(current_min, fd_limit) + if isinstance(fd_count, int): + usage = round(100.0 * fd_count / fd_limit, 2) + current_usage = group["max_fd_usage_pct"] + group["max_fd_usage_pct"] = usage if current_usage is None else max(current_usage, usage) + old = before.get(pid, {}).get("cpu_ticks") + new = item.get("cpu_ticks") + if old and new: + ticks = max(0, (new[0] + new[1]) - (old[0] + old[1])) + group["cpu_pct_sum"] += 100.0 * ticks / max(tick_rate * elapsed_seconds, 0.001) + for group in grouped.values(): + group["cpu_pct_sum"] = round(group["cpu_pct_sum"], 2) + return dict(grouped) + + +def collect_linux_snapshot(sample_seconds: float = 1.0) -> dict[str, Any]: + if platform.system() != "Linux" or not Path("/proc").is_dir(): + return {"supported": False, "reason": "server checks require Linux /proc"} + sample_seconds = max(0.25, min(float(sample_seconds), 5.0)) + cpu_before = _read_proc_stat() + disk_before = _read_diskstats() + net_before = _read_netdev() + vm_before = _read_vmstat() + process_before = _scan_relevant_processes() + started = time.perf_counter() + time.sleep(sample_seconds) + elapsed = time.perf_counter() - started + cpu_after = _read_proc_stat() + disk_after = _read_diskstats() + net_after = _read_netdev() + vm_after = _read_vmstat() + process_after = _scan_relevant_processes() + + mem = _read_meminfo() + total_memory = mem.get("MemTotal", 0) + available_memory = mem.get("MemAvailable", 0) + swap_total = mem.get("SwapTotal", 0) + swap_free = mem.get("SwapFree", 0) + load_values: tuple[float, float, float] | tuple[()] = () + with contextlib.suppress(OSError): + load_values = os.getloadavg() + file_nr: list[int] = [] + with contextlib.suppress(OSError, ValueError): + file_nr = [int(item) for item in Path("/proc/sys/fs/file-nr").read_text().split()] + file_max = _read_cgroup_number("/proc/sys/fs/file-max") + + disk_paths: dict[str, Any] = {} + for path in (Path("/"), Path.cwd()): + try: + usage = shutil.disk_usage(path) + except OSError: + continue + disk_paths[str(path)] = { + "total_bytes": usage.total, + "free_bytes": usage.free, + "free_pct": round(100.0 * usage.free / usage.total, 2) if usage.total else None, + } + + return { + "supported": True, + "sample_seconds": round(elapsed, 3), + "hostname_hash": hashlib.sha256(socket.gethostname().encode()).hexdigest()[:12], + "kernel": platform.release(), + "cpu_count": os.cpu_count(), + "load_average": list(load_values), + "cpu": _cpu_delta(cpu_before, cpu_after), + "memory": { + "total_bytes": total_memory, + "available_bytes": available_memory, + "available_pct": round(100.0 * available_memory / total_memory, 2) if total_memory else None, + "swap_total_bytes": swap_total, + "swap_used_bytes": max(0, swap_total - swap_free), + "swap_used_pct": round(100.0 * (swap_total - swap_free) / swap_total, 2) if swap_total else 0.0, + "dirty_bytes": mem.get("Dirty", 0), + }, + "vm_delta": {key: max(0, vm_after.get(key, 0) - vm_before.get(key, 0)) for key in set(vm_before) | set(vm_after)}, + "pressure": _read_pressure(), + "disk_io": _disk_delta(disk_before, disk_after, elapsed), + "disk_space": disk_paths, + "network": _net_delta(net_before, net_after, elapsed), + "processes": _process_delta(process_before, process_after, elapsed), + "cgroup": _read_cgroup_limits(), + "file_handles": { + "allocated": file_nr[0] if file_nr else None, + "unused": file_nr[1] if len(file_nr) > 1 else None, + "system_limit": file_max or (file_nr[2] if len(file_nr) > 2 else None), + }, + } + + +def _port_from_address(value: str) -> int | None: + if value.startswith("[") and "]:" in value: + raw = value.rsplit(":", 1)[-1] + elif ":" in value: + raw = value.rsplit(":", 1)[-1] + else: + return None + with contextlib.suppress(ValueError): + return int(raw) + return None + + +def collect_socket_snapshot(relevant_ports: set[int]) -> dict[str, Any]: + if shutil.which("ss") is None: + return {"available": False, "reason": "ss command not found"} + listeners_result = _run_command(["ss", "-H", "-lnt"], timeout=5) + connections_result = _run_command(["ss", "-H", "-tan"], timeout=8) + listeners: list[dict[str, Any]] = [] + if listeners_result.get("returncode") == 0: + for raw_line in listeners_result["output"].decode(errors="replace").splitlines(): + parts = raw_line.split() + if len(parts) < 5: + continue + port = _port_from_address(parts[3]) + if port is None or (relevant_ports and port not in relevant_ports): + continue + try: + receive_queue = int(parts[1]) + backlog = int(parts[2]) + except ValueError: + continue + listeners.append( + { + "port": port, + "receive_queue": receive_queue, + "backlog": backlog, + "backlog_usage_pct": round(100.0 * receive_queue / backlog, 2) if backlog else None, + } + ) + global_states: Counter[str] = Counter() + by_port: dict[int, Counter[str]] = defaultdict(Counter) + if connections_result.get("returncode") == 0: + for raw_line in connections_result["output"].decode(errors="replace").splitlines(): + parts = raw_line.split() + if len(parts) < 5: + continue + state = parts[0].upper() + global_states[state] += 1 + local_port = _port_from_address(parts[3]) + peer_port = _port_from_address(parts[4]) + for port in relevant_ports: + if port in {local_port, peer_port}: + by_port[port][state] += 1 + return { + "available": bool(listeners_result.get("available") and connections_result.get("available")), + "listeners": listeners, + "global_states": dict(global_states), + "states_by_relevant_port": {str(port): dict(states) for port, states in by_port.items()}, + } + + +def _parse_nginx_config(text: str, *, source: str) -> dict[str, Any]: + def _last_value(pattern: str) -> str | None: + matches = re.findall(pattern, text, flags=re.IGNORECASE | re.MULTILINE) + return matches[-1] if matches else None + + worker_processes_raw = _last_value(r"^\s*worker_processes\s+([^;\s]+)\s*;") + worker_connections_raw = _last_value(r"^\s*worker_connections\s+(\d+)\s*;") + worker_rlimit_raw = _last_value(r"^\s*worker_rlimit_nofile\s+(\d+)\s*;") + keepalive_timeout_raw = _last_value(r"^\s*keepalive_timeout\s+([^;]+)\s*;") + multi_accept_raw = _last_value(r"^\s*multi_accept\s+([^;\s]+)\s*;") + workers: int | None + if worker_processes_raw == "auto": + workers = os.cpu_count() + else: + try: + workers = int(worker_processes_raw) if worker_processes_raw else None + except ValueError: + workers = None + worker_connections = int(worker_connections_raw) if worker_connections_raw else 512 + theoretical = workers * worker_connections if workers else None + return { + "source": source, + "worker_processes_config": worker_processes_raw, + "worker_processes_resolved": workers, + "worker_connections": worker_connections, + "worker_rlimit_nofile": int(worker_rlimit_raw) if worker_rlimit_raw else None, + "theoretical_connection_slots": theoretical, + "reverse_proxy_estimated_client_ceiling": theoretical // 2 if theoretical else None, + "multi_accept": multi_accept_raw, + "keepalive_timeout": keepalive_timeout_raw[:80] if keepalive_timeout_raw else None, + "limit_conn_directives": len(re.findall(r"\blimit_conn\s+", text)), + "upstream_keepalive_directives": len(re.findall(r"\bkeepalive\s+\d+\s*;", text)), + "proxy_connect_timeout": _last_value(r"^\s*proxy_connect_timeout\s+([^;]+)\s*;"), + "proxy_read_timeout": _last_value(r"^\s*proxy_read_timeout\s+([^;]+)\s*;"), + "proxy_send_timeout": _last_value(r"^\s*proxy_send_timeout\s+([^;]+)\s*;"), + } + + +def collect_nginx_config(container_ids: Sequence[str] = ()) -> dict[str, Any]: + candidates: list[dict[str, Any]] = [] + if shutil.which("nginx"): + result = _run_command(["nginx", "-T"], timeout=8) + if result.get("returncode") == 0: + candidates.append(_parse_nginx_config(result["output"].decode(errors="replace"), source="host:nginx -T")) + if shutil.which("docker"): + for container_id in list(container_ids)[:5]: + result = _run_command(["docker", "exec", container_id, "nginx", "-T"], timeout=8) + if result.get("returncode") == 0: + candidates.append( + _parse_nginx_config( + result["output"].decode(errors="replace"), + source=f"container:{container_id[:12]}", + ) + ) + if not candidates: + return {"available": False, "reason": "no readable running Nginx configuration"} + return {"available": True, "instances": candidates} + + +def collect_docker_snapshot() -> dict[str, Any]: + if shutil.which("docker") is None: + return {"available": False, "reason": "docker command not found"} + listing = _run_command( + ["docker", "ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}"], + timeout=8, + ) + if listing.get("returncode") != 0: + return {"available": False, "reason": "docker ps failed"} + containers: list[dict[str, Any]] = [] + relevant_ids: list[str] = [] + nginx_ids: list[str] = [] + for line in listing["output"].decode(errors="replace").splitlines(): + parts = line.split("\t", 3) + if len(parts) != 4: + continue + container_id, name, image, status = parts + lowered = f"{name} {image}".lower() + if not any(token in lowered for token in ("deerflow", "gateway", "backend", "nginx", "frontend", "mysql", "mariadb")): + continue + role = "other" + if any(token in lowered for token in ("mysql", "mariadb")): + role = "mysql" + elif "nginx" in lowered or "frontend" in lowered: + role = "nginx_or_frontend" + nginx_ids.append(container_id) + elif any(token in lowered for token in ("deerflow", "gateway", "backend")): + role = "deerflow_gateway" + item = { + "id": container_id[:12], + "name": name[:120], + "image": image[:180], + "role": role, + "status": status[:160], + } + containers.append(item) + relevant_ids.append(container_id) + + stats_by_name: dict[str, dict[str, Any]] = {} + if relevant_ids: + stats = _run_command( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}\t{{.NetIO}}\t{{.BlockIO}}", + *relevant_ids, + ], + timeout=15, + ) + if stats.get("returncode") == 0: + for line in stats["output"].decode(errors="replace").splitlines(): + fields = line.split("\t") + if len(fields) != 7: + continue + name, cpu, memory, memory_pct, pids, network, block = fields + stats_by_name[name] = { + "cpu_pct": _parse_percent(cpu), + "memory_usage": memory[:100], + "memory_pct": _parse_percent(memory_pct), + "pids": int(pids) if pids.isdigit() else None, + "network_io": network[:100], + "block_io": block[:100], + } + for container, container_id in zip(containers, relevant_ids): + container["stats"] = stats_by_name.get(container["name"], {}) + inspect = _run_command( + [ + "docker", + "inspect", + "--format", + "{{.HostConfig.Memory}}\t{{.HostConfig.NanoCpus}}\t{{.HostConfig.PidsLimit}}\t{{.State.OOMKilled}}\t{{.RestartCount}}", + container_id, + ], + timeout=5, + ) + if inspect.get("returncode") == 0: + fields = inspect["output"].decode(errors="replace").strip().split("\t") + if len(fields) == 5: + memory_limit, nano_cpus, pids_limit, oom_killed, restart_count = fields + container["limits"] = { + "memory_bytes": int(memory_limit) if memory_limit.isdigit() and int(memory_limit) > 0 else None, + "cpu_cores": round(int(nano_cpus) / 1_000_000_000, 3) if nano_cpus.isdigit() and int(nano_cpus) > 0 else None, + "pids": int(pids_limit) if re.fullmatch(r"-?\d+", pids_limit) and int(pids_limit) > 0 else None, + "oom_killed": oom_killed.lower() == "true", + "restart_count": int(restart_count) if restart_count.isdigit() else None, + } + return {"available": True, "containers": containers, "nginx_container_ids": nginx_ids} + + +def _parse_percent(raw: str) -> float | None: + try: + return round(float(raw.strip().rstrip("%")), 2) + except ValueError: + return None + + +LOG_PATTERNS: dict[str, re.Pattern[str]] = { + "nginx_connection_capacity": re.compile(r"worker_connections are not enough", re.IGNORECASE), + "too_many_open_files": re.compile(r"too many open files", re.IGNORECASE), + "nginx_upstream_timeout": re.compile(r"upstream timed out", re.IGNORECASE), + "nginx_upstream_connect_failure": re.compile(r"connect\(\) failed|no live upstreams|upstream.*connection refused", re.IGNORECASE), + "db_pool_timeout": re.compile(r"QueuePool limit|connection pool.*(?:exhaust|timeout)|pool timeout", re.IGNORECASE), + "db_lock_or_deadlock": re.compile(r"lock wait timeout|deadlock found|database is locked", re.IGNORECASE), + "db_disconnect": re.compile(r"server has gone away|lost connection.*mysql|connection.*closed|connection reset", re.IGNORECASE), + "deerflow_concurrency_reject": re.compile(r"reached the concurrency limit|selected model is busy|ModelConcurrencyLimit", re.IGNORECASE), + "resource_exhaustion": re.compile(r"cannot allocate memory|out of memory|oomkilled|resource temporarily unavailable", re.IGNORECASE), + "gateway_timeout": re.compile(r"504 Gateway Time-out|Gateway Timeout", re.IGNORECASE), + "skill_view_timing": re.compile(r"SKILL_VIEW_CACHE", re.IGNORECASE), + "start_run_timing": re.compile(r"\[start-run-timing\]", re.IGNORECASE), + "operation_timing": re.compile(r"GATEWAY_OPERATION_TIMING", re.IGNORECASE), + "llm_request_timing": re.compile(r"LLM_REQUEST_(?:DONE|FAILED)", re.IGNORECASE), +} + + +def _tail_bytes(path: Path, maximum: int) -> bytes: + with path.open("rb") as handle: + handle.seek(0, os.SEEK_END) + size = handle.tell() + handle.seek(max(0, size - maximum)) + return handle.read(maximum) + + +def discover_log_paths(explicit: Sequence[str], config_path: Path | None) -> list[Path]: + candidates = [Path(path) for path in explicit] + roots = [Path.cwd(), Path(__file__).resolve().parents[2]] + if config_path: + roots.append(config_path.parent) + for root in roots: + candidates.extend( + [ + root / ".runtime/logs/backend.log", + root / ".runtime/logs/gateway.log", + root / "logs/backend.log", + root / "logs/gateway.log", + ] + ) + for pattern in ("/var/log/nginx/*.log", "/var/log/mysql/*.log", "/var/log/mariadb/*.log"): + candidates.extend(Path(item) for item in glob.glob(pattern)) + output: list[Path] = [] + seen: set[Path] = set() + for candidate in candidates: + with contextlib.suppress(OSError): + resolved = candidate.expanduser().resolve() + if resolved not in seen and resolved.is_file(): + seen.add(resolved) + output.append(resolved) + return output + + +def scan_logs(paths: Sequence[Path], tail_bytes: int = 4 * 1024 * 1024) -> dict[str, Any]: + files: list[dict[str, Any]] = [] + totals: Counter[str] = Counter() + timing_max: dict[str, float] = {} + for path in paths: + try: + text = _tail_bytes(path, tail_bytes).decode("utf-8", errors="replace") + stat = path.stat() + except OSError as exc: + files.append({"path": str(path), "error": _safe_error(exc)}) + continue + counts: dict[str, int] = {} + for name, pattern in LOG_PATTERNS.items(): + count = len(pattern.findall(text)) + if count: + counts[name] = count + totals[name] += count + numeric_patterns = { + "skill_view_total_ms": r"SKILL_VIEW_CACHE[^\n]*\btotal_ms=([0-9.]+)", + "agent_build_elapsed_ms": r"AGENT_BUILD_TIMING[^\n]*\belapsed_ms=([0-9.]+)", + "llm_total_ms": r"LLM_REQUEST_(?:DONE|FAILED)[^\n]*\btotal_ms=([0-9.]+)", + "start_run_total_seconds": r"\[start-run-timing\][^\n]*\btotal=([0-9.]+)s", + } + for name, pattern in numeric_patterns.items(): + values = [] + for raw in re.findall(pattern, text, flags=re.IGNORECASE): + with contextlib.suppress(ValueError): + values.append(float(raw)) + if values: + timing_max[name] = max(timing_max.get(name, 0.0), max(values)) + files.append( + { + "path": str(path), + "bytes_scanned": min(stat.st_size, tail_bytes), + "modified_at": dt.datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(), + "pattern_counts": counts, + } + ) + return {"files": files, "pattern_counts": dict(totals), "timing_max": timing_max} + + +def collect_catalogue_scan(config_path: Path | None) -> dict[str, Any]: + if config_path is None: + return {"available": False, "reason": "config.yaml not found"} + runtime_home = next((path for path in _runtime_home_candidates(config_path) if path.exists()), config_path.parent / ".deer-flow") + skills_root = Path(os.environ.get("DEER_FLOW_SKILLS_PATH", str(config_path.parent / "skills"))) + agents_root = Path(os.environ.get("DEER_FLOW_AGENTS_PATH", str(runtime_home / "agents"))) + output: dict[str, Any] = {"available": True} + for label, root, names in ( + ("skills", skills_root, {"SKILL.md", "config.yaml"}), + ("agents", agents_root, {"config.yaml", "SOUL.md"}), + ): + started = time.perf_counter() + directories = 0 + files = 0 + bytes_total = 0 + errors = 0 + cap_reason: str | None = None + deadline = time.perf_counter() + 3.0 + if root.is_dir(): + try: + for current_root, dirnames, filenames in os.walk(root): + if time.perf_counter() >= deadline: + dirnames.clear() + cap_reason = "3s_deadline" + break + directories += len(dirnames) + if directories >= 20_000: + dirnames.clear() + cap_reason = "20000_directories" + break + for filename in filenames: + if filename not in names and not (label == "skills" and filename.endswith((".md", ".py"))): + continue + files += 1 + with contextlib.suppress(OSError): + bytes_total += (Path(current_root) / filename).stat().st_size + if files >= 100_000: + cap_reason = "100000_files" + break + if files >= 100_000: + break + except OSError: + errors += 1 + output[label] = { + "root": str(root), + "exists": root.is_dir(), + "directories": directories, + "matching_files": files, + "bytes": bytes_total, + "metadata_walk_ms": monotonic_ms(started), + "errors": errors, + "capped": cap_reason is not None, + "cap_reason": cap_reason, + } + return output + + +def _escape_mysql_option(value: str) -> str: + if "\n" in value or "\r" in value: + raise ValueError("MySQL option contains a newline") + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def _mysql_url_parts(url: str) -> dict[str, Any]: + normalized = re.sub(r"^mysql(?:\+[A-Za-z0-9_]+)?://", "mysql://", url, count=1) + parsed = urllib.parse.urlsplit(normalized) + if parsed.scheme != "mysql" or not parsed.hostname: + raise ValueError("MySQL URL must use mysql:// or mysql+driver://") + if parsed.fragment: + raise ValueError("MySQL URL fragments are unsupported; use --mysql-defaults-file") + query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True, strict_parsing=True) + normalized_query: dict[str, list[str]] = {} + for key, values in query.items(): + normalized_key = key.lower().replace("_", "-") + if normalized_key in normalized_query: + raise ValueError("MySQL URL has duplicate normalized options; use --mysql-defaults-file") + normalized_query[normalized_key] = values + allowed = {"ssl-mode", "ssl-ca", "ssl-cert", "ssl-key"} + unknown = set(normalized_query) - allowed + if unknown or any(len(values) != 1 for values in normalized_query.values()): + raise ValueError("MySQL URL has unsupported or repeated options; use --mysql-defaults-file") + ssl_mode_values = normalized_query.get("ssl-mode") or [] + ssl_mode = ssl_mode_values[0].upper() if ssl_mode_values else None + if ssl_mode not in {None, "REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"}: + raise ValueError("MySQL URL ssl-mode must be REQUIRED, VERIFY_CA, or VERIFY_IDENTITY") + output = { + "host": parsed.hostname, + "port": parsed.port or 3306, + "user": urllib.parse.unquote(parsed.username or ""), + "password": urllib.parse.unquote(parsed.password or ""), + "database": urllib.parse.unquote(parsed.path.lstrip("/")), + "ssl-mode": ssl_mode, + } + for name in ("ssl-ca", "ssl-cert", "ssl-key"): + values = normalized_query.get(name) + if values: + # parse_qs() already percent-decodes query values. Decoding a + # second time could change a literal "%2F" in a file name. + output[name] = values[0] + output["protocol"] = "TCP" + return output + + +MYSQL_SAFE_OPTIONS = { + "host", + "port", + "user", + "password", + "database", + "protocol", + "socket", + "ssl-mode", + "ssl-ca", + "ssl-capath", + "ssl-cert", + "ssl-key", + "ssl-cipher", + "tls-version", + "ssl-crl", + "ssl-crlpath", +} + + +def _decode_mysql_option_value(raw: str) -> str: + value = raw.strip() + if value[:1] in {'"', "'"}: + if len(value) < 2 or value[-1] != value[0]: + raise ValueError("unterminated quote in MySQL defaults file") + value = value[1:-1] + output: list[str] = [] + index = 0 + escapes = {"b": "\b", "t": "\t", "n": "\n", "r": "\r", "s": " ", "\\": "\\", '"': '"', "'": "'"} + while index < len(value): + if value[index] == "\\" and index + 1 < len(value): + index += 1 + escaped = value[index] + output.append(escapes.get(escaped, f"\\{escaped}")) + else: + output.append(value[index]) + index += 1 + decoded = "".join(output) + if "\x00" in decoded or "\r" in decoded or "\n" in decoded: + raise ValueError("MySQL option contains a forbidden control character") + return decoded + + +def _read_safe_mysql_defaults(path: Path) -> dict[str, str]: + options: dict[str, str] = {} + section = "" + for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + stripped = raw_line.strip() + if not stripped or stripped.startswith(("#", ";")): + continue + if stripped.lower().startswith("!include"): + raise ValueError("MySQL defaults file include directives are not allowed") + if stripped.startswith("[") and stripped.endswith("]"): + section = stripped[1:-1].strip().lower() + continue + if section != "client": + continue + key, separator, raw_value = stripped.partition("=") + if not separator: + raise ValueError(f"MySQL [client] option on line {line_number} must use key=value") + normalized_key = key.strip().lower().replace("_", "-") + if normalized_key not in MYSQL_SAFE_OPTIONS: + raise ValueError(f"unsafe or unsupported MySQL [client] option: {normalized_key}") + if normalized_key in options: + raise ValueError(f"duplicate MySQL [client] option: {normalized_key}") + options[normalized_key] = _decode_mysql_option_value(raw_value) + if not options: + raise ValueError("MySQL defaults file has no supported [client] options") + if options.get("port"): + try: + port = int(options["port"]) + except ValueError as exc: + raise ValueError("MySQL port must be an integer") from exc + if not 1 <= port <= 65535: + raise ValueError("MySQL port must be between 1 and 65535") + options["port"] = str(port) + if options.get("protocol") and options["protocol"].upper() not in {"TCP", "SOCKET"}: + raise ValueError("MySQL protocol must be TCP or SOCKET") + if options.get("protocol"): + options["protocol"] = options["protocol"].upper() + if options.get("ssl-mode"): + ssl_mode = options["ssl-mode"].upper() + if ssl_mode not in {"REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"}: + raise ValueError("MySQL ssl-mode must be REQUIRED, VERIFY_CA, or VERIFY_IDENTITY") + options["ssl-mode"] = ssl_mode + return options + + +@contextlib.contextmanager +def _mysql_defaults_file( + *, + explicit_file: str | None, + mysql_url_env: str | None, + mysql_host: str | None, + mysql_port: int, +) -> Iterable[tuple[str | None, dict[str, Any] | None, str | None]]: + env_name = mysql_url_env or "MYSQL_DATABASE_URL" + connection: dict[str, Any] + options: dict[str, Any] + if explicit_file: + path = Path(explicit_file).expanduser() + if not path.is_file(): + yield None, None, f"MySQL defaults file not found: {path}" + return + try: + options = _read_safe_mysql_defaults(path) + except (OSError, UnicodeError, ValueError) as exc: + yield None, None, f"could not safely parse MySQL defaults file: {exc}" + return + connection = { + "source": "sanitized_defaults_file", + "host": options.get("host") or mysql_host, + "port": int(options.get("port") or mysql_port), + "database": options.get("database"), + "ssl-mode": options.get("ssl-mode"), + "protocol": options.get("protocol"), + } + if os.name == "posix": + with contextlib.suppress(OSError): + mode = path.stat().st_mode & 0o777 + if mode & 0o077: + connection["permissions_warning"] = f"source defaults file mode {mode:o} is group/world accessible" + protocol = str(options.get("protocol") or "").upper() + local_socket = bool(protocol == "SOCKET" or (os.name == "posix" and protocol != "TCP" and (options.get("socket") or str(options.get("host") or "").lower() in {"", "localhost"}))) + connection["transport"] = "unix_socket" if local_socket else "tcp" + if not local_socket and not options.get("ssl-mode"): + yield ( + None, + connection, + "MySQL TCP authentication requires ssl-mode=REQUIRED (or VERIFY_CA/VERIFY_IDENTITY) in the sanitized [client] file. Only TCP timing was collected.", + ) + return + else: + raw_url = os.environ.get(env_name) + if not raw_url: + if mysql_host: + yield None, {"source": "host_only", "host": mysql_host, "port": mysql_port}, None + else: + yield None, None, f"{env_name} is not set and --mysql-defaults-file was not provided" + return + try: + options = _mysql_url_parts(raw_url) + except ValueError as exc: + yield None, None, str(exc) + return + connection = {key: value for key, value in options.items() if key in {"host", "port", "database", "ssl-mode"} and value not in (None, "")} + connection["source"] = f"env:{env_name}" + if not options.get("ssl-mode"): + yield ( + None, + connection, + "MySQL URL authentication requires ?ssl-mode=REQUIRED (or VERIFY_CA/VERIFY_IDENTITY); otherwise use --mysql-defaults-file. Only TCP timing was collected.", + ) + return + + content = "[client]\n" + for key in sorted(MYSQL_SAFE_OPTIONS): + value = options.get(key) + if value in (None, ""): + continue + content += f'{key}="{_escape_mysql_option(str(value))}"\n' + file_path: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix="deerflow-mysql-", + suffix=".cnf", + delete=False, + ) as handle: + file_path = handle.name + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(file_path, 0o600) + if os.name == "posix" and (Path(file_path).stat().st_mode & 0o077): + yield None, None, "could not enforce 0600 permissions on temporary MySQL credentials" + return + yield file_path, connection, None + finally: + if file_path: + with contextlib.suppress(OSError): + Path(file_path).unlink() + + +def _clean_mysql_environment(secret_env_name: str | None) -> dict[str, str]: + clean_env = dict(os.environ) + for name in ( + "MYSQL_PWD", + "MYSQL_HOST", + "MYSQL_TCP_PORT", + "MYSQL_UNIX_PORT", + "MYSQL_TEST_LOGIN_FILE", + "MYSQL_GROUP_SUFFIX", + secret_env_name, + ): + if name: + clean_env.pop(name, None) + return clean_env + + +def _mysql_supports_no_login_paths(secret_env_name: str | None) -> bool: + result = _run_command( + ["mysql", "--no-defaults", "--help"], + timeout=5.0, + env=_clean_mysql_environment(secret_env_name), + ) + return b"--no-login-paths" in result.get("output", b"") + + +def _mysql_command( + defaults_file: str, + sql: str, + timeout: float = 8.0, + *, + secret_env_name: str | None = None, + no_login_paths_supported: bool, +) -> dict[str, Any]: + clean_env = _clean_mysql_environment(secret_env_name) + arguments = ["mysql"] + if no_login_paths_supported: + arguments.append("--no-login-paths") + else: + clean_env["MYSQL_TEST_LOGIN_FILE"] = f"{defaults_file}.disabled-login-path" + arguments.append(f"--defaults-file={defaults_file}") + arguments.extend( + [ + "--batch", + "--raw", + "--skip-column-names", + "--connect-timeout=5", + "--execute", + sql, + ] + ) + return _run_command( + arguments, + timeout=timeout, + env=clean_env, + ) + + +def _parse_key_value_tsv(raw: bytes) -> dict[str, Any]: + output: dict[str, Any] = {} + for line in raw.decode(errors="replace").splitlines(): + key, separator, value = line.partition("\t") + if not separator: + continue + value = value.strip() + try: + parsed: Any = int(value) + except ValueError: + try: + parsed = float(value) + except ValueError: + parsed = value[:200] + output[key.strip()] = parsed + return output + + +def _tcp_connect_timing(host: str, port: int, timeout: float = 3.0) -> dict[str, Any]: + started = time.perf_counter() + try: + sock, details = _connect_socket(host, port, timeout=timeout, family="auto") + except ProbeFailure as exc: + return {"success": False, "error_phase": exc.phase, "error": str(exc), "total_ms": monotonic_ms(started)} + with contextlib.suppress(OSError): + sock.close() + return { + "success": True, + "dns_ms": details["dns_ms"], + "connect_ms": details["connect_ms"], + "peer_ip": details["peer_ip"], + "total_ms": monotonic_ms(started), + } + + +def collect_mysql_diagnostics( + *, + enabled: bool, + mysql_url_env: str | None, + defaults_file: str | None, + mysql_host: str | None, + mysql_port: int, + probes: int = 3, +) -> dict[str, Any]: + if not enabled: + return {"enabled": False} + with _mysql_defaults_file( + explicit_file=defaults_file, + mysql_url_env=mysql_url_env, + mysql_host=mysql_host, + mysql_port=mysql_port, + ) as (resolved_file, connection, config_error): + output: dict[str, Any] = {"enabled": True, "connection": connection} + if connection and connection.get("host"): + output["tcp"] = _tcp_connect_timing(str(connection["host"]), int(connection.get("port") or mysql_port)) + if config_error: + output["available"] = False + output["reason"] = config_error + return output + if shutil.which("mysql") is None: + output["available"] = False + output["reason"] = "mysql command-line client not found; TCP timing may still be present" + return output + if resolved_file is None: + output["available"] = False + output["reason"] = "credentials were not provided" + return output + + no_login_paths_supported = _mysql_supports_no_login_paths(mysql_url_env) + output["client_option_isolation"] = { + "defaults_file_sanitized": True, + "login_paths_disabled": no_login_paths_supported, + "login_file_redirected_when_unsupported": not no_login_paths_supported, + "ambient_mysql_environment_removed": True, + } + + def mysql_query(sql: str, timeout: float = 8.0) -> dict[str, Any]: + return _mysql_command( + resolved_file, + sql, + timeout=timeout, + secret_env_name=mysql_url_env, + no_login_paths_supported=no_login_paths_supported, + ) + + ping_ms: list[float] = [] + ping_errors: list[str] = [] + for _index in range(max(1, min(probes, 10))): + result = mysql_query("SELECT 1;", timeout=8) + if result.get("returncode") == 0: + ping_ms.append(float(result["elapsed_ms"])) + else: + ping_errors.append("timeout" if result.get("timed_out") else "query_failed") + output["select_1_ms"] = summarize_numbers(ping_ms) + output["select_1_errors"] = dict(Counter(ping_errors)) + output["available"] = bool(ping_ms) + if not ping_ms: + output["reason"] = "all authenticated SELECT 1 probes failed; subsequent status queries were skipped" + return output + + variables_sql = "SHOW GLOBAL VARIABLES WHERE Variable_name IN ('max_connections','wait_timeout','interactive_timeout','slow_query_log','long_query_time');" + status_sql = ( + "SHOW GLOBAL STATUS WHERE Variable_name IN " + "('Threads_connected','Threads_running','Threads_created','Connections','Aborted_connects'," + "'Max_used_connections','Slow_queries','Questions','Uptime','Innodb_row_lock_current_waits'," + "'Innodb_row_lock_time','Innodb_row_lock_waits','Created_tmp_disk_tables','Created_tmp_tables'," + "'Innodb_buffer_pool_reads','Innodb_buffer_pool_read_requests');" + ) + variables_result = mysql_query(variables_sql) + status_result = mysql_query(status_sql) + if variables_result.get("returncode") == 0: + output["variables"] = _parse_key_value_tsv(variables_result["output"]) + else: + output["variables_error"] = "query_failed" + if status_result.get("returncode") == 0: + output["status"] = _parse_key_value_tsv(status_result["output"]) + else: + output["status_error"] = "query_failed" + + process_sql = ( + "SELECT COUNT(*)," + "COALESCE(SUM(COMMAND='Sleep'),0)," + "COALESCE(SUM(COMMAND<>'Sleep'),0)," + "COALESCE(SUM(COMMAND<>'Sleep' AND TIME>=5),0)," + "COALESCE(SUM(COMMAND<>'Sleep' AND TIME>=30),0)," + "COALESCE(SUM(LOWER(COALESCE(STATE,'')) LIKE '%lock%'),0)," + "COALESCE(MAX(CASE WHEN COMMAND<>'Sleep' THEN TIME ELSE 0 END),0) " + "FROM information_schema.PROCESSLIST;" + ) + process_result = mysql_query(process_sql) + if process_result.get("returncode") == 0: + fields = process_result["output"].decode(errors="replace").strip().split("\t") + if len(fields) == 7: + names = [ + "total", + "sleeping", + "active", + "active_at_least_5s", + "active_at_least_30s", + "lock_state_count", + "max_active_seconds", + ] + with contextlib.suppress(ValueError): + output["processlist"] = {name: int(value) for name, value in zip(names, fields)} + else: + output["processlist_error"] = "query_failed_or_permission_denied" + + lock_queries = { + "data_lock_waits": "SELECT COUNT(*) FROM performance_schema.data_lock_waits;", + "pending_metadata_locks": ("SELECT COUNT(*) FROM performance_schema.metadata_locks WHERE LOCK_STATUS='PENDING';"), + } + locks: dict[str, Any] = {} + for name, sql in lock_queries.items(): + result = mysql_query(sql) + if result.get("returncode") == 0: + raw = result["output"].decode(errors="replace").strip() + with contextlib.suppress(ValueError): + locks[name] = int(raw) + else: + locks[f"{name}_error"] = "unavailable_or_permission_denied" + output["locks"] = locks + + digest_sql = "SELECT COUNT(*),COALESCE(ROUND(MAX(AVG_TIMER_WAIT)/1000000000,3),0),COALESCE(ROUND(MAX(MAX_TIMER_WAIT)/1000000000,3),0) FROM performance_schema.events_statements_summary_by_digest WHERE COUNT_STAR>0;" + digest_result = mysql_query(digest_sql) + if digest_result.get("returncode") == 0: + fields = digest_result["output"].decode(errors="replace").strip().split("\t") + if len(fields) == 3: + with contextlib.suppress(ValueError): + output["statement_digest_summary"] = { + "digests": int(fields[0]), + "max_average_ms": float(fields[1]), + "max_single_ms": float(fields[2]), + } + else: + output["statement_digest_error"] = "unavailable_or_permission_denied" + return output + + +def collect_sqlite_diagnostics(config_path: Path | None, config: Mapping[str, Any]) -> dict[str, Any]: + if str(config.get("database.backend") or "").lower() != "sqlite" or config_path is None: + return {"enabled": False} + import sqlite3 + + sqlite_dir = str(config.get("database.sqlite_dir") or ".deer-flow/data") + base = Path(sqlite_dir) + if not base.is_absolute(): + base = config_path.parent / base + candidates = [base / "deerflow.db", base / "deerflow.sqlite", base / "deerflow.sqlite3"] + db_path = next((item for item in candidates if item.is_file()), None) + if db_path is None: + return {"enabled": True, "available": False, "reason": "SQLite database file not found"} + started = time.perf_counter() + try: + uri = f"file:{urllib.parse.quote(str(db_path))}?mode=ro" + connection = sqlite3.connect(uri, uri=True, timeout=0.5) + try: + connection.execute("SELECT 1").fetchone() + journal_mode = connection.execute("PRAGMA journal_mode").fetchone()[0] + page_count = int(connection.execute("PRAGMA page_count").fetchone()[0]) + page_size = int(connection.execute("PRAGMA page_size").fetchone()[0]) + finally: + connection.close() + except sqlite3.Error as exc: + return { + "enabled": True, + "available": False, + "path": str(db_path), + "error": _safe_error(exc), + "probe_ms": monotonic_ms(started), + } + return { + "enabled": True, + "available": True, + "path": str(db_path), + "probe_ms": monotonic_ms(started), + "journal_mode": journal_mode, + "page_count": page_count, + "page_size": page_size, + "database_bytes": db_path.stat().st_size, + "wal_bytes": db_path.with_name(db_path.name + "-wal").stat().st_size if db_path.with_name(db_path.name + "-wal").is_file() else 0, + "shm_bytes": db_path.with_name(db_path.name + "-shm").stat().st_size if db_path.with_name(db_path.name + "-shm").is_file() else 0, + } + + +def _json_request( + url: str, + *, + headers: Mapping[str, str], + timeout: float, + insecure: bool, + ca_file: str | None, +) -> tuple[int | None, float, Any | None, str | None]: + started = time.perf_counter() + try: + request = urllib.request.Request(url, headers=dict(headers), method="GET") + handlers: list[Any] = [urllib.request.ProxyHandler({}), _NoRedirectHandler()] + if urllib.parse.urlsplit(url).scheme == "https": + context = ssl._create_unverified_context() if insecure else ssl.create_default_context(cafile=ca_file) # noqa: SLF001 + handlers.append(urllib.request.HTTPSHandler(context=context)) + opener = urllib.request.build_opener(*handlers) + with opener.open(request, timeout=timeout) as response: + status = int(response.status) + body = response.read(4 * 1024 * 1024) + except urllib.error.HTTPError as exc: + return int(exc.code), monotonic_ms(started), None, f"HTTP {exc.code}" + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc: + return None, monotonic_ms(started), None, _safe_error(exc) + try: + payload = json.loads(body) + except (json.JSONDecodeError, UnicodeDecodeError): + return status, monotonic_ms(started), None, "response was not JSON" + return status, monotonic_ms(started), payload, None + + +def collect_admin_concurrency( + origin: str | None, + *, + token_env: str | None, + timeout: float, + insecure: bool, + ca_file: str | None, +) -> dict[str, Any]: + if not token_env: + return {"enabled": False, "reason": "--admin-token-env not supplied"} + token = os.environ.get(token_env) + if not token: + return {"enabled": True, "available": False, "reason": f"environment variable {token_env} is not set"} + if not origin: + return {"enabled": True, "available": False, "reason": "admin origin is unavailable"} + headers = {"Authorization": token if token.lower().startswith("bearer ") else f"Bearer {token}"} + paths = { + "active": "/api/admin/active-runs", + "concurrency": "/api/admin/active-runs/concurrency?granularity=minute&hours=1", + "token_speed": "/api/admin/active-runs/token-speed?granularity=minute&hours=1", + "limits": "/api/system-settings/model-concurrency", + } + output: dict[str, Any] = { + "enabled": True, + "available": False, + "origin": sanitize_origin(origin), + "proxy_mode": "direct", + } + successful_endpoints = 0 + failed_endpoints: list[str] = [] + required_keys = { + "active": {"count", "runs"}, + "concurrency": {"enabled", "points"}, + "token_speed": {"points"}, + "limits": {"enabled", "default_limit", "user_default_limit", "model_limits", "user_model_limits"}, + } + + def safe_int(value: Any, default: int = 0) -> int: + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError): + return default + return parsed + + def safe_float(value: Any, default: float = 0.0) -> float: + try: + parsed = float(value) + except (TypeError, ValueError, OverflowError): + return default + return parsed if math.isfinite(parsed) else default + + def payload_schema_ok(name: str, payload: Any) -> bool: + if not isinstance(payload, dict) or not required_keys[name].issubset(payload): + return False + if name == "active": + return isinstance(payload.get("count"), int) and isinstance(payload.get("runs"), list) + if name == "concurrency": + return isinstance(payload.get("enabled"), bool) and isinstance(payload.get("points"), list) + if name == "token_speed": + return isinstance(payload.get("points"), list) + return ( + isinstance(payload.get("enabled"), bool) + and isinstance(payload.get("default_limit"), int) + and isinstance(payload.get("user_default_limit"), int) + and isinstance(payload.get("model_limits"), dict) + and isinstance(payload.get("user_model_limits"), dict) + ) + + for name, path in paths.items(): + url = urllib.parse.urljoin(origin.rstrip("/") + "/", path.lstrip("/")) + status, elapsed_ms, payload, error = _json_request( + url, + headers=headers, + timeout=timeout, + insecure=insecure, + ca_file=ca_file, + ) + entry: dict[str, Any] = {"status_code": status, "elapsed_ms": elapsed_ms} + if error: + entry["error"] = error + endpoint_ok = status is not None and 200 <= status < 300 and payload_schema_ok(name, payload) + if endpoint_ok: + successful_endpoints += 1 + else: + failed_endpoints.append(name) + if not error and status is not None and 200 <= status < 300: + entry["error"] = "unexpected JSON schema" + entry["available"] = endpoint_ok + if name == "active" and endpoint_ok: + runs = payload.get("runs") if isinstance(payload.get("runs"), list) else [] + entry.update( + { + "count": safe_int(payload.get("count"), len(runs)), + "by_model": dict(Counter(str(run.get("model_name") or "unknown") for run in runs if isinstance(run, dict))), + "by_kind": dict(Counter(str(run.get("kind") or "unknown") for run in runs if isinstance(run, dict))), + "max_elapsed_seconds": max([safe_int(run.get("elapsed_seconds")) for run in runs if isinstance(run, dict)] or [0]), + } + ) + elif name == "concurrency" and endpoint_ok: + points = payload.get("points") if isinstance(payload.get("points"), list) else [] + entry.update( + { + "monitor_enabled": bool(payload.get("enabled")), + "points": len(points), + "peak": max([safe_int(point.get("peak")) for point in points if isinstance(point, dict)] or [0]), + "max_average": max([safe_float(point.get("avg")) for point in points if isinstance(point, dict)] or [0.0]), + } + ) + elif name == "token_speed" and endpoint_ok: + points = payload.get("points") if isinstance(payload.get("points"), list) else [] + speeds = [safe_float(point.get("avg_tokens_per_sec")) for point in points if isinstance(point, dict)] + entry.update( + { + "points": len(points), + "min_bucket_average_tokens_per_sec": min(speeds) if speeds else None, + "max_bucket_average_tokens_per_sec": max(speeds) if speeds else None, + } + ) + elif name == "limits" and endpoint_ok: + safe_payload: dict[str, Any] = {} + for key in ("enabled", "default_limit", "user_default_limit", "model_limits", "user_model_limits"): + value = payload.get(key) + if key in {"model_limits", "user_model_limits"} and isinstance(value, dict): + safe_payload[key] = {str(model)[:120]: safe_int(limit) for model, limit in list(value.items())[:500] if safe_int(limit) > 0} + elif key == "enabled": + safe_payload[key] = bool(value) + elif key in {"default_limit", "user_default_limit"}: + parsed_limit = safe_int(value, -1) + safe_payload[key] = parsed_limit if parsed_limit >= 0 else None + entry["settings"] = safe_payload + output[name] = entry + output["available"] = successful_endpoints > 0 + output["successful_endpoints"] = successful_endpoints + output["failed_endpoints"] = failed_endpoints + output["partial"] = 0 < successful_endpoints < len(paths) + if successful_endpoints == 0 and all(output[name].get("status_code") in {401, 403} for name in paths): + output["reason"] = "admin token was rejected" + elif successful_endpoints == 0: + output["reason"] = "no admin endpoint returned a 2xx JSON object" + return output + + +def run_deerflow_suite(origin: str, options: ProbeOptions, *, authenticated: bool) -> dict[str, Any]: + paths = [ + ("health", "/api/health", False), + ("auth_options", "/api/v1/auth/options", False), + ] + if authenticated: + paths.extend( + [ + ("auth_me", "/api/v1/auth/me", True), + ("agents_selector", "/api/agents/selector", True), + ( + "agents_mine", + "/api/agents?scope=mine&page=1&page_size=20&sort=time_desc", + True, + ), + ( + "agents_square", + "/api/agents?scope=square&page=1&page_size=20&sort=time_desc", + True, + ), + ("skills", "/api/skills?sort=time_desc", True), + ] + ) + output: dict[str, Any] = {} + public_headers: dict[str, str] = {} + for name, path, use_auth in paths: + suite_options = ProbeOptions( + method="GET", + headers=dict(options.headers if use_auth else public_headers), + connect_timeout=options.connect_timeout, + first_byte_timeout=options.first_byte_timeout, + read_timeout=options.read_timeout, + total_timeout=options.total_timeout, + max_response_bytes=options.max_response_bytes, + read_mode="full", + insecure=options.insecure, + ca_file=options.ca_file, + follow_redirects=False, + proxy=options.proxy, + use_env_proxy=options.use_env_proxy, + ip_family=options.ip_family, + ) + url = urllib.parse.urljoin(origin.rstrip("/") + "/", path.lstrip("/")) + result = probe_url(url, suite_options) + output[name] = { + "url": result["url"], + "success": result["success"], + "status_code": result["status_code"], + "error_phase": result["error_phase"], + "timings_ms": result["timings_ms"], + "body_bytes_read": result["hops"][-1].get("response", {}).get("body_bytes_read", 0), + } + return output + + +def run_health_baselines(origins: Sequence[str], options: ProbeOptions) -> dict[str, Any]: + paths = ("/health", "/api/health", "/api/langgraph/health") + output: dict[str, Any] = {} + public_headers: dict[str, str] = {} + health_options = ProbeOptions( + method="GET", + headers=public_headers, + connect_timeout=options.connect_timeout, + first_byte_timeout=min(options.first_byte_timeout, 10.0), + read_timeout=min(options.read_timeout, 3.0), + total_timeout=min(options.total_timeout, 15.0), + max_response_bytes=65536, + read_mode="full", + insecure=options.insecure, + ca_file=options.ca_file, + follow_redirects=False, + proxy=options.proxy, + use_env_proxy=options.use_env_proxy, + ip_family=options.ip_family, + ) + for origin in dict.fromkeys(origins): + origin_output: dict[str, Any] = {} + for path in paths: + url = urllib.parse.urljoin(origin.rstrip("/") + "/", path.lstrip("/")) + result = probe_url(url, health_options) + origin_output[path] = { + "status_code": result["status_code"], + "success": result["success"], + "error_phase": result["error_phase"], + "timings_ms": result["timings_ms"], + } + output[sanitize_origin(origin)] = origin_output + return output + + +def run_interference_probe( + target_url: str, + sentinel_url: str, + target_options: ProbeOptions, + *, + interval: float, + max_sentinel_probes: int, +) -> dict[str, Any]: + """Probe health repeatedly while one target request is in flight.""" + stop = threading.Event() + sentinel_results: list[dict[str, Any]] = [] + public_headers: dict[str, str] = {} + sentinel_options = ProbeOptions( + method="GET", + headers=public_headers, + connect_timeout=min(target_options.connect_timeout, 3.0), + first_byte_timeout=min(target_options.first_byte_timeout, 5.0), + read_timeout=min(target_options.read_timeout, 2.0), + total_timeout=min(target_options.total_timeout, 10.0), + max_response_bytes=65536, + read_mode="full", + insecure=target_options.insecure, + ca_file=target_options.ca_file, + proxy=target_options.proxy, + use_env_proxy=target_options.use_env_proxy, + ip_family=target_options.ip_family, + ) + + def sentinel_loop() -> None: + while not stop.is_set() and len(sentinel_results) < max_sentinel_probes: + sentinel_results.append(probe_url(sentinel_url, sentinel_options)) + stop.wait(max(0.1, interval)) + + thread = threading.Thread(target=sentinel_loop, name="deerflow-health-sentinel", daemon=True) + started = time.perf_counter() + thread.start() + target_result = probe_url(target_url, target_options) + stop.set() + thread.join(timeout=max(1.0, target_options.read_timeout + 1.0)) + return { + "target_url": sanitize_url(target_url), + "sentinel_url": sanitize_url(sentinel_url), + "wall_ms": monotonic_ms(started), + "target": target_result, + "sentinel_results": sentinel_results, + "sentinel_summary": summarize_probe_results(sentinel_results), + "caveat": "Multiple Gateway workers or load balancers can route sentinel requests around a blocked worker.", + } + + +def probe_nginx_stub_status(url: str | None, options: ProbeOptions) -> dict[str, Any]: + if not url: + return {"enabled": False} + request = urllib.request.Request(url, method="GET", headers={"User-Agent": USER_AGENT}) + handlers: list[Any] = [urllib.request.ProxyHandler({}), _NoRedirectHandler()] + if urllib.parse.urlsplit(url).scheme == "https": + context = ( + ssl._create_unverified_context() # noqa: SLF001 + if options.insecure + else ssl.create_default_context(cafile=options.ca_file) + ) + handlers.append(urllib.request.HTTPSHandler(context=context)) + started = time.perf_counter() + try: + with urllib.request.build_opener(*handlers).open(request, timeout=options.first_byte_timeout) as response: + status = response.status + text = response.read(65536).decode(errors="replace") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc: + return { + "enabled": True, + "available": False, + "proxy_mode": "direct", + "url": sanitize_url(url), + "elapsed_ms": monotonic_ms(started), + "error": _safe_error(exc), + } + active_match = re.search(r"Active connections:\s*(\d+)", text) + state_match = re.search(r"Reading:\s*(\d+)\s+Writing:\s*(\d+)\s+Waiting:\s*(\d+)", text) + accepts_match = re.search(r"server accepts handled requests\s*\n\s*(\d+)\s+(\d+)\s+(\d+)", text) + return { + "enabled": True, + "available": bool(active_match and state_match), + "proxy_mode": "direct", + "url": sanitize_url(url), + "status_code": int(status), + "elapsed_ms": monotonic_ms(started), + "active_connections": int(active_match.group(1)) if active_match else None, + "reading": int(state_match.group(1)) if state_match else None, + "writing": int(state_match.group(2)) if state_match else None, + "waiting": int(state_match.group(3)) if state_match else None, + "accepted": int(accepts_match.group(1)) if accepts_match else None, + "handled": int(accepts_match.group(2)) if accepts_match else None, + "requests": int(accepts_match.group(3)) if accepts_match else None, + } + + +SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2, "ok": 3} + + +def _add_finding( + findings: list[dict[str, Any]], + severity: str, + area: str, + code: str, + title: str, + evidence: Sequence[str], + interpretation: str, + next_step: str, +) -> None: + findings.append( + { + "severity": severity, + "area": area, + "code": code, + "title": title, + "evidence": list(evidence), + "interpretation": interpretation, + "next_step": next_step, + } + ) + + +def _summary_p50(summary: Mapping[str, Any], key: str) -> float | None: + value = summary.get("timings_ms", {}).get(key, {}).get("p50") + return float(value) if value is not None else None + + +def _target_response_signatures(target: Mapping[str, Any]) -> tuple[tuple[int, str], ...]: + signatures: set[tuple[int, str]] = set() + for sample in target.get("samples", []): + if not sample.get("success"): + continue + hops = sample.get("hops") or [] + if not hops: + continue + hop = hops[-1] + status = hop.get("status_code") + if not isinstance(status, int): + continue + content_type = str(hop.get("response", {}).get("headers", {}).get("content-type") or "") + media_type = content_type.split(";", 1)[0].strip().lower() + signatures.add((status, media_type)) + return tuple(sorted(signatures)) + + +def _suite_total(suite: Mapping[str, Any], key: str) -> float | None: + value = suite.get(key, {}).get("timings_ms", {}).get("total") + return float(value) if value is not None else None + + +def _health_total_for_origin(health: Mapping[str, Any], origin: str, path: str = "/api/health") -> float | None: + value = health.get(sanitize_origin(origin), {}).get(path, {}).get("timings_ms", {}).get("total") + return float(value) if value is not None else None + + +def build_findings(report: Mapping[str, Any]) -> list[dict[str, Any]]: + findings: list[dict[str, Any]] = [] + http_section = report.get("http", {}) + targets = http_section.get("targets", []) + for target in targets: + label = str(target.get("label") or "target") + summary = target.get("summary", {}) + errors = int(summary.get("transport_errors") or 0) + requests = int(summary.get("requests") or 0) + if errors: + _add_finding( + findings, + "critical" if errors == requests else "warning", + "network", + "transport_errors", + f"{label} 存在传输层失败", + [f"{errors}/{requests} 次失败", f"失败阶段={summary.get('error_phase_counts', {})}"], + "失败阶段可以先把问题定位到 DNS、TCP、TLS、首响应等待或正文读取；HTTP 4xx/5xx 不计为传输失败。", + "按报告中的 error_phase 处理；如果是 first_byte，继续对照直连 Gateway 与 /api/health。", + ) + status_counts = summary.get("status_counts", {}) + bad_gateway = sum(int(status_counts.get(str(code), 0)) for code in (502, 503, 504)) + if bad_gateway: + _add_finding( + findings, + "warning", + "nginx", + "upstream_http_errors", + f"{label} 返回网关/上游错误", + [f"502/503/504 共 {bad_gateway} 次", f"状态分布={status_counts}"], + "代理成功接到请求，但上游连接、上游处理或代理超时失败。", + "对照 Nginx error log 的 upstream timed out/connect failed，并检查直连 Gateway 是否正常。", + ) + if int(status_counts.get("429", 0)): + _add_finding( + findings, + "warning", + "deerflow", + "http_429", + f"{label} 触发限流或并发上限", + [f"429 次数={status_counts.get('429')}"], + "可能是 DeerFlow 模型并发限制、上游配额或其他限流；仅凭状态码不能区分。", + "查看管理员 active-runs/model-concurrency 数据和 Gateway 同时段日志。", + ) + + total = _summary_p50(summary, "total") + dns = _summary_p50(summary, "dns") or 0.0 + tcp = _summary_p50(summary, "tcp_connect") or 0.0 + tls = _summary_p50(summary, "tls_handshake") or 0.0 + response_wait = _summary_p50(summary, "response_wait") or 0.0 + body_read = _summary_p50(summary, "body_read") or 0.0 + if dns >= 200: + _add_finding( + findings, + "warning", + "network", + "slow_dns", + f"{label} DNS 解析慢", + [f"DNS p50={dns:.1f}ms"], + "连接尚未到达 Nginx 或 Gateway，先查 DNS/解析链路。", + "在客户端和服务器分别检查 resolv.conf、DNS 服务器延迟与缓存。", + ) + if tcp >= 300: + _add_finding( + findings, + "warning", + "network", + "slow_tcp_connect", + f"{label} TCP 建连慢", + [f"TCP p50={tcp:.1f}ms"], + "延迟发生在应用收到 HTTP 请求前，可能涉及网络、SYN 队列、防火墙或连接耗尽。", + "查看 ss 监听队列/SYN-RECV，并从同网段及服务器本机分别复测。", + ) + if tls >= 500: + _add_finding( + findings, + "warning", + "network", + "slow_tls", + f"{label} TLS 握手慢", + [f"TLS p50={tls:.1f}ms"], + "延迟主要在证书/加密握手或中间代理，尚未进入业务处理。", + "检查 TLS 终止层 CPU、证书链、OCSP/中间安全设备，并与 HTTP 直连作对照。", + ) + if total and response_wait >= 800 and response_wait / max(total, 1.0) >= 0.55: + _add_finding( + findings, + "warning", + "application", + "response_wait_dominates", + f"{label} 主要卡在请求发完到首响应之间", + [f"总耗时 p50={total:.1f}ms", f"response_wait p50={response_wait:.1f}ms"], + "请求已建连并上传，等待可能来自 Nginx 上游排队、Gateway 事件循环、数据库、文件扫描或下游服务；这不是单独证明 MySQL 慢。", + "结合直连、并行 health、MySQL processlist/locks 和日志中的分阶段计时继续定位。", + ) + if body_read >= 1000 and response_wait < body_read: + _add_finding( + findings, + "warning", + "network", + "slow_body_transfer", + f"{label} 首响应后正文读取更慢", + [f"body_read p50={body_read:.1f}ms", f"response_wait p50={response_wait:.1f}ms"], + "更像大响应、慢流式输出、网络吞吐或代理缓冲，而不是请求开始前排队。", + "检查响应字节数、Content-Type、Nginx buffering 与客户端渲染；SSE 用 --read-mode first-byte。", + ) + + if len(targets) >= 2: + public = targets[0] + public_p50 = _summary_p50(public.get("summary", {}), "total") + public_signature = _target_response_signatures(public) + for compare in targets[1:]: + direct_p50 = _summary_p50(compare.get("summary", {}), "total") + compare_signature = _target_response_signatures(compare) + gap_is_large = bool(public_p50 and direct_p50 and public_p50 >= direct_p50 * 1.5 and public_p50 - direct_p50 >= 200) + responses_are_comparable = bool( + len(public_signature) == 1 + and public_signature == compare_signature + and 200 <= public_signature[0][0] < 300 + ) + if gap_is_large and responses_are_comparable: + _add_finding( + findings, + "warning", + "nginx", + "public_direct_gap", + "公网/代理路径明显慢于对照地址", + [ + f"{public.get('label')} p50={public_p50:.1f}ms", + f"{compare.get('label')} p50={direct_p50:.1f}ms", + ], + "差值位于两条路径不同的网络、TLS、负载均衡或 Nginx 层；前提是两个 URL 实际执行同一业务。", + "确认 Host/路径/请求体一致，再检查 Nginx 连接、错误日志和上游 keepalive。", + ) + elif gap_is_large and not responses_are_comparable: + _add_finding( + findings, + "info", + "nginx", + "public_direct_not_comparable", + "公网与对照地址响应不一致，不能据此归因 Nginx", + [f"公网响应签名={public_signature}", f"对照响应签名={compare_signature}"], + "两条路径的状态码或 Content-Type 不同，快速的 401/404/HTML 页面不能作为同一业务的直连基线。", + "先确认 Host、路由、鉴权和路径一致，再进行耗时对比。", + ) + + local_gateway = http_section.get("local_gateway_health", {}) + target_host_confirmed = bool(report.get("server", {}).get("scope", {}).get("target_host_confirmed")) + if target_host_confirmed and targets and local_gateway.get("success") and local_gateway.get("status_code") == 200: + primary_origin = _origin_for_request(str(targets[0].get("url") or "")) + public_health = http_section.get("health_baselines", {}).get(sanitize_origin(primary_origin), {}).get("/api/health", {}) + public_health_ms = public_health.get("timings_ms", {}).get("total") + local_health_ms = local_gateway.get("timings_ms", {}).get("total") + if ( + public_health.get("success") + and public_health.get("status_code") == 200 + and public_health_ms is not None + and local_health_ms is not None + and float(public_health_ms) >= float(local_health_ms) * 1.5 + and float(public_health_ms) - float(local_health_ms) >= 200 + ): + _add_finding( + findings, + "warning", + "nginx", + "public_local_health_gap", + "公网 health 明显慢于本机 DeerFlow Gateway health", + [f"公网 /api/health={float(public_health_ms):.1f}ms", f"本机 /api/health={float(local_health_ms):.1f}ms"], + "两个 200 health 响应执行同一轻量路径，差值更偏向入口网络、TLS、负载均衡或 Nginx。", + "结合 Nginx 活跃连接、监听队列、FD 和错误日志判断是否连接容量或代理层问题。", + ) + + suite = http_section.get("deerflow_suite", {}) + if suite: + health_ms = _suite_total(suite, "health") + options_ms = _suite_total(suite, "auth_options") + me_ms = _suite_total(suite, "auth_me") + agents_ms = max([_suite_total(suite, name) or 0.0 for name in ("agents_selector", "agents_mine", "agents_square")] or [0.0]) + skills_ms = _suite_total(suite, "skills") + db_paths = [value for value in (options_ms, me_ms, agents_ms) if value is not None and value > 0] + if health_ms is not None and health_ms < 500 and db_paths and sum(value >= 1500 for value in db_paths) >= 2: + _add_finding( + findings, + "warning", + "mysql", + "health_fast_db_paths_slow", + "静态 health 快，但多条数据库相关接口慢", + [ + f"health={health_ms:.1f}ms", + f"auth/options={options_ms}ms", + f"auth/me={me_ms}ms", + f"agents最大={agents_ms:.1f}ms", + ], + "这是业务数据库、连接池、锁或数据库磁盘的强关联证据；auth/options、auth/me、agents 都会查业务库。", + "立即在卡顿时抓 MySQL Threads/PROCESSLIST/data_lock_waits/metadata_locks，并核对池上限。", + ) + if health_ms is not None and health_ms < 500 and skills_ms and skills_ms >= 1500: + others = [value for value in (options_ms, me_ms, agents_ms) if value is not None and value > 0] + if not others or max(others) < skills_ms * 0.6: + _add_finding( + findings, + "warning", + "deerflow", + "skills_path_specific", + "技能列表明显慢于健康和其他业务接口", + [f"health={health_ms:.1f}ms", f"skills={skills_ms:.1f}ms"], + "当前实现每次 /api/skills 都同步扫描并解析技能目录，再查所有权/收藏/标签；慢磁盘或技能很多会放大。", + "运行 interference 探针；若 health 同时尖峰，优先查同步文件扫描阻塞单 worker 事件循环。", + ) + if health_ms is not None and health_ms < 500 and agents_ms >= 1500: + _add_finding( + findings, + "info", + "deerflow", + "agents_path_slow", + "Agents 接口慢但静态 health 正常", + [f"health={health_ms:.1f}ms", f"agents最大={agents_ms:.1f}ms"], + "可能是首次/每小时 legacy agent 目录同步或 agents 多次串行 DB 查询；需看同一时段日志。", + "grep [start-run-timing] 和 agents 同步日志，并在第二次立即复测判断是否只有冷启动/TTL 首次慢。", + ) + + interference = http_section.get("interference") + if isinstance(interference, dict): + target_total = interference.get("target", {}).get("timings_ms", {}).get("total") + sentinel_p95 = interference.get("sentinel_summary", {}).get("timings_ms", {}).get("total", {}).get("p95") + sentinel_count = interference.get("sentinel_summary", {}).get("successful_transport", 0) + if target_total and float(target_total) >= 1000 and sentinel_count and sentinel_p95 is not None: + if float(sentinel_p95) >= 500: + _add_finding( + findings, + "warning", + "deerflow", + "health_stalls_with_target", + "慢接口执行期间 health 也出现明显延迟", + [f"目标总耗时={float(target_total):.1f}ms", f"并行 health p95={float(sentinel_p95):.1f}ms"], + "强烈指向共享 Gateway worker/事件循环、CPU 或宿主 IO 阻塞；单 worker 下同步技能目录扫描尤其可疑。多 worker/负载均衡可能掩盖该现象。", + "对照进程 CPU/IO PSI、技能目录遍历时间和 Gateway 日志；不要先盲目扩大 MySQL 连接。", + ) + elif float(sentinel_p95) < 250: + _add_finding( + findings, + "info", + "application", + "health_fast_during_target", + "慢接口执行期间 health 保持快速", + [f"目标总耗时={float(target_total):.1f}ms", f"并行 health p95={float(sentinel_p95):.1f}ms"], + "更偏向该请求自己的数据库/下游服务/模型等待，而不是整个 Gateway 事件循环被卡死。", + "继续对照 auth/me、MySQL 和该接口专属下游；多 worker 环境下此结论只能算辅助证据。", + ) + + server = report.get("server", {}) + easy_mode = bool(report.get("request", {}).get("easy_mode")) + target_host_confirmed = bool(server.get("scope", {}).get("target_host_confirmed")) + if easy_mode and not target_host_confirmed: + _add_finding( + findings, + "info", + "resources", + "execution_host_not_confirmed", + "未确认脚本执行机就是目标 DeerFlow 服务器", + ["本机未发现经过身份校验的 loopback DeerFlow Gateway"], + "已采集的 CPU、内存、Docker、Nginx 等数据属于脚本执行机，不能直接用于判断目标服务器配置不足。", + "把脚本复制到实际部署服务器运行；网络接口计时结果仍然有效。", + ) + server = {} + socket_info = server.get("sockets", {}) + for listener in socket_info.get("listeners", []): + usage = listener.get("backlog_usage_pct") + if usage is not None and float(usage) >= 70: + _add_finding( + findings, + "critical" if float(usage) >= 95 else "warning", + "nginx", + "listen_backlog_pressure", + f"端口 {listener.get('port')} 监听队列接近上限", + [ + f"Recv-Q={listener.get('receive_queue')}", + f"backlog={listener.get('backlog')}", + f"占用={float(usage):.1f}%", + ], + "新 TCP 连接可能在内核/Nginx/Gateway 接收前排队或丢弃。", + "核对 worker 是否处理不过来、somaxconn/tcp_max_syn_backlog、CPU 和文件句柄；先解决处理能力而非只放大 backlog。", + ) + global_states = socket_info.get("global_states", {}) + if int(global_states.get("SYN-RECV", 0)) >= 100: + _add_finding( + findings, + "warning", + "network", + "many_syn_recv", + "存在大量半连接", + [f"SYN-RECV={global_states.get('SYN-RECV')}"], + "可能是连接洪峰、backlog 压力或异常客户端。", + "按监听端口查看 ss -ant state syn-recv，并检查负载均衡、SYN cookies 与网络来源。", + ) + + nginx_status = server.get("nginx_status", {}) + nginx_config = server.get("nginx", {}) + if nginx_status.get("available"): + capacities = [int(item.get("theoretical_connection_slots")) for item in nginx_config.get("instances", []) if item.get("theoretical_connection_slots")] + capacity = min(capacities) if capacities else None + active = nginx_status.get("active_connections") + if capacity and active is not None and int(active) >= capacity * 0.7: + _add_finding( + findings, + "critical" if int(active) >= capacity * 0.9 else "warning", + "nginx", + "nginx_connection_slots", + "Nginx 活跃连接接近理论 worker_connections 上限", + [f"active={active}", f"理论槽位={capacity}"], + "反向代理通常每个客户端请求还占一个上游连接，实际可承载客户端数低于理论槽位。", + "先确认连接是否被长 SSE/keepalive 占用，再评估 worker_connections、worker_rlimit_nofile 和 worker 数。", + ) + + logs = server.get("logs", {}) + pattern_counts = logs.get("pattern_counts", {}) + log_finding_map = { + "nginx_connection_capacity": ("critical", "nginx", "Nginx 日志明确出现 worker_connections 不足"), + "too_many_open_files": ("critical", "resources", "日志出现文件句柄耗尽"), + "nginx_upstream_timeout": ("warning", "nginx", "Nginx 记录上游超时"), + "db_pool_timeout": ("critical", "mysql", "日志记录数据库连接池等待超时"), + "db_lock_or_deadlock": ("warning", "mysql", "日志记录数据库锁等待或死锁"), + "deerflow_concurrency_reject": ("warning", "deerflow", "日志记录 DeerFlow 并发拒绝"), + "resource_exhaustion": ("critical", "resources", "日志记录系统资源耗尽"), + } + for key, (severity, area, title) in log_finding_map.items(): + count = int(pattern_counts.get(key, 0)) + if count: + _add_finding( + findings, + severity, + area, + f"log_{key}", + title, + [f"最近日志尾部命中 {count} 次"], + "这是实际日志证据，但需按日志时间确认是否与本次卡顿同一时段。", + "打开原日志核对时间、请求和完整堆栈；本报告为防泄密只保存模式计数，不保存原行。", + ) + + mysql = server.get("mysql", {}) + if mysql.get("available"): + ping_p50 = mysql.get("select_1_ms", {}).get("p50") + if ping_p50 is not None and float(ping_p50) >= 100: + _add_finding( + findings, + "warning", + "mysql", + "mysql_ping_slow", + "MySQL 新连接 + SELECT 1 延迟偏高", + [f"p50={float(ping_p50):.1f}ms"], + "包含 mysql 客户端启动、DNS、TCP、认证和查询，说明 DB 路径值得优先调查，但不等同于业务 SQL 本身慢。", + "在应用连接池内复测并抓 PROCESSLIST/慢查询；同时看 TCP timing 区分网络。", + ) + mysql_status = mysql.get("status", {}) + variables = mysql.get("variables", {}) + connected = int(mysql_status.get("Threads_connected") or 0) + max_connections = int(variables.get("max_connections") or 0) + if max_connections and connected >= max_connections * 0.8: + _add_finding( + findings, + "critical" if connected >= max_connections * 0.95 else "warning", + "mysql", + "mysql_connections_high", + "MySQL 当前连接数接近 max_connections", + [f"Threads_connected={connected}", f"max_connections={max_connections}"], + "新请求可能在应用池或 MySQL 握手阶段等待/失败。", + "检查各 Gateway worker 的池总上限、Sleep 连接、泄漏和其他应用连接；不要只盲目调大 max_connections。", + ) + processlist = mysql.get("processlist", {}) + if int(processlist.get("lock_state_count") or 0) or int(processlist.get("active_at_least_5s") or 0): + _add_finding( + findings, + "critical" if int(processlist.get("active_at_least_30s") or 0) else "warning", + "mysql", + "mysql_active_waits", + "MySQL 当前存在长执行或锁状态会话", + [ + f">=5s active={processlist.get('active_at_least_5s', 0)}", + f">=30s active={processlist.get('active_at_least_30s', 0)}", + f"lock state={processlist.get('lock_state_count', 0)}", + f"max active={processlist.get('max_active_seconds', 0)}s", + ], + "这是卡顿时的实时数据库证据，可能直接拖慢 login/agents/skills 的串行查询。", + "由 DBA 查看对应 PROCESSLIST 详情、执行计划和事务持锁者；本脚本不保存 SQL 正文。", + ) + locks = mysql.get("locks", {}) + if int(locks.get("data_lock_waits") or 0) or int(locks.get("pending_metadata_locks") or 0): + _add_finding( + findings, + "critical", + "mysql", + "mysql_lock_waits", + "MySQL performance_schema 显示锁等待", + [ + f"data_lock_waits={locks.get('data_lock_waits', 0)}", + f"pending_metadata_locks={locks.get('pending_metadata_locks', 0)}", + ], + "数据锁或 metadata lock 可让相关查询成批等待；skills 路径的建表 checkfirst 尤其应对照 metadata lock。", + "立即保留锁等待链、持锁事务和表名证据，再由 DBA 处理；避免无证据重启掩盖现场。", + ) + + config = report.get("deerflow", {}).get("config", {}) + backend = str(config.get("database.backend") or "").lower() + if backend == "mysql" and mysql.get("available"): + pool_size = int(config.get("database.pool_size") or 0) + overflow = int(config.get("database.max_overflow") or 0) + gateway_workers = int(server.get("host", {}).get("processes", {}).get("deerflow_gateway", {}).get("processes") or 1) + max_connections = int(mysql.get("variables", {}).get("max_connections") or 0) + theoretical_pool = (pool_size + overflow) * max(1, gateway_workers) + if max_connections and theoretical_pool >= max_connections * 0.8: + _add_finding( + findings, + "warning", + "mysql", + "pool_budget_risk", + "Gateway 理论连接池总量接近 MySQL 上限", + [ + f"(pool_size {pool_size} + overflow {overflow}) × worker {gateway_workers} = {theoretical_pool}", + f"max_connections={max_connections}", + ], + "还未计入其他应用、运维和 MySQL 保留连接，高峰时可能先在池或数据库侧排队。", + "为所有客户端预留余量，结合真实 Max_used_connections/Threads_connected 调整；先排查泄漏。", + ) + + host = server.get("host", {}) + if host.get("supported"): + cpu_count = int(host.get("cpu_count") or 1) + cpu = host.get("cpu", {}) + busy = cpu.get("busy_pct") + iowait = cpu.get("iowait_pct") + load = host.get("load_average") or [] + if busy is not None and float(busy) >= 90: + _add_finding( + findings, + "critical", + "resources", + "cpu_saturated", + "服务器 CPU 已接近饱和", + [f"CPU busy={float(busy):.1f}%", f"CPU cores={cpu_count}", f"load={load}"], + "Gateway、TLS、Python 同步扫描和数据库客户端都可能排队。", + "用 pidstat/top 定位进程和线程；若是技能扫描先修热点，不要只扩机器。", + ) + elif load and float(load[0]) >= cpu_count * 1.2: + _add_finding( + findings, + "warning", + "resources", + "load_high", + "系统 1 分钟负载高于 CPU 容量", + [f"load1={float(load[0]):.2f}", f"CPU cores={cpu_count}"], + "可运行任务或不可中断 IO 等待过多，接口可能在调度层排队。", + "结合 CPU busy、iowait、PSI 和进程 CPU 判断是算力还是 IO。", + ) + if iowait is not None and float(iowait) >= 15: + _add_finding( + findings, + "warning", + "resources", + "cpu_iowait_high", + "CPU iowait 偏高", + [f"iowait={float(iowait):.1f}%"], + "磁盘或网络块设备等待可能拖慢 MySQL、SQLite、技能目录扫描与日志。", + "用 iostat -xz 1 10 对准 MySQL/技能所在盘确认 await 和 util。", + ) + memory = host.get("memory", {}) + available_pct = memory.get("available_pct") + if available_pct is not None and float(available_pct) < 10: + _add_finding( + findings, + "critical", + "resources", + "memory_low", + "可用内存不足", + [f"MemAvailable={float(available_pct):.1f}%", f"Swap used={memory.get('swap_used_pct')}%"], + "回收、换页或 OOM 会造成所有接口抖动，重启后短暂变快也与此相符。", + "定位 RSS 最大进程、容器内存限制和 OOM 记录；为模型/数据库/文件缓存保留余量。", + ) + disk_util = host.get("disk_io", {}).get("max_util_pct") + if disk_util is not None and float(disk_util) >= 85: + _add_finding( + findings, + "warning", + "resources", + "disk_busy", + "块设备采样期间接近满负载", + [ + f"device={host.get('disk_io', {}).get('busiest_device')}", + f"util={float(disk_util):.1f}%", + ], + "可能拖慢数据库和技能/Agent 文件扫描；一次 1 秒采样只代表当前窗口。", + "卡顿持续期间运行 iostat -xz 1 10，确认 r_await/w_await 与具体挂载点。", + ) + for path, usage in host.get("disk_space", {}).items(): + free_pct = usage.get("free_pct") + if free_pct is not None and float(free_pct) < 10: + _add_finding( + findings, + "warning", + "resources", + "disk_space_low", + f"文件系统 {path} 剩余空间不足", + [f"free={float(free_pct):.1f}%"], + "数据库临时表、WAL、日志和容器写层可能受影响。", + "清理前先确认数据归属和备份；不要直接删除数据库/卷。", + ) + for role, process in host.get("processes", {}).items(): + fd_usage = process.get("max_fd_usage_pct") + if fd_usage is not None and float(fd_usage) >= 80: + _add_finding( + findings, + "critical" if float(fd_usage) >= 95 else "warning", + "resources", + "process_fd_high", + f"{role} 文件句柄接近进程上限", + [f"最大 FD 占用={float(fd_usage):.1f}%", f"进程数={process.get('processes')}"], + "新 socket、日志或文件打开会失败，表现为连接卡顿/502。", + "查 /proc//fd 类型与泄漏，结合 limits/worker_connections 调整。", + ) + if cpu_count <= 2: + _add_finding( + findings, + "info", + "resources", + "small_cpu_capacity", + "服务器 CPU 核数较少", + [f"CPU cores={cpu_count}"], + "这不是当前卡顿的直接证据，但单 worker Gateway、Nginx、数据库或容器共享时余量有限。", + "先用实测 CPU/PSI/并发退化证明容量瓶颈，再决定扩容。", + ) + + docker = server.get("docker", {}) + for container in docker.get("containers", []): + limits = container.get("limits", {}) + stats = container.get("stats", {}) + if limits.get("oom_killed"): + _add_finding( + findings, + "critical", + "resources", + "container_oom", + f"容器 {container.get('name')} 曾被 OOM kill", + [f"restart_count={limits.get('restart_count')}"], + "内存限制或宿主内存不足已经造成服务中断。", + "查 docker events/kernel OOM 时间和容器 RSS，调整限制或修复内存增长。", + ) + if stats.get("memory_pct") is not None and float(stats["memory_pct"]) >= 90: + _add_finding( + findings, + "critical", + "resources", + "container_memory_high", + f"容器 {container.get('name')} 内存接近限制", + [f"memory={stats.get('memory_usage')}", f"memory_pct={stats.get('memory_pct')}%"], + "容器回收/OOM 风险可拖慢或重启服务。", + "结合宿主内存和容器内进程 RSS判断是否扩限或修复增长。", + ) + + catalogue = server.get("catalogue_scan", {}) + skills = catalogue.get("skills", {}) + if skills.get("exists") and float(skills.get("metadata_walk_ms") or 0) >= 500: + _add_finding( + findings, + "warning", + "deerflow", + "skill_directory_walk_slow", + "技能目录仅元数据遍历就很慢", + [ + f"目录={skills.get('directories')}", + f"文件={skills.get('matching_files')}", + f"walk={float(skills.get('metadata_walk_ms')):.1f}ms", + ], + "/api/skills 会进一步读取和 YAML 解析每个技能文件，实际阻塞可能更长。", + "检查技能目录是否位于慢盘/NFS、是否残留过多文件，并用 interference 探针确认事件循环影响。", + ) + elif int(skills.get("matching_files") or 0) >= 500: + _add_finding( + findings, + "info", + "deerflow", + "large_skill_catalogue", + "技能目录文件量较大", + [f"匹配文件={skills.get('matching_files')}", f"metadata walk={skills.get('metadata_walk_ms')}ms"], + "当前 /api/skills 每请求全量扫描，前端重复刷新会放大成本；文件量本身不是卡顿证明。", + "对照访问日志中的 /api/skills 请求簇与 SKILL_VIEW_CACHE/health 干扰数据。", + ) + + admin = report.get("deerflow", {}).get("admin_concurrency", {}) + if admin.get("available"): + active = admin.get("active", {}) + count = int(active.get("count") or 0) + max_elapsed = int(active.get("max_elapsed_seconds") or 0) + limits = admin.get("limits", {}).get("settings", {}) + default_limit = int(limits.get("default_limit") or 0) + if count and default_limit and count >= default_limit: + _add_finding( + findings, + "warning", + "deerflow", + "active_runs_at_limit", + "活动运行数已达到默认模型并发限制", + [f"active={count}", f"default_limit={default_limit}", f"max_elapsed={max_elapsed}s"], + "新 Agent 请求可能收到 429；这不是 Nginx 排队，而是应用明确限流。", + "按模型查看 active-runs，处理异常长运行或评估上游模型真实配额后再调限制。", + ) + elif max_elapsed >= 300: + _add_finding( + findings, + "warning", + "deerflow", + "long_active_run", + "存在持续很久的活动 Agent 运行", + [f"active={count}", f"max_elapsed={max_elapsed}s"], + "长 SSE/模型/工具任务会持续占用模型配额、连接和内存，但不一定异常。", + "在管理员并发页核对对应任务是否仍在产出；仅在确认卡死后中断。", + ) + + if not findings: + _add_finding( + findings, + "info", + "overall", + "no_decisive_signal", + "本次快照没有抓到决定性异常", + ["没有命中脚本的高置信阈值"], + "间歇性卡顿恢复后采样只能说明恢复后的状态，不能排除曾发生过连接池、锁或资源尖峰。", + "在卡顿正在发生时重新运行，并启用 --deerflow-suite 与 --interference。", + ) + findings.sort(key=lambda item: (SEVERITY_ORDER.get(str(item["severity"]), 99), str(item["area"]), str(item["code"]))) + return findings + + +def build_quick_verdict(findings: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + labels = { + "nginx": "Nginx/入口代理", + "mysql": "MySQL/数据库", + "deerflow": "DeerFlow 并发或热点路径", + "application": "接口自身或下游服务", + "resources": "服务器资源", + "network": "网络/DNS/TLS", + "overall": "暂未定位", + } + decisive = [item for item in findings if item.get("severity") in {"critical", "warning"}] + if not decisive: + return { + "headline": "本次没有抓到决定性异常；建议在卡顿正在发生时再次运行", + "areas": [], + "confidence": "low", + "evidence_titles": [str(item.get("title")) for item in findings[:3]], + } + areas: list[str] = [] + for item in decisive: + label = labels.get(str(item.get("area")), str(item.get("area"))) + if label not in areas: + areas.append(label) + if len(areas) >= 3: + break + primary_severity = str(decisive[0].get("severity")) + return { + "headline": "当前证据优先指向：" + "；同时关注：".join(areas), + "areas": areas, + "confidence": "high" if primary_severity == "critical" else "medium", + "evidence_titles": [str(item.get("title")) for item in decisive[:3]], + "note": "这是基于当前快照的定位方向，不是对根因的绝对证明。", + } + + +def _origin_for_request(url: str) -> str: + scheme, hostname, port = _origin_tuple(url) + try: + rendered_host = f"[{hostname}]" if ipaddress.ip_address(hostname).version == 6 else hostname + except ValueError: + rendered_host = hostname + default_port = 443 if scheme == "https" else 80 + if port != default_port: + rendered_host = f"{rendered_host}:{port}" + return f"{scheme}://{rendered_host}" + + +def _report_output_path(raw: str | None) -> Path: + if raw: + requested = Path(raw).expanduser() + parent = requested.parent.resolve() + output = parent / requested.name + if output.is_symlink(): + raise ValueError(f"refusing to replace symlink output path: {output}") + if output.exists() and not output.is_file(): + raise ValueError(f"output path exists and is not a regular file: {output}") + return output + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + return Path.cwd().resolve() / f"deerflow-latency-report-{stamp}.json" + + +def write_private_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + temporary_path = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + with contextlib.suppress(OSError): + os.chmod(temporary_path, 0o600) + os.replace(temporary_path, path) + except Exception: + with contextlib.suppress(OSError): + os.close(descriptor) + with contextlib.suppress(OSError): + temporary_path.unlink() + raise + with contextlib.suppress(OSError): + os.chmod(path, 0o600) + + +def _format_ms(value: Any) -> str: + if value is None: + return "-" + try: + return f"{float(value):.1f}ms" + except (TypeError, ValueError): + return "-" + + +def print_console_summary(report: Mapping[str, Any], output_path: Path | None) -> None: + print("\n=== DeerFlow 接口卡顿诊断 ===") + print(f"时间: {report.get('started_at')}") + print("说明: HTTP/1.1 新连接测量；请求体、响应体、Token、Cookie 和查询参数值未写入报告。") + print("注意: 报告仍含路径、IP、容器名等基础设施元数据，请勿公开发布。") + warnings = report.get("security", {}).get("warnings", []) + if warnings: + print("安全警告:") + for warning in warnings: + print(f" - {warning}") + print("\n接口测量:") + for target in report.get("http", {}).get("targets", []): + summary = target.get("summary", {}) + timings = summary.get("timings_ms", {}) + print( + f" [{target.get('label')}] {target.get('url')} " + f"成功 {summary.get('successful_transport')}/{summary.get('requests')} " + f"总耗时p50={_format_ms(timings.get('total', {}).get('p50'))} " + f"等待首响应p50={_format_ms(timings.get('response_wait', {}).get('p50'))} " + f"DNS/TCP/TLS={_format_ms(timings.get('dns', {}).get('p50'))}/" + f"{_format_ms(timings.get('tcp_connect', {}).get('p50'))}/" + f"{_format_ms(timings.get('tls_handshake', {}).get('p50'))} " + f"状态={summary.get('status_counts')}" + ) + + host = report.get("server", {}).get("host", {}) + if host.get("supported"): + print("\n服务器快照:") + print(f" CPU核数={host.get('cpu_count')} busy={host.get('cpu', {}).get('busy_pct')}% iowait={host.get('cpu', {}).get('iowait_pct')}% load={host.get('load_average')}") + print(f" 可用内存={host.get('memory', {}).get('available_pct')}% Swap使用={host.get('memory', {}).get('swap_used_pct')}% 最忙磁盘={host.get('disk_io', {}).get('busiest_device')} util={host.get('disk_io', {}).get('max_util_pct')}%") + + verdict = report.get("quick_verdict", {}) + if verdict: + print("\n>>> 一键判断:") + print(f" {verdict.get('headline')}") + for title in verdict.get("evidence_titles", []): + print(f" - {title}") + if verdict.get("note"): + print(f" 说明: {verdict.get('note')}") + + print("\n优先结论:") + for index, finding in enumerate(report.get("findings", []), start=1): + severity = {"critical": "严重", "warning": "警告", "info": "提示", "ok": "正常"}.get(str(finding.get("severity")), str(finding.get("severity"))) + print(f" {index}. [{severity}/{finding.get('area')}] {finding.get('title')}") + for evidence in finding.get("evidence", []): + print(f" 证据: {evidence}") + print(f" 判断: {finding.get('interpretation')}") + print(f" 下一步: {finding.get('next_step')}") + + missing = report.get("coverage", {}).get("unavailable", []) + if missing: + print("\n未完成检查:") + for item in missing: + print(f" - {item}") + if output_path: + print(f"\n凭据脱敏 JSON 报告: {output_path}") + + +def _normalize_easy_url(raw: str) -> str: + value = raw.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1].strip() + if not value: + raise ValueError("接口地址不能为空") + if "://" not in value: + value = "https://" + value + return value + + +def _option_present(arguments: Sequence[str], *names: str) -> bool: + return any(argument in names or any(argument.startswith(name + "=") for name in names) for argument in arguments) + + +def _verified_local_gateway_origin() -> str | None: + """Return a loopback Gateway origin only after checking its public identity.""" + ports: list[int] = [8001] + config_path = find_config_path(None) + if config_path: + config = read_safe_config(config_path) + configured_port = config.get("gateway.port") + if isinstance(configured_port, int) and 1 <= configured_port <= 65535: + ports.insert(0, configured_port) + for port in dict.fromkeys(ports): + origin = f"http://127.0.0.1:{port}" + request = urllib.request.Request( + origin + "/api/health", + method="GET", + headers={"User-Agent": USER_AGENT, "Accept": "application/json"}, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirectHandler()) + try: + with opener.open(request, timeout=1.5) as response: + if response.status != 200: + continue + payload = json.loads(response.read(4096).decode("utf-8")) + except (OSError, TimeoutError, UnicodeError, ValueError, urllib.error.URLError, urllib.error.HTTPError): + continue + if isinstance(payload, dict) and payload.get("status") == "healthy" and payload.get("service") == "deer-flow-gateway": + return origin + return None + + +def _easy_optional_environment_args() -> list[str]: + arguments: list[str] = [] + config_path = os.environ.get("DEERFLOW_DIAG_CONFIG") + mysql_file = os.environ.get("DEERFLOW_DIAG_MYSQL_CNF") + mysql_url = os.environ.get("DEERFLOW_DIAG_MYSQL_URL") + nginx_status = os.environ.get("DEERFLOW_DIAG_NGINX_STATUS_URL") + admin_token = os.environ.get("DEERFLOW_DIAG_ADMIN_TOKEN") + if config_path: + arguments.extend(["--config", config_path]) + if mysql_file: + arguments.extend(["--mysql-defaults-file", mysql_file]) + elif mysql_url: + arguments.extend(["--mysql", "--mysql-url-env", "DEERFLOW_DIAG_MYSQL_URL"]) + if nginx_status: + arguments.extend(["--nginx-status-url", nginx_status]) + if admin_token: + arguments.extend(["--admin-token-env", "DEERFLOW_DIAG_ADMIN_TOKEN"]) + return arguments + + +def _easy_auth_from_environment() -> list[str]: + if os.environ.get("DEERFLOW_DIAG_TOKEN"): + return ["--bearer-env", "DEERFLOW_DIAG_TOKEN"] + if os.environ.get("DEERFLOW_DIAG_COOKIE"): + return ["--cookie-env", "DEERFLOW_DIAG_COOKIE"] + return [] + + +def _store_easy_secret(kind: str, value: str) -> tuple[list[str], str | None]: + if not value: + return [], None + env_name = f"_DEERFLOW_DIAG_EASY_{kind.upper()}_{os.getpid()}" + os.environ[env_name] = value + option = "--bearer-env" if kind == "token" else "--cookie-env" + return [option, env_name], env_name + + +def _interactive_easy_arguments() -> tuple[list[str], list[str]]: + print("\n=== DeerFlow 一键卡顿诊断 ===") + print("只需选择接口类型并粘贴地址；不会修改、重启或替换后端。") + target_url = _normalize_easy_url(input("\n请粘贴卡顿接口 URL: ")) + print("\n接口类型：") + print(" 1. 普通只读 GET / Skills（默认，深度检查，会请求目标 4 次）") + print(" 2. 登录 POST（只请求 1 次）") + print(" 3. Agent SSE POST（只请求 1 次，只等待首个响应实体字节）") + mode = input("请选择 [1/2/3，默认 1]: ").strip() or "1" + if mode not in {"1", "2", "3"}: + raise ValueError("接口类型只能选择 1、2 或 3") + + arguments = [ + "--easy-mode", + "--url", + target_url, + "--server-checks", + "--deerflow-suite", + "--concurrency", + "1", + "--connect-timeout", + "5", + ] + cleanup_env_names: list[str] = [] + if mode == "1": + arguments.extend( + [ + "--count", + "3", + "--interference", + "--first-byte-timeout", + "30", + "--total-timeout", + "60", + ] + ) + else: + body_path = input("请求 JSON 文件路径: ").strip() + if not body_path: + raise ValueError("POST 诊断必须提供请求 JSON 文件") + resolved_body_path = Path(body_path).expanduser().resolve() + if not resolved_body_path.is_file(): + raise ValueError(f"请求文件不存在: {resolved_body_path}") + arguments.extend( + [ + "--method", + "POST", + "--header", + "Content-Type: application/json", + "--data-file", + str(resolved_body_path), + "--count", + "1", + ] + ) + if mode == "2": + arguments.extend(["--read-mode", "full", "--first-byte-timeout", "60", "--total-timeout", "90"]) + else: + arguments.extend( + [ + "--header", + "Accept: text/event-stream", + "--read-mode", + "first-byte", + "--first-byte-timeout", + "180", + "--total-timeout", + "240", + ] + ) + + environment_auth = _easy_auth_from_environment() + if environment_auth: + env_name = environment_auth[-1] + print(f"检测到 {env_name}，将用于用户输入的目标地址；值不会写入报告。") + arguments.extend(environment_auth) + else: + default_auth = "1" if mode == "3" else "0" + auth_choice = input(f"鉴权方式 [0=无, 1=Bearer Token, 2=Cookie，默认 {default_auth}]: ").strip() or default_auth + if auth_choice not in {"0", "1", "2"}: + raise ValueError("鉴权方式只能选择 0、1 或 2") + if auth_choice == "1": + secret_args, env_name = _store_easy_secret("token", getpass.getpass("Bearer Token（输入不显示）: ").strip()) + arguments.extend(secret_args) + if env_name: + cleanup_env_names.append(env_name) + elif auth_choice == "2": + secret_args, env_name = _store_easy_secret("cookie", getpass.getpass("Cookie 值（输入不显示，不含 Cookie: 前缀）: ").strip()) + arguments.extend(secret_args) + if env_name: + cleanup_env_names.append(env_name) + + local_origin = _verified_local_gateway_origin() + if local_origin: + arguments.extend(["--easy-local-origin", local_origin]) + print(f"已确认本机 DeerFlow Gateway：{local_origin}；仅对比公开 health，不会向它复制目标查询参数、凭据或请求体。") + on_server = input("当前脚本是否运行在这个 DeerFlow 的实际部署服务器上？[Y/n]: ").strip().lower() + if on_server in {"", "y", "yes", "是"}: + arguments.append("--easy-target-host-confirmed") + elif on_server not in {"n", "no", "否"}: + raise ValueError("请回答 y 或 n") + arguments.extend(_easy_optional_environment_args()) + return arguments, cleanup_env_names + + +def _positional_easy_arguments(raw_arguments: Sequence[str]) -> list[str]: + target_url = _normalize_easy_url(str(raw_arguments[0])) + rest = list(raw_arguments[1:]) + arguments = [ + "--easy-mode", + "--url", + target_url, + "--count", + "1", + "--concurrency", + "1", + "--deerflow-suite", + "--server-checks", + "--connect-timeout", + "5", + "--first-byte-timeout", + "30", + "--total-timeout", + "60", + ] + if not _option_present(rest, "--bearer-env", "--cookie-env", "--header-env", "--header"): + arguments.extend(_easy_auth_from_environment()) + local_origin = _verified_local_gateway_origin() + if local_origin: + arguments.extend(["--easy-local-origin", local_origin]) + if os.environ.get("DEERFLOW_DIAG_ON_SERVER", "").strip().lower() in {"1", "true", "yes", "y"}: + arguments.append("--easy-target-host-confirmed") + arguments.extend(_easy_optional_environment_args()) + arguments.extend(rest) + return arguments + + +def prepare_cli_arguments(raw_arguments: Sequence[str]) -> tuple[list[str], list[str]]: + arguments = list(raw_arguments) + if not arguments: + if not sys.stdin.isatty(): + raise ValueError("无参数模式需要交互终端；非交互执行请使用: python3 diagnose_deerflow_latency.py 'https://地址'") + return _interactive_easy_arguments() + if not arguments[0].startswith("-"): + return _positional_easy_arguments(arguments), [] + return arguments, [] + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=("Diagnose DeerFlow latency across HTTP/Nginx/database/concurrency/host layers. Run it while the slowdown is happening for the strongest evidence.")) + parser.add_argument("--url", action="append", default=[], help="Target URL; may be repeated") + parser.add_argument( + "--url-env", + action="append", + default=[], + metavar="ENV_NAME", + help="Read a target URL from an environment variable; may be repeated", + ) + parser.add_argument( + "--compare-url", + "--direct-url", + action="append", + default=[], + dest="compare_url", + help="Same request through another path, e.g. http://127.0.0.1:8001/...", + ) + parser.add_argument( + "--allow-cross-origin-compare", + action="store_true", + help="Allow credentials/request body to reach an explicitly supplied comparison origin", + ) + parser.add_argument("--method", default="GET", help="HTTP method (default: GET)") + parser.add_argument( + "--query-env", + action="append", + default=[], + metavar="KEY=ENV_NAME", + help="Append/replace a query value from an environment variable on every target URL", + ) + parser.add_argument("--header", action="append", default=[], help="Request header NAME:VALUE; secrets should use --header-env") + parser.add_argument( + "--header-env", + action="append", + default=[], + metavar="HEADER=ENV_NAME", + help="Read a header value from an environment variable", + ) + parser.add_argument("--bearer-env", help="Environment variable containing a Bearer token") + parser.add_argument("--cookie-env", help="Environment variable containing a Cookie header value") + parser.add_argument("--data-file", help="Request body file; '-' reads stdin. Body content is never reported") + parser.add_argument("--count", type=int, help="Total fresh-connection samples (safe methods default 5; unsafe default 1)") + parser.add_argument("--concurrency", type=int, default=1, help="Concurrent samples; keep low on production (default 1)") + parser.add_argument("--interval", type=float, default=0.2, help="Seconds between sequential samples") + parser.add_argument( + "--allow-unsafe-replay", + action="store_true", + help="Explicitly allow repeating/concurrently sending POST/PUT/PATCH/DELETE requests", + ) + parser.add_argument( + "--read-mode", + choices=("auto", "headers", "first-byte", "full"), + default="auto", + help="auto uses first-byte for SSE and full for other responses", + ) + parser.add_argument("--connect-timeout", type=float, default=5.0) + parser.add_argument("--first-byte-timeout", type=float, default=30.0) + parser.add_argument("--read-timeout", type=float, default=5.0) + parser.add_argument("--total-timeout", type=float, default=120.0, help="Hard deadline per HTTP request") + parser.add_argument("--max-response-bytes", type=int, default=1024 * 1024) + parser.add_argument("--follow-redirects", action="store_true", help="Follow at most --max-redirects redirects") + parser.add_argument("--max-redirects", type=int, default=3) + parser.add_argument("--proxy", help="Explicit http:// forward proxy; credentials are never reported") + parser.add_argument("--proxy-env", help="Environment variable holding an explicit http:// proxy URL") + parser.add_argument("--use-env-proxy", action="store_true", help="Use HTTP(S)_PROXY/NO_PROXY environment settings") + parser.add_argument("--ip-family", choices=("auto", "ipv4", "ipv6"), default="auto") + parser.add_argument("--ca-file", help="Custom CA bundle for HTTPS") + parser.add_argument("--insecure", action="store_true", help="Disable TLS certificate verification (recorded as unsafe)") + parser.add_argument( + "--allow-insecure-http", + action="store_true", + help="Allow credentials, environment-backed query values, or a request body over non-loopback plaintext HTTP", + ) + parser.add_argument( + "--health-baselines", + action=argparse.BooleanOptionalAction, + default=True, + help="Probe /health, /api/health and /api/langgraph/health once per origin", + ) + parser.add_argument( + "--deerflow-suite", + action="store_true", + help="Probe safe DeerFlow baselines (auth/options and, with auth, me/agents/skills)", + ) + parser.add_argument( + "--interference", + action="store_true", + help="Send one extra target request while repeatedly probing /api/health", + ) + parser.add_argument("--interference-url", help="Custom sentinel URL (default: target origin /api/health)") + parser.add_argument("--sentinel-interval", type=float, default=0.25) + parser.add_argument("--max-sentinel-probes", type=int, default=120) + parser.add_argument( + "--server-checks", + action=argparse.BooleanOptionalAction, + default=None, + help="Collect Linux/Nginx/Docker/log/database checks (auto-enabled on Linux)", + ) + parser.add_argument("--resource-sample-seconds", type=float, default=1.0) + parser.add_argument("--config", help="DeerFlow config.yaml path") + parser.add_argument("--log", action="append", default=[], help="Additional log file to scan; may be repeated") + parser.add_argument("--log-tail-bytes", type=int, default=4 * 1024 * 1024) + parser.add_argument( + "--mysql", + action=argparse.BooleanOptionalAction, + default=None, + help="Run read-only MySQL checks (requires this flag, --mysql-defaults-file, or --mysql-host)", + ) + parser.add_argument("--mysql-url-env", default="MYSQL_DATABASE_URL", help="Environment variable holding MySQL URL") + parser.add_argument("--mysql-defaults-file", help="Existing mysql client option file (recommended alternative to URL env)") + parser.add_argument("--mysql-host", help="Host-only TCP test when credentials are unavailable") + parser.add_argument("--mysql-port", type=int, default=3306) + parser.add_argument("--mysql-probes", type=int, default=3) + parser.add_argument("--nginx-status-url", help="Optional Nginx stub_status URL") + parser.add_argument("--admin-origin", help="Gateway origin for admin read-only concurrency APIs") + parser.add_argument("--admin-token-env", help="Environment variable holding an admin JWT") + parser.add_argument( + "--allow-cross-origin-admin", + action="store_true", + help="Allow the admin JWT to be sent to an explicitly supplied origin different from the primary target", + ) + parser.add_argument("--output", help="JSON report path (default: ./deerflow-latency-report-