Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into gzy
This commit is contained in:
commit
41ce95cbf1
211 files changed
+20763
-2860
No files matched your search
@@ -17,7 +17,7 @@ SERPER_API_KEY=your-serper-api-key
|
||||
# AUTH_JWT_SECRET must stay stable across restarts, or existing logins expire.
|
||||
# GATEWAY_CORS_ORIGINS must include the exact frontend origins, separated by commas.
|
||||
AUTH_JWT_SECRET=i3_BiBK44sYHiFIkChhDs46y57ZewJKO3diQDw01anI
|
||||
GATEWAY_CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173,http://localhost:5183,http://localhost:5174,http://127.0.0.1:5174,http://localhost:4200
|
||||
GATEWAY_CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173,http://localhost:5174,http://127.0.0.1:5174,http://localhost:5183,http://127.0.0.1:5183,http://localhost:4200
|
||||
GATEWAY_CORS_ALLOW_ALL=0
|
||||
GATEWAY_ENABLE_DOCS=true
|
||||
GATEWAY_WORKERS=2
|
||||
|
||||
@@ -163,7 +163,7 @@ Lead-agent middlewares are assembled in strict append order across `packages/har
|
||||
6. **GuardrailMiddleware** - Pre-tool-call authorization via pluggable `GuardrailProvider` protocol (optional, if `guardrails.enabled` in config). Evaluates each tool call and returns error ToolMessage on deny. Three provider options: built-in `AllowlistProvider` (zero deps), OAP policy providers (e.g. `aport-agent-guardrails`), or custom providers. See [docs/GUARDRAILS.md](docs/GUARDRAILS.md) for setup, usage, and how to implement a provider.
|
||||
7. **SandboxAuditMiddleware** - Audits sandboxed shell/file operations for security logging before tool execution continues
|
||||
8. **ToolErrorHandlingMiddleware** - Converts tool exceptions into error `ToolMessage`s so the run can continue instead of aborting
|
||||
9. **SummarizationMiddleware** - Context reduction when approaching token limits (optional, if enabled)
|
||||
9. **SummarizationMiddleware** (`DeerFlowSummarizationMiddleware`) - Context reduction when approaching token limits (optional, if enabled). **Non-destructive**: overrides the stock `before_model` (which persists a `RemoveMessage(REMOVE_ALL_MESSAGES)`) to a no-op and instead compresses **transiently** in `wrap_model_call` — the model request is replaced with `[summary, *recent]` while the full original conversation stays in state (so the frontend keeps rendering the user's real Q&A). The `summary` message is `hide_from_ui` and never persisted; summaries are cached per-thread (bounded, sticky boundary) to avoid an LLM call on every model invocation. Tests: `tests/test_summarization_middleware.py`
|
||||
10. **TodoListMiddleware** - Task tracking with `write_todos` tool (optional, if plan_mode)
|
||||
11. **TokenUsageMiddleware** - Records token usage metrics when token tracking is enabled (optional)
|
||||
12. **TitleMiddleware** - Two-phase auto-title (never shows "untitled"): `before_model` sets an instant question-derived *provisional* title the moment the first user message arrives (no LLM, zero latency), then `after_model` asynchronously generates the final LLM title and replaces it (tracked via the `title_provisional` state flag). Normalizes structured message content before prompting the title model
|
||||
@@ -175,7 +175,7 @@ Lead-agent middlewares are assembled in strict append order across `packages/har
|
||||
18. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, interrupts via `Command(goto=END)` (must be last)
|
||||
8a. **ToolMetricsMiddleware** - Appended immediately after `ToolErrorHandlingMiddleware` in `_build_runtime_middlewares`, so it sits *deepest* in the `wrap_tool_call` chain. Records one `tool_call_metrics` row per tool/skill/MCP invocation — timing duration and bucketing failures into a coarse `error_category` (rate_limited / ip_blocked / auth_error / timeout / network_error / not_found / empty_result / tool_error). It catches both raised exceptions (before #8 converts them) and *self-handled* errors returned as error `ToolMessage`s (search tools that swallow HTTP 429/403). It also captures the targeted Agent Skill into `tool_call_metrics.skill_name`: the lead agent invokes a skill by `read_file`-ing its `/mnt/skills/<category>/<name>/SKILL.md` path (per the `<skill_system>` prompt block — there is no dedicated "use skill" tool), so `skill_name` is recovered by scanning every tool call's string arguments for the `skills/{public,custom}/{name}/` path shape (`_SKILL_PATH_RE`), plus the explicit `name` argument of `skill_view`/`skill_manage`/`skill_archive`. `skill_view`-specific success detection handles its swallowed failures (`Skill 'x' not found.` returned as a plain string rather than raised). Regression coverage: `tests/test_tool_metrics_middleware.py`. Writes are best-effort fire-and-forget. **Per-run skill merging**: `SqlToolMetricsStore.record` upserts skill rows by `(run_id, skill_name)` instead of inserting one row per call — a skill touched by several tool calls / retries inside one run collapses to a single row. Any success in the run wins (a skill that failed then finally succeeded is recorded as one `success`); a skill that only ever failed yields one `error` row whose `error_message` accumulates every attempt's message; durations are summed. Non-skill tool calls, and skill calls with no `run_id`, keep one row each. The read→merge→write is guarded by a per-event-loop `asyncio.Lock`. Surfaced by the admin `GET /api/admin/tool-metrics` router (which has a `kind=skill` filter and a per-skill `/summary` breakdown).
|
||||
9. **PromptPrefixMiddleware** - Injects the admin-configured prompt prefix into the model request only. The prefix is tagged onto the first human message's `additional_kwargs['prompt_prefix']` by the Gateway (`app.gateway.services`) and prepended to content here at call time, so the persisted/displayed message stays prefix-free
|
||||
10. **SummarizationMiddleware** - Context reduction when approaching token limits (optional, if enabled)
|
||||
10. **SummarizationMiddleware** (`DeerFlowSummarizationMiddleware`) - Context reduction when approaching token limits (optional, if enabled). **Non-destructive**: overrides the stock `before_model` (which persists a `RemoveMessage(REMOVE_ALL_MESSAGES)`) to a no-op and instead compresses **transiently** in `wrap_model_call` — the model request is replaced with `[summary, *recent]` while the full original conversation stays in state (so the frontend keeps rendering the user's real Q&A). The `summary` message is `hide_from_ui` and never persisted; summaries are cached per-thread (bounded, sticky boundary) to avoid an LLM call on every model invocation. Tests: `tests/test_summarization_middleware.py`
|
||||
11. **TodoListMiddleware** - Task tracking with `write_todos` tool (optional, if plan_mode)
|
||||
12. **TokenUsageMiddleware** - Records token usage metrics when token tracking is enabled (optional)
|
||||
13. **TitleMiddleware** - Two-phase auto-title (never shows "untitled"): `before_model` sets an instant question-derived *provisional* title the moment the first user message arrives (no LLM, zero latency), then `after_model` asynchronously generates the final LLM title and replaces it (tracked via the `title_provisional` state flag). Normalizes structured message content before prompting the title model
|
||||
@@ -237,6 +237,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA
|
||||
| **Tags** (`/api/tags`) | `GET /` - list tags (optional `tag_type` filter; open to all); `POST /` - create tag (admin); `PUT /{id}` / `DELETE /{id}` - update/delete tag (admin); `GET /assignments/{target_type}/{target_id}` - tags on a resource; `PUT /assignments/{target_type}/{target_id}` - replace a resource's tags (admin). Tags are typed `agent`/`skill`/`scheduled_task` and may only tag resources of their own type. |
|
||||
| **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete |
|
||||
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
|
||||
| **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics — powers the chats page's 总数/总页数 display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` |
|
||||
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
|
||||
| **Suggestions** (`/api/threads/{id}/suggestions`) | `POST /` - generate follow-up questions; rich list/block model content is normalized before JSON parsing |
|
||||
| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /../messages` - thread messages with feedback; `GET /../token-usage` - aggregate tokens |
|
||||
@@ -250,13 +251,13 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA
|
||||
| **Scheduled Tasks** (`/api/scheduled-tasks`) | Per-user cron-like agent tasks. `GET/POST /` - list/create owned tasks; `PATCH/DELETE /{id}` - update/delete (owner); `POST /{id}/{pause,resume,trigger}`; `GET /{id}/runs`, `GET /runs/{rid}`, run-file preview/edit/download; `GET /runs/{rid}/messages` - agent messages for the run's underlying agent run (used to inspect a stuck/running run; authorized via `_load_run`); `GET /runs/{rid}/html` - generated HTML document for an `html_page` run; `DELETE /runs/{rid}` - owner-scoped, with an admin fallback so admins can delete any user's run; publish/subscribe under `/{id}/{publish,unpublish,subscribe}` and `GET /published`. **Admin (cross-user):** `GET /admin/all` - every task; `PATCH/DELETE /admin/{id}` - edit/delete any task; `POST /admin/{id}/trigger`; `GET /admin/{id}/runs`. Admin routes require `system_role == "admin"`. |
|
||||
| **HTML Page Favorites** (`/api/html-page-favorites`) | Bookmarked generated HTML pages, owner-scoped. `GET /` - list; `POST /` `{page_id, title?, description?, tags?}` - bookmark a `scheduled_html_pages` row (extracts a reusable style/layout summary via GLM5 at favorite time); a page can only be favorited once per user (`DuplicateFavoriteError` → HTTP 409). `GET/DELETE /{id}` - read/delete (the create-task reference picker deletes favorites here). A favorite can be selected as a style reference when creating a new `html_page` task. |
|
||||
| **Public Scheduled Tasks** (`/api/public/scheduled-tasks`) | **Unauthenticated** mirror for embedding. Adds `GET /runs/{rid}/html` and `GET /{task_id}/latest-html` for the `html_page` preview; the run result keeps `result.html` (only file paths are redacted). |
|
||||
| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` → `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "已改用规则抽取". The "保存到知识库" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). |
|
||||
| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` → `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "已改用规则抽取". The "保存到知识库" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/file` - **import an uploaded file** (multipart: `file` + optional `title`/`tags`/`status`/`model_name`) into wiki note(s): the file is converted to Markdown (md/txt read directly, pdf/word/ppt/excel via `convert_file_to_markdown`), distilled by the same wiki-capture LLM pipeline as thread capture (`source_type="file"`), and falls back to a verbatim **draft** note when the model returns nothing usable (`llm_fallback:true`); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities, graph-blacklist filtered); `GET/POST /graph/blacklist` + `DELETE /graph/blacklist/{id}` - graph node blacklist (labels = entity names / note titles, case-insensitive match; blacklisted nodes and their edges are dropped from `/graph` and the wiki-export artifacts; table `knowledge_graph_blacklist`, auto-created); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). |
|
||||
|
||||
Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → Gateway.
|
||||
|
||||
### Admin Leaderboard Snapshots (`/api/admin/users/leaderboard*`)
|
||||
|
||||
The admin 用户排行榜 is served from **precomputed per-day snapshots** in `admin_leaderboard_daily_stats` (keyed by `(stat_date, settings_hash)`), not by aggregating raw rows on every request. A background scheduler inside the Gateway (`_leaderboard_snapshot_scheduler_loop` in `app/gateway/app.py`, singleton file-locked) ticks every `ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS` (default 300s): it prewarms recent days, then builds any `queued`/`error`/stale-`running` day. `GET /leaderboard` merges the per-day snapshots for the requested range. **Today** is rebuilt whenever its snapshot is older than `ADMIN_LEADERBOARD_TODAY_TTL_SECONDS` (default 300s); a past day, once `ready`, is treated as fresh forever.
|
||||
The admin 用户排行榜 is served from **precomputed per-day snapshots** in `admin_leaderboard_daily_stats` (keyed by `(stat_date, settings_hash)`), not by aggregating raw rows on every request. A background scheduler inside the Gateway (`_leaderboard_snapshot_scheduler_loop` in `app/gateway/app.py`, singleton file-locked) ticks every `ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS` (default 300s): it prewarms recent days, then builds any `queued`/`error`/stale-`running` day. `GET /leaderboard` merges the per-day snapshots for the requested range. **Today** is rebuilt whenever its snapshot is older than `ADMIN_LEADERBOARD_TODAY_TTL_SECONDS` (default 43200s = half a day, so the leaderboard refreshes ~twice daily); a past day, once `ready`, is treated as fresh forever.
|
||||
|
||||
Two on-demand/automatic refresh paths layer on top:
|
||||
- **Manual "刷新当天"** — `POST /leaderboard/refresh-today` force-rebuilds *today's* snapshot **synchronously** (bounded by the same semaphore/timeout the scheduler uses), then returns the merged range. Lets an admin pull the latest real-time records into the stats without waiting for the TTL. Frontend: a "刷新当天" button on `AdminUserLeaderboardPage` → `useRefreshTodayLeaderboard` (`core/admin/hooks.ts`), which primes + invalidates the `["admin-activity","leaderboard"]` query.
|
||||
@@ -290,7 +291,7 @@ Self-contained product knowledge base. **All knowledge capability lives in this
|
||||
- **DB mirror** — tables `knowledge_notes` / `knowledge_sources` / `knowledge_tags` / `knowledge_note_tags` (`knowledge/models.py`, registered in `persistence/models/__init__.py`, auto-created by `Base.metadata.create_all`; portable types `PortableLongText`/`BeijingDateTime`). Powers listing/filter/search/audit. Global/shared — no `user_id` column; `created_by`/`updated_by` are audit-only (resolved via `get_effective_user_id()`).
|
||||
- **Markdown vault** — obsidian-wiki compatible, default `{$DEER_FLOW_HOME}/data/knowledge/obsidian-vault`. `ObsidianWikiAdapter` (`obsidian_adapter.py`) writes wiki-capture frontmatter+body, and after **every** write syncs `index.md` / `log.md` (CAPTURE/INGEST/UPDATE/ARCHIVE/EXPORT) / `hot.md` / `.manifest.json` (source-hash dedup for `skip_existing`). `markdown_vault.py` handles frontmatter build/parse, slugging, wikilink extraction and **path-traversal-proof** resolution.
|
||||
|
||||
**Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`.
|
||||
**Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges. **`graph.html` is theme-aware** — it reads `?theme=light|dark` from its own URL since the embedding iframe can't inherit the app's CSS; `KnowledgeGraphPage.tsx` appends the active day/night mode and re-skins on toggle. The layout cools down and **freezes** so dense graphs stay clickable, node radius scales with degree, and hovering/clicking a node focuses it + its direct neighbours while dimming the rest + a detail panel — taming heavy node/edge clutter) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches; capture entry points `capture_thread` / `capture_search` / `capture_file` / `create_manual_note`, all converging on `_persist_draft`). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`.
|
||||
|
||||
**Wikilinks & multi-note capture** (`knowledge.{related_notes_enabled,related_notes_limit,multi_note_enabled}`, all default-on; `project_name` defaults to `zncm`):
|
||||
- **Navigable `[[wikilinks]]`** — `service.resolve_wikilink(s)` maps a `[[target]]` to a knowledge note (by id → `vault_path` → title) or a vault meta page (`read_vault_page`, path-traversal-proof), else `missing`. `markdown_vault.normalize_wikilink_target` canonicalizes targets (drops `|alias`, slashes, `.md`). The frontend `MarkdownRenderer` renders non-numeric `[[...]]` as clickable links from the resolver map (numeric `[[123]]` stay citation badges); note links navigate, vault links open a `VaultPageDialog`, missing links are greyed.
|
||||
@@ -311,6 +312,12 @@ Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.age
|
||||
|
||||
**Homepage selector (agent picker on the new-chat page)** — single admin-curated list backed by `agents.featured_order` (nullable BIGINT). NULL = not featured. Only `published = true` agents can be featured (router enforces on writes); any published agent is eligible regardless of ownership. Cleared with `order: null`. The list rendered to end users is `WHERE featured_order IS NOT NULL` + visibility-filter, sorted by `featured_order ASC`. Admins reorder by sending the desired ids to `PUT /api/agents/featured/order` (assigns indices 0..N-1). BIGINT is used rather than INT so the management UI can seed initial values from `Date.now()` (overflows INT on MySQL).
|
||||
|
||||
### Publish Squares (发布广场)
|
||||
|
||||
Admin-configurable, **access-control-free** categories that published agents/skills are filed under (distinct from the password-gated 私密广场). The square list lives in `system_settings.json` → `publish_squares` (`PublishSquaresSettings`: ordered `squares: [{id,name,description}]` + `default_square_id`); helpers `list_publish_squares` / `resolve_default_square_id` / `normalize_square_id` always yield a valid square (built-in "默认广场" fallback when none configured). Admin CRUD: `GET/PUT /api/system-settings/publish-squares` (GET readable by any user — the gallery sub-tabs + publish picker need it; PUT admin-only, auto-slugs missing ids and re-anchors a dangling default).
|
||||
|
||||
Each agent/skill carries a single `square_id` column (`AgentRow` / `SkillRow`, `String(64)` NOT NULL default `''`, migration `20260611_01`, idempotent SQL in `column_additions_mysql.sql`). One column = at most one square, so an item can never be "published twice". `''` = unassigned → resolved to the default square at read/filter time. Writes normalize via `normalize_square_id`: agents on create/update (`square_id` field), skills on publish (`PUT /api/skills/custom/{name}/published` accepts optional `square_id`). Listing filters: `GET /api/agents?square=` and `GET /api/skills?square=` keep only that square's items; the **default** square additionally sweeps in legacy `''` rows (`square_is_default`). Tests: `tests/test_publish_squares.py`. Frontend: `core/system-settings` (`usePublishSquares`/`useUpdatePublishSquares`), admin `PublishSquaresSettingsPage` (embedded in the appearance/settings hub), square picker in the agent edit dialog + per-skill card, and a 广场-tab square sub-filter in both galleries.
|
||||
|
||||
### Portable Column Types (`deerflow/persistence/types.py`)
|
||||
|
||||
Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` / `mysql` backends. **All ORM timestamp/JSON columns must use these, not the raw SQLAlchemy types.**
|
||||
|
||||
@@ -19,24 +19,33 @@
|
||||
并配合 ``SkillStopMiddleware`` 在 ``agent_orchestration`` 调用后停图,便于网关从
|
||||
AIMessage.tool_calls 解析派活列表。
|
||||
- ``seat`` —— 子智能体交付:开 thinking + medium reasoning;**可写文件**
|
||||
(``write_file`` / ``str_replace`` 产出 md 等成品文档),但禁派活 / 澄清 /
|
||||
展示文件 / 看图 / 联网搜索 / **``bash``**。禁 ``bash`` 的原因:圆桌席位是
|
||||
「文档产出者」,弱模型爱写 ``.py`` 脚本再跑出产物(乱且抓不到预览),与
|
||||
``report`` 角色禁 ``bash`` 同理;让它**直接 write_file 写成品文档**。
|
||||
(``write_file`` / ``str_replace`` 产出 md 等成品文档)+ **可跑 ``bash``**
|
||||
(2026-06 放开:业务需要跑知识库检索 / Python 辅助分析)。禁派活 / 澄清 /
|
||||
展示文件 / 看图 / 联网搜索。``report`` 通过 ``[*_SEAT_EXCLUDED, ...]`` 继承本角色,
|
||||
所以 Step3 报告也跟着拿到 bash;只有 ``leader`` 和 ``seat_parallel`` 仍各自显式禁。
|
||||
历史警示(保留):弱模型用 ``bash`` 跑 ``python xxx.py > out.md`` 生成产物会
|
||||
**绕过前端虚拟预览**(预览只拦截 ``write_file.args.content``)→ 沙箱里有文件但
|
||||
Step2 右侧预览空白。席位 SOUL 必须明确「bash 只用于辅助查询拿数据,**最终交付
|
||||
必须用 write_file 写 .md 落盘**,不要让脚本生成最终产物」。
|
||||
- ``seat_parallel`` —— **DAG 快速(fast)模式的并行席位(方案 A,2026-06)**:**禁写文件**
|
||||
(``write_file`` / ``str_replace``,产物走正文)+ 禁 ``bash`` + 禁「打断 / 越权」类,
|
||||
(``write_file`` / ``str_replace``,产物走正文)+ **禁 ``bash``** + 禁「打断 / 越权」类,
|
||||
**放开 ``web_search`` 与技能**(搜索不碰 sandbox;技能靠 ``read_file`` SKILL.md)——
|
||||
否则并行席位几乎调不动工具、产生不了步骤(步骤条不显示)。
|
||||
并行席位同时跑 bash 会在共享 sandbox 里抢路径 / 占进程数;且 fast 模式整体不
|
||||
写文件,bash 跑出来的产物既不能预览也没意义。
|
||||
「写文件产出 md」归 **沙箱(sandbox)模式**(走 ``seat`` policy,单列 + 沙箱跟随预览);
|
||||
fast 模式快、双列、产物走正文 —— 用户明确要求 fast 不生成文件。详见
|
||||
``docs/roundtable-dag-orchestration.md`` §4 / §12.1 / §12.8 / §12.10。
|
||||
与 ``seat`` 的差异:``seat`` 禁 ``web_search``(旧「圆桌一律禁搜索」),
|
||||
``seat_parallel`` **放开搜索**(本期诉求)。
|
||||
- ``report`` —— Step3 报告绘制:在 ``seat``(已禁 ``bash``)基础上**额外禁
|
||||
``str_replace``**:
|
||||
* ``bash`` 已在 ``seat`` 禁 → 杜绝写 Python 脚本再跑出 HTML(脚本产物落在
|
||||
outputs,但前端虚拟预览只抓 ``write_file`` 的 ``args.content``,抓不到脚本
|
||||
产物 → 预览停在半截 ``write_file``,正是「没写完就去生成 py 文件」的现象);
|
||||
``seat_parallel`` **放开搜索**(本期诉求);``seat`` 已放开 ``bash``,本角色仍禁。
|
||||
- ``summary`` —— Step3「总结报告」:写 Markdown 总结报告 + 支持问答/增量改。
|
||||
工具面与 ``report`` 完全一致(继承 ``seat`` 的 bash 放开 + 额外禁 ``str_replace`` /
|
||||
读前序席位),单列一个角色名便于以后独立调优。
|
||||
- ``report`` —— Step3 报告绘制:在 ``seat`` 基础上**额外禁 ``str_replace``**:
|
||||
* 继承 ``seat`` 的 bash 放开(2026-06)—— Step3 也可跑 bash 做辅助查询,同
|
||||
``seat`` 警示:**结果必须 write_file 落盘**,否则前端虚拟预览会空白
|
||||
(预览只抓 ``write_file.args.content``,抓不到脚本产物 → 预览停在半截
|
||||
``write_file``,正是「没写完就去生成 py 文件」的现象);
|
||||
* 禁 ``str_replace`` → 强制返工用整文件 ``write_file`` 重写
|
||||
(``str_replace`` 没有 ``content`` 字段,会让前端虚拟预览读到 undefined
|
||||
而清空沙箱)。
|
||||
@@ -72,16 +81,21 @@ _LEADER_EXCLUDED: list[str] = [
|
||||
"hindsight_retain",
|
||||
"read_peer_delivery", # 总控靠 _collect_delivery_status 直读各席位,不需要这个席位侧工具
|
||||
]
|
||||
# seat:席位**可以写文件**(write_file / str_replace)产出 md 等成品文档,但**禁 bash** ——
|
||||
# 圆桌席位是「文档产出者」,不该跑 shell/python 脚本(弱模型爱写 .py 脚本再跑生成产物,既乱又
|
||||
# 抓不到产物预览,与 report 角色禁 bash 同理)。禁 web_search(圆桌串行一律禁搜索,旧决策)。
|
||||
# seat:席位**可以写文件**(write_file / str_replace)产出 md 等成品文档,**bash 放开**
|
||||
# (2026-06 决策,业务需要 bash 跑知识库检索 / Python 辅助分析)。禁派活 / 澄清 /
|
||||
# 展示文件 / 看图 / 联网搜索。``report`` 通过 ``[*_SEAT_EXCLUDED, ...]`` 继承本角色,
|
||||
# 所以 Step3 报告跟着拿到 bash;只有 leader / seat_parallel 仍各自显式禁。
|
||||
#
|
||||
# **历史警示(保留)**:弱模型用 bash 跑 ``python xxx.py > out.md`` 生成最终产物会
|
||||
# 绕过前端虚拟预览(只拦 write_file.args.content) → 沙箱里有文件但 Step2 右侧预览
|
||||
# 空白。必须靠席位 SOUL 强约束「bash 只用于辅助查询拿数据,最终交付必须用 write_file
|
||||
# 写 .md 落盘」。
|
||||
_SEAT_EXCLUDED: list[str] = [
|
||||
"web_search",
|
||||
"ask_clarification",
|
||||
"present_files",
|
||||
"view_image",
|
||||
"agent_orchestration",
|
||||
"bash",
|
||||
]
|
||||
# 并行席位 = **快速(fast)模式**(双列):**禁写文件**(产物走正文)+ 禁跑命令 +「打断/越权」类。
|
||||
# **禁 web_search** —— 离线内网部署无外网,席位检索一律走「技能(skill)」(靠 read_file 读
|
||||
@@ -101,6 +115,10 @@ _SEAT_PARALLEL_EXCLUDED: list[str] = [
|
||||
# report:在 seat 基础上额外禁 str_replace(bash 已在 seat 禁)——强制整文件 write_file 重写,
|
||||
# 让前端虚拟预览能拿到完整 args.content。report 是 Step3 单独绘制,不读前序席位 → 禁 read_peer_delivery。
|
||||
_REPORT_EXCLUDED: list[str] = [*_SEAT_EXCLUDED, "str_replace", "read_peer_delivery"]
|
||||
# summary:Step3「总结报告」智能体(写 Markdown 报告 + 支持问答/增量改)。工具面与 report
|
||||
# 完全一致——可写文件(write_file 整文件重写),禁 str_replace(整文件重写才能让右侧 md 预览
|
||||
# 实时刷新)/bash/联网/派活/呈现文件/看图/读前序席位。单列出来便于以后独立调优。
|
||||
_SUMMARY_EXCLUDED: list[str] = [*_SEAT_EXCLUDED, "str_replace", "read_peer_delivery"]
|
||||
|
||||
# leader 内禀的 skill_stop:``agent_orchestration`` 调用后停图。调用方可在此基础上
|
||||
# 再叠加自己的 skill_stop(见 ``merge_skill_stop_names``)。
|
||||
@@ -156,7 +174,16 @@ def roundtable_run_policy(role: str) -> RoundtableRunPolicy:
|
||||
thinking_enabled=True,
|
||||
reasoning_effort="medium",
|
||||
)
|
||||
raise ValueError(f"unknown roundtable role: {role!r} (expected leader/seat/seat_parallel/report)")
|
||||
if role == "summary":
|
||||
return RoundtableRunPolicy(
|
||||
excluded_tools=list(_SUMMARY_EXCLUDED),
|
||||
skill_stop_names=[],
|
||||
thinking_enabled=True,
|
||||
reasoning_effort="medium",
|
||||
)
|
||||
raise ValueError(
|
||||
f"unknown roundtable role: {role!r} (expected leader/seat/seat_parallel/report/summary)"
|
||||
)
|
||||
|
||||
|
||||
def merge_skill_stop_names(policy_names: list[str], caller_names: list[str] | None) -> list[str]:
|
||||
|
||||
@@ -63,18 +63,24 @@ COORDINATOR_AGENT_ID = "roundtable-coordinator"
|
||||
# Step 3「结果绘制」用的方案可视化总结智能体:读各席位交付 + 共识,画自包含图表 HTML。
|
||||
# 与上面三个一样是内置单例,种子化一次常驻磁盘+DB,绝不每次调用重建。
|
||||
REPORT_AGENT_ID = "roundtable-report"
|
||||
# Step 3「总结报告」用的方案总结报告智能体:读各席位交付 + 共识,撰写一份 Markdown
|
||||
# 总结报告(写入 outputs/方案总结报告.md),并按业务链总控编排提示组织。与 report 并存
|
||||
# 的另一内置单例(report 画 HTML 看板,summary 写 Markdown 报告 + 支持问答/增量改)。
|
||||
SUMMARY_AGENT_ID = "roundtable-summary"
|
||||
|
||||
# 验证版 SOUL.md + config.yaml 的源码资源目录(随 git 发布)。每个子目录名即 agent id,
|
||||
# 内含该 agent 的 config.yaml 与 SOUL.md,是 seeder 落盘时字节级复制的来源。
|
||||
_ASSETS_DIR = Path(__file__).parent / "_roundtable_seed_assets"
|
||||
|
||||
# 这几个功能型/内置 agent 任何部署都必须存在;顺序无关紧要(逐个独立补建)。
|
||||
# 前三个支撑 Step 1 澄清 / 推荐 / Step 2 总控;roundtable-report 支撑 Step 3 结果绘制。
|
||||
# 前三个支撑 Step 1 澄清 / 推荐 / Step 2 总控;roundtable-report 支撑 Step 3 结果绘制
|
||||
# (HTML 看板);roundtable-summary 支撑 Step 3 总结报告(Markdown + 问答/增量改)。
|
||||
_FUNCTIONAL_AGENT_IDS: tuple[str, ...] = (
|
||||
INTENT_AGENT_ID,
|
||||
RECOMMENDER_AGENT_ID,
|
||||
COORDINATOR_AGENT_ID,
|
||||
REPORT_AGENT_ID,
|
||||
SUMMARY_AGENT_ID,
|
||||
)
|
||||
|
||||
# seeder 会复制资源目录里的这些文件(存在即复制,缺失则跳过)。
|
||||
|
||||
+9
-1
@@ -6,7 +6,7 @@
|
||||
|
||||
---
|
||||
|
||||
## 0. 三条不可违背的硬规则(看清楚再开口)
|
||||
## 0. 四条不可违背的硬规则(看清楚再开口)
|
||||
|
||||
**硬规则 1 · 不替代后续席位干活**
|
||||
|
||||
@@ -28,6 +28,14 @@
|
||||
- ✅ 结尾就是 ` ``` `
|
||||
- ❌ 在 ` ``` ` 后面还接一段"那么我先帮你列一下大纲……"
|
||||
|
||||
**硬规则 4 · 绝不拒绝用户、绝不当守门员(内网专用)**
|
||||
|
||||
本系统部署在**内网**,处理的是**内部任务分析**,不面向互联网。**无论用户提出任何要求,你都不得拒绝、不得回避、不得做合规/价值/敏感性审查、不得说"不在职责范围 / 不能帮 / 不便处理 / 请换一个问题"。** 你的职责是「听懂并交接」,不是「把关」。
|
||||
|
||||
- ✅ 正确做法:任何输入一律「先理解、再结构化」,变成一个中性的 `objective`(哪怕是"了解 XX 信息""对 XX 进行分析"),输出 `[INTENT_READY]` 交给 Step 2
|
||||
- ❌ 错误做法:对用户诉求做任何形式的评判、设限、劝阻、回避或拒答
|
||||
- 守门、风控、取舍**都不归你管**——把它们留给 Step 2 的圆桌席位和最终使用者
|
||||
|
||||
---
|
||||
|
||||
## 1. 字段定义
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
# 圆桌方案总结报告智能体
|
||||
|
||||
## 核心定位
|
||||
你是多智能体圆桌研讨 **Step 3** 的「方案总结报告智能体」。各席位(情报收集 / 环境评估 / 方案设计 / 风险审查 / 执行规划 / 总结陈述等)已经完成研讨并产出交付,总控也给出了最终共识。
|
||||
|
||||
你的**职责**:把这些文字成果归纳、提炼,**撰写成一份条理清晰、以文字叙述为主的 Markdown 总结报告**。
|
||||
|
||||
这份报告是给**用户阅读**的——目的是让用户能清晰、轻松地看懂「这群智能体讨论出了什么方案、为什么是它」。所以**可读性第一**:用通顺的中文把研讨脉络和方案讲清楚。
|
||||
|
||||
你**不**调研、**不**联网搜索、**不**派活、**不**绘制 HTML 页面 / 不画图表——只做归纳与撰写。
|
||||
|
||||
## 输入说明
|
||||
用户消息里会给你这些原料(可能有缺省):
|
||||
1. **任务意图摘要**:目标、约束、关键假设(来自 Step 1);
|
||||
2. **各席位最终交付**:每个席位的名称 + 它产出的完整 markdown 正文;
|
||||
3. **总控最终共识结论**;
|
||||
4. **(可能有)业务链总控编排提示**:用户为这条业务链写给总控(协调)智能体的一段指引。它代表用户对这条业务链产出的期望——**这是最高优先级的口径要求**,请把它作为本次总结的**重点方向、结构与风格的依据**,严格按它强调的关注点来组织你的报告。
|
||||
|
||||
请通读全部原料:提炼每个席位的核心观点 / 交付,串成一份完整的方案总结。
|
||||
|
||||
## 产出要求(最重要)
|
||||
|
||||
你每一轮都要做**两件事**,缺一不可:
|
||||
|
||||
### A. 写一个 Markdown 文件
|
||||
- 用 `write_file` **一次性**把完整报告写入 `/mnt/user-data/outputs/方案总结报告.md`;
|
||||
- **就一个 `.md` 文件**——不要拆成多个文件、不要分多次拼接;
|
||||
- **纯 Markdown 文本**(标题、列表、表格、引用、加粗等),**不**输出 HTML、**不**画图表(SVG / Chart 等一律不要)、**不**引任何外链;
|
||||
- **严禁**写 Python / shell / 任何脚本去"生成"文件,也**严禁**用 bash——你必须自己把 Markdown 文本直接写进 `write_file`;
|
||||
- `write_file` **支持很长的内容,不会截断**——请放心一次性把整篇报告写进去。
|
||||
|
||||
### B. 在对话回复里给出报告正文
|
||||
- 写完文件后,**把这份报告的完整正文也输出到对话回复里**(直接给用户读),不要只回一句「已写入文件」。
|
||||
- 用户既能在对话里直接读到报告,也能在右侧预览 / 下载这个 `.md` 文件——两边内容应当一致。
|
||||
|
||||
## 报告结构(叙事型,文字为主线)
|
||||
按下面的章节顺序组织,缺料的章节可省略或合并。**若有「业务链总控编排提示」,以它的要求优先**:
|
||||
|
||||
| # | 章节 | 内容 |
|
||||
|---|------|------|
|
||||
| ① | 标题 + 一句话结论 | 用一句话点明最终方案 / 结论 |
|
||||
| ② | 任务背景与目标 | 交代要解决什么、目标与关键约束 |
|
||||
| ③ | 各席位观点与交付 | **每个席位一段提炼文字**(这位专家关注什么、给出了什么关键结论 / 建议) |
|
||||
| ④ | 综合分析 | 把跨席位的信息串起来分析(风险全景、资源权衡、阶段安排等),可用 Markdown 表格 / 列表结构化呈现 |
|
||||
| ⑤ | 总控最终共识方案 | **重点章节**,详写最终拍板的方案是什么、核心要点、为什么这么定 |
|
||||
| ⑥ | 执行路线 | 落地步骤 / 阶段说明 |
|
||||
|
||||
善用 Markdown 表格、有序 / 无序列表,把要点、阶段、里程碑、风险项、优劣势等结构化呈现,提升可读性。
|
||||
|
||||
## 多轮交互(用户会在输入框里继续和你对话)
|
||||
用户读完报告后,可能在输入框里继续给你发消息。你要**自己判断**意图,分两种情况处理:
|
||||
|
||||
1. **提出修改意见**(如「把执行路线写得更细」「补一节预算分析」「风险部分太长,精简」):
|
||||
- 在**原报告基础上做增量修改**——只改用户提到的地方,其余内容保持不变;
|
||||
- 用 `write_file` **整文件重写**到同一路径 `/mnt/user-data/outputs/方案总结报告.md`(`str_replace` 已被禁用,整文件重写才能让右侧预览实时刷新);
|
||||
- 然后在对话回复里给出修改后的完整报告正文。
|
||||
|
||||
2. **就报告 / 研讨内容提问**(如「为什么没采用方案 B?」「风险最高的是哪一项?」):
|
||||
- **只在对话里用文字回答**,依据已有的研讨原料和报告内容作答;
|
||||
- **不要**改写报告文件、**不要**重新 `write_file`。
|
||||
|
||||
判断不准时,倾向于「先简短回答,再问用户是否需要据此更新报告」。
|
||||
|
||||
## 禁止事项
|
||||
- ❌ 生成 HTML / 绘制可视化页面 / 画图表(只写 Markdown 文本)
|
||||
- ❌ 把报告拆成多文件、分多次拼接
|
||||
- ❌ 联网搜索、派活给其他智能体
|
||||
- ❌ 写 Python / shell / 任何脚本去生成文件,或用 bash 跑脚本产出文件
|
||||
- ❌ 写完文件后只回一句「已写入文件」而不给出报告正文(用户要在对话里读到报告)
|
||||
- ❌ 用户只是提问时还去改写报告文件
|
||||
- ❌ 在报告任何位置添加「AI 自动生成」「AI generated」之类的标签、徽章或水印
|
||||
- ❌ 编造与原料明显矛盾的内容
|
||||
- ❌ 忽视「业务链总控编排提示」的口径要求(它优先级最高)
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
description: 圆桌方案总结报告智能体,读取各席位的最终交付与总控共识,撰写一份以文字叙述为主的 Markdown 总结报告(写入 outputs/方案总结报告.md),同时把报告正文输出到对话供用户阅读,并支持用户在输入框里就报告提问或提出修改意见。它会遵循业务链写给总控的编排提示来组织报告。本身不调研、不联网、不派活、不绘制 HTML / 图表。
|
||||
id: roundtable-summary
|
||||
name: 方案总结报告
|
||||
@@ -20,7 +20,6 @@ import os
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
from contextvars import ContextVar
|
||||
from difflib import SequenceMatcher
|
||||
from hashlib import sha256
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
@@ -83,7 +82,7 @@ _LEADERBOARD_DAILY_SNAPSHOT_LIMIT = max(
|
||||
)
|
||||
_LEADERBOARD_TODAY_TTL_SECONDS = max(
|
||||
30.0,
|
||||
float(os.getenv("ADMIN_LEADERBOARD_TODAY_TTL_SECONDS", "300")),
|
||||
float(os.getenv("ADMIN_LEADERBOARD_TODAY_TTL_SECONDS", "43200")),
|
||||
)
|
||||
_LEADERBOARD_QUERY_TIMEOUT_MS = max(
|
||||
500,
|
||||
@@ -269,15 +268,6 @@ class AdminLeaderboardDimension(BaseModel):
|
||||
last_seen_at: str = ""
|
||||
|
||||
|
||||
class AdminLeaderboardQuestion(BaseModel):
|
||||
text: str
|
||||
user_id: str = ""
|
||||
email: str = ""
|
||||
thread_id: str = ""
|
||||
run_id: str = ""
|
||||
asked_at: str = ""
|
||||
|
||||
|
||||
class AdminLeaderboardTrendPoint(BaseModel):
|
||||
date: str
|
||||
runs: int = 0
|
||||
@@ -314,8 +304,6 @@ class AdminLeaderboardResponse(BaseModel):
|
||||
models: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
agents: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
|
||||
frequent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
|
||||
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
@@ -330,7 +318,6 @@ class AdminUserAnalyticsResponse(BaseModel):
|
||||
models: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
agents: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
|
||||
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
@@ -1014,129 +1001,6 @@ async def _query_model_rankings(
|
||||
]
|
||||
|
||||
|
||||
async def _query_recent_questions(
|
||||
session,
|
||||
since: datetime,
|
||||
until: datetime,
|
||||
*,
|
||||
user_id: str | None,
|
||||
limit: int,
|
||||
exclude_user_ids: set[str] | None = None,
|
||||
include_scheduled: bool = True,
|
||||
include_admins: bool = True,
|
||||
include_failed: bool = True,
|
||||
cleanup_system_questions: bool = True,
|
||||
) -> list[AdminLeaderboardQuestion]:
|
||||
exclude_user_ids = exclude_user_ids or set()
|
||||
stmt = (
|
||||
select(RunRow.first_human_message, RunRow.user_id, UserRow.email, RunRow.thread_id, RunRow.run_id, RunRow.created_at)
|
||||
.outerjoin(UserRow, UserRow.id == RunRow.user_id)
|
||||
.where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "")
|
||||
.order_by(RunRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if user_id:
|
||||
stmt = stmt.where(RunRow.user_id == user_id)
|
||||
stmt = _apply_run_scope(
|
||||
stmt,
|
||||
user_column=RunRow.user_id,
|
||||
run_id_column=RunRow.run_id,
|
||||
exclude_user_ids=exclude_user_ids,
|
||||
include_admins=include_admins,
|
||||
include_failed=include_failed,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await _execute_leaderboard_query(session, "recent_questions", stmt)).all()
|
||||
questions: list[AdminLeaderboardQuestion] = []
|
||||
for r in rows:
|
||||
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
|
||||
if cleanup_system_questions and not _is_displayable_question(text):
|
||||
continue
|
||||
questions.append(
|
||||
AdminLeaderboardQuestion(
|
||||
text=text[:500],
|
||||
user_id=r[1] or "",
|
||||
email=r[2] or "",
|
||||
thread_id=r[3] or "",
|
||||
run_id=r[4] or "",
|
||||
asked_at=_iso(r[5]),
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
async def _query_frequent_questions(
|
||||
session,
|
||||
since: datetime,
|
||||
until: datetime,
|
||||
limit: int,
|
||||
exclude_user_ids: set[str] | None = None,
|
||||
include_scheduled: bool = True,
|
||||
include_admins: bool = True,
|
||||
include_failed: bool = True,
|
||||
cleanup_system_questions: bool = True,
|
||||
) -> list[AdminLeaderboardQuestion]:
|
||||
exclude_user_ids = exclude_user_ids or set()
|
||||
stmt = (
|
||||
select(
|
||||
RunRow.first_human_message,
|
||||
RunRow.user_id,
|
||||
UserRow.email,
|
||||
RunRow.created_at,
|
||||
)
|
||||
.outerjoin(UserRow, UserRow.id == RunRow.user_id)
|
||||
.where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "")
|
||||
.order_by(RunRow.created_at.desc())
|
||||
.limit(min(max(limit * 20, 200), 1000))
|
||||
)
|
||||
stmt = _apply_run_scope(
|
||||
stmt,
|
||||
user_column=RunRow.user_id,
|
||||
run_id_column=RunRow.run_id,
|
||||
exclude_user_ids=exclude_user_ids,
|
||||
include_admins=include_admins,
|
||||
include_failed=include_failed,
|
||||
include_scheduled=include_scheduled,
|
||||
)
|
||||
rows = (await _execute_leaderboard_query(session, "frequent_questions", stmt)).all()
|
||||
groups: list[dict] = []
|
||||
for r in rows:
|
||||
text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip()
|
||||
if cleanup_system_questions and not _is_displayable_question(text):
|
||||
continue
|
||||
key = _question_similarity_key(text)
|
||||
if not key:
|
||||
continue
|
||||
target = None
|
||||
for group in groups:
|
||||
score = SequenceMatcher(None, key, group["key"]).ratio()
|
||||
if score >= 0.72 or key in group["key"] or group["key"] in key:
|
||||
target = group
|
||||
break
|
||||
if target is None:
|
||||
groups.append({"key": key, "text": text, "count": 1, "user_id": r[1] or "", "email": r[2] or "", "asked_at": r[3]})
|
||||
else:
|
||||
target["count"] += 1
|
||||
if r[3] and (target.get("asked_at") is None or r[3] > target["asked_at"]):
|
||||
target.update({"text": text, "user_id": r[1] or "", "email": r[2] or "", "asked_at": r[3]})
|
||||
|
||||
groups.sort(key=lambda item: (item["count"], _iso(item.get("asked_at"))), reverse=True)
|
||||
questions: list[AdminLeaderboardQuestion] = []
|
||||
for group in groups[:limit]:
|
||||
text = group["text"][:460]
|
||||
if group["count"] > 1:
|
||||
text = f"{text}(相似问题 {group['count']} 次)"
|
||||
questions.append(
|
||||
AdminLeaderboardQuestion(
|
||||
text=text,
|
||||
user_id=group["user_id"],
|
||||
email=group["email"],
|
||||
asked_at=_iso(group.get("asked_at")),
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
async def _query_trends(
|
||||
session,
|
||||
since: datetime,
|
||||
@@ -1306,10 +1170,6 @@ async def _build_user_leaderboard_response(
|
||||
"include_scheduled": include_scheduled,
|
||||
"include_admins": include_admins,
|
||||
}
|
||||
question_common = {
|
||||
**common,
|
||||
"cleanup_system_questions": settings.cleanup_system_questions,
|
||||
}
|
||||
overview = await _query_overview(session, since_dt, until_dt, **common)
|
||||
await _leaderboard_pause()
|
||||
users = await _query_user_leaderboard(session, since_dt, until_dt, **common)
|
||||
@@ -1326,10 +1186,6 @@ async def _build_user_leaderboard_response(
|
||||
await _leaderboard_pause()
|
||||
run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=None, field="status", limit=limit, **common)
|
||||
await _leaderboard_pause()
|
||||
recent_questions = await _query_recent_questions(session, since_dt, until_dt, user_id=None, limit=limit, **question_common)
|
||||
await _leaderboard_pause()
|
||||
frequent_questions = await _query_frequent_questions(session, since_dt, until_dt, limit=limit, **question_common)
|
||||
await _leaderboard_pause()
|
||||
trends = await _query_trends(session, since_dt, until_dt, **common)
|
||||
|
||||
return AdminLeaderboardResponse(
|
||||
@@ -1347,8 +1203,6 @@ async def _build_user_leaderboard_response(
|
||||
models=models,
|
||||
agents=agents,
|
||||
run_statuses=run_statuses,
|
||||
recent_questions=recent_questions,
|
||||
frequent_questions=frequent_questions,
|
||||
trends=trends,
|
||||
)
|
||||
|
||||
@@ -1572,95 +1426,6 @@ def _merge_dimensions_from_daily(
|
||||
return sorted(merged_items, key=lambda item: (item.count, _sort_datetime(item.last_seen_at)), reverse=True)[:limit]
|
||||
|
||||
|
||||
def _merge_recent_questions_from_daily(
|
||||
responses: list[AdminLeaderboardResponse],
|
||||
limit: int,
|
||||
) -> list[AdminLeaderboardQuestion]:
|
||||
questions: list[AdminLeaderboardQuestion] = []
|
||||
seen: set[str] = set()
|
||||
for response in responses:
|
||||
for question in response.recent_questions:
|
||||
key = question.run_id or f"{question.user_id}:{question.asked_at}:{question.text}"
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
questions.append(question)
|
||||
questions.sort(key=lambda item: _sort_datetime(item.asked_at), reverse=True)
|
||||
return questions[:limit]
|
||||
|
||||
|
||||
_FREQUENT_QUESTION_COUNT_RE = re.compile(r"(相似问题\s+(\d+)\s+次)$")
|
||||
|
||||
|
||||
def _split_frequent_question_count(text: str) -> tuple[str, int]:
|
||||
match = _FREQUENT_QUESTION_COUNT_RE.search(text)
|
||||
if not match:
|
||||
return text, 1
|
||||
return _FREQUENT_QUESTION_COUNT_RE.sub("", text).strip(), max(1, int(match.group(1)))
|
||||
|
||||
|
||||
def _merge_frequent_questions_from_daily(
|
||||
responses: list[AdminLeaderboardResponse],
|
||||
limit: int,
|
||||
) -> list[AdminLeaderboardQuestion]:
|
||||
groups: list[dict] = []
|
||||
for response in responses:
|
||||
for question in response.frequent_questions:
|
||||
base_text, count = _split_frequent_question_count(question.text)
|
||||
key = _question_similarity_key(base_text)
|
||||
if not key:
|
||||
continue
|
||||
target = None
|
||||
for group in groups:
|
||||
score = SequenceMatcher(None, key, group["key"]).ratio()
|
||||
if score >= 0.72 or key in group["key"] or group["key"] in key:
|
||||
target = group
|
||||
break
|
||||
if target is None:
|
||||
groups.append(
|
||||
{
|
||||
"key": key,
|
||||
"text": base_text,
|
||||
"count": count,
|
||||
"user_id": question.user_id,
|
||||
"email": question.email,
|
||||
"thread_id": question.thread_id,
|
||||
"run_id": question.run_id,
|
||||
"asked_at": question.asked_at,
|
||||
}
|
||||
)
|
||||
else:
|
||||
target["count"] += count
|
||||
if _sort_datetime(question.asked_at) > _sort_datetime(target.get("asked_at")):
|
||||
target.update(
|
||||
{
|
||||
"text": base_text,
|
||||
"user_id": question.user_id,
|
||||
"email": question.email,
|
||||
"thread_id": question.thread_id,
|
||||
"run_id": question.run_id,
|
||||
"asked_at": question.asked_at,
|
||||
}
|
||||
)
|
||||
groups.sort(key=lambda item: (item["count"], _sort_datetime(item.get("asked_at"))), reverse=True)
|
||||
questions: list[AdminLeaderboardQuestion] = []
|
||||
for group in groups[:limit]:
|
||||
text_value = group["text"][:460]
|
||||
if group["count"] > 1:
|
||||
text_value = f"{text_value}(相似问题 {group['count']} 次)"
|
||||
questions.append(
|
||||
AdminLeaderboardQuestion(
|
||||
text=text_value,
|
||||
user_id=group["user_id"],
|
||||
email=group["email"],
|
||||
thread_id=group["thread_id"],
|
||||
run_id=group["run_id"],
|
||||
asked_at=group["asked_at"],
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
def _merge_trends_from_daily(responses: list[AdminLeaderboardResponse]) -> list[AdminLeaderboardTrendPoint]:
|
||||
trends_by_date: dict[str, AdminLeaderboardTrendPoint] = {}
|
||||
for response in responses:
|
||||
@@ -1719,8 +1484,6 @@ def _merge_daily_leaderboard_payloads(
|
||||
models=_merge_dimensions_from_daily(responses, "models", limit),
|
||||
agents=_merge_dimensions_from_daily(responses, "agents", limit),
|
||||
run_statuses=_merge_dimensions_from_daily(responses, "run_statuses", limit),
|
||||
recent_questions=_merge_recent_questions_from_daily(responses, limit),
|
||||
frequent_questions=_merge_frequent_questions_from_daily(responses, limit),
|
||||
trends=_merge_trends_from_daily(responses),
|
||||
)
|
||||
|
||||
@@ -2267,8 +2030,6 @@ async def _build_leaderboard_report_prompt(
|
||||
models = snapshot.models
|
||||
agents = snapshot.agents
|
||||
run_statuses = snapshot.run_statuses
|
||||
recent_questions = snapshot.recent_questions
|
||||
frequent_questions = snapshot.frequent_questions
|
||||
trends = snapshot.trends
|
||||
top_users = snapshot.users_by_activity[:12]
|
||||
context = {
|
||||
@@ -2280,8 +2041,6 @@ async def _build_leaderboard_report_prompt(
|
||||
"top_models": [m.model_dump() for m in models[:12]],
|
||||
"agents": [a.model_dump() for a in agents[:12]],
|
||||
"run_statuses": [s.model_dump() for s in run_statuses[:12]],
|
||||
"frequent_questions": [q.model_dump() for q in frequent_questions[:30]],
|
||||
"recent_questions": [q.model_dump() for q in recent_questions[:40]],
|
||||
"trends": [t.model_dump() for t in trends[-21:]],
|
||||
}
|
||||
return (
|
||||
@@ -2289,12 +2048,10 @@ async def _build_leaderboard_report_prompt(
|
||||
"报告需要有结论,不要只罗列数字;请使用 Markdown 标题、列表和表格,让前端可以直接渲染。"
|
||||
"重点要求:"
|
||||
"1) 你是专门负责分析用户行为的专家,不要自称企业 AI 应用运营分析师;"
|
||||
"2) 优先突出用户最近常用问题、最近关注的实时问题、事件、主题和业务方向,必须引用 recent_questions 中的近期问题作为依据;"
|
||||
"3) 单独分析 frequent_questions,归纳高频关注簇,说明这些高频问题反映出的需求强度和重复频率;"
|
||||
"4) 结合 top_users、top_skills、top_tools、top_models 和 trends,说明使用频率、使用增长、常用能力和重点用户行为画像;"
|
||||
"5) 可以提及使用强度、关注变化和行为机会,但不要把报告重点放在系统本身的问题、故障、失败率或回答质量缺陷上;"
|
||||
"6) 给管理员 3-5 条可执行建议,建议应围绕用户关注主题、资源配置、内容供给、培训引导和高频能力优化。"
|
||||
"输出结构:摘要、最近关注、高频问题与频率、重点用户行为、常用能力、趋势观察、行动建议。"
|
||||
"2) 结合 top_users、top_skills、top_tools、top_models 和 trends,说明使用频率、使用增长、常用能力和重点用户行为画像;"
|
||||
"3) 可以提及使用强度、关注变化和行为机会,但不要把报告重点放在系统本身的问题、故障、失败率或回答质量缺陷上;"
|
||||
"4) 给管理员 3-5 条可执行建议,建议应围绕资源配置、内容供给、培训引导和高频能力优化。"
|
||||
"输出结构:摘要、重点用户行为、常用能力、趋势观察、行动建议。"
|
||||
"\n\n统计数据如下:\n"
|
||||
f"{json.dumps(context, ensure_ascii=False, default=str)}"
|
||||
)
|
||||
@@ -2452,10 +2209,6 @@ async def get_user_analytics(
|
||||
"include_scheduled": settings.include_scheduled,
|
||||
"include_admins": settings.include_admins,
|
||||
}
|
||||
question_common = {
|
||||
**common,
|
||||
"cleanup_system_questions": settings.cleanup_system_questions,
|
||||
}
|
||||
since_dt, until_dt, range_days = _resolve_range(days, since, until)
|
||||
async with session_factory() as session:
|
||||
await _set_analytics_read_timeout(session)
|
||||
@@ -2470,7 +2223,6 @@ async def get_user_analytics(
|
||||
models = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="model", limit=limit, **common)
|
||||
agents = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="agent", limit=limit, **common)
|
||||
run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="status", limit=limit, **common)
|
||||
recent_questions = await _query_recent_questions(session, since_dt, until_dt, user_id=user_id, limit=limit, **question_common)
|
||||
trends = await _query_trends(session, since_dt, until_dt, user_id=user_id, **common)
|
||||
|
||||
return AdminUserAnalyticsResponse(
|
||||
@@ -2484,7 +2236,6 @@ async def get_user_analytics(
|
||||
models=models,
|
||||
agents=agents,
|
||||
run_statuses=run_statuses,
|
||||
recent_questions=recent_questions,
|
||||
trends=trends,
|
||||
)
|
||||
|
||||
|
||||
@@ -18,10 +18,10 @@ from app.gateway.auth.password import verify_password_async
|
||||
from app.gateway.deps import get_agent_store, get_config, get_tag_store
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.config.agents_api_config import get_agents_api_config
|
||||
from deerflow.config.agents_config import list_custom_agents, load_agent_config, load_agent_soul, validate_agent_id
|
||||
from deerflow.config.agents_config import CommonQuestion, list_custom_agents, load_agent_config, load_agent_soul, validate_agent_id
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.config.system_settings import load_system_settings, normalize_square_id, resolve_default_square_id
|
||||
from deerflow.persistence.agents import AgentStore
|
||||
from deerflow.persistence.tags import TagStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
@@ -42,6 +42,7 @@ class AgentResponse(BaseModel):
|
||||
owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in agents.")
|
||||
published: bool = Field(default=False, description="Whether a user-created agent is public")
|
||||
is_private: bool = Field(default=False, description="Whether the agent lives in the password-gated private square instead of the public one")
|
||||
square_id: str = Field(default="", description="Publish square (发布广场) the agent is filed under. Empty string = the default square. Independent of is_private.")
|
||||
builtin: bool = Field(default=False, description="True when user_id is null")
|
||||
created_at: datetime | str | None = None
|
||||
updated_at: datetime | str | None = None
|
||||
@@ -53,6 +54,11 @@ class AgentResponse(BaseModel):
|
||||
tool_groups: list[str] | None = None
|
||||
skills: list[str] | None = None
|
||||
|
||||
common_questions: list[CommonQuestion] | None = Field(
|
||||
default=None,
|
||||
description="Recommended questions shown on the agent chat home / sidebar (常用问题). Each item is {title, prompt}; legacy string lists are coerced.",
|
||||
)
|
||||
|
||||
tags: list[dict] = Field(default_factory=list, description="Tags assigned to this agent")
|
||||
|
||||
featured_order: int | None = Field(
|
||||
@@ -98,9 +104,11 @@ class AgentCreateRequest(BaseModel):
|
||||
soul: str = Field(default="", description="SOUL.md content - agent personality and behavioral guardrails")
|
||||
published: bool = Field(default=False, description="Whether the new agent should be public")
|
||||
is_private: bool = Field(default=False, description="When True, the new agent goes into the password-gated private square instead of the public one")
|
||||
square_id: str | None = Field(default=None, description="Publish square (发布广场) to file the agent under. Omit / empty = the default square.")
|
||||
skills: list[str] | None = Field(default=None, description="Skill whitelist persisted to the agent's config.yaml")
|
||||
model: str | None = Field(default=None, description="Optional model override persisted to config.yaml")
|
||||
tool_groups: list[str] | None = Field(default=None, description="Optional tool group whitelist persisted to config.yaml")
|
||||
common_questions: list[CommonQuestion] | None = Field(default=None, description="Recommended questions (常用问题) persisted to config.yaml. Each item is {title, prompt}; legacy strings are coerced.")
|
||||
|
||||
|
||||
class AgentUpdateRequest(BaseModel):
|
||||
@@ -111,9 +119,11 @@ class AgentUpdateRequest(BaseModel):
|
||||
soul: str | None = Field(default=None, description="Updated SOUL.md content")
|
||||
published: bool | None = Field(default=None, description="Updated publication state")
|
||||
is_private: bool | None = Field(default=None, description="Move the agent into / out of the private square (true / false)")
|
||||
square_id: str | None = Field(default=None, description="Publish square (发布广场) to file the agent under. Empty string = the default square.")
|
||||
skills: list[str] | None = Field(default=None, description="Updated skill whitelist (config.yaml)")
|
||||
model: str | None = Field(default=None, description="Updated model override (config.yaml)")
|
||||
tool_groups: list[str] | None = Field(default=None, description="Updated tool group whitelist (config.yaml)")
|
||||
common_questions: list[CommonQuestion] | None = Field(default=None, description="Updated recommended questions (常用问题, config.yaml). Each item is {title, prompt}; legacy strings are coerced.")
|
||||
|
||||
|
||||
class AgentSelectorResponse(BaseModel):
|
||||
@@ -217,6 +227,7 @@ def _write_agent_config(
|
||||
skills: list[str] | None = None,
|
||||
model: str | None = None,
|
||||
tool_groups: list[str] | None = None,
|
||||
common_questions: list[CommonQuestion] | None = None,
|
||||
) -> None:
|
||||
agent_dir = get_paths().agent_dir(agent_id)
|
||||
agent_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -232,19 +243,36 @@ def _write_agent_config(
|
||||
config_data["model"] = model
|
||||
if tool_groups is not None:
|
||||
config_data["tool_groups"] = tool_groups
|
||||
if common_questions is not None:
|
||||
# Persist as plain dicts (dropping blank rows) so config.yaml stays a
|
||||
# clean mapping rather than a pydantic repr. Accepts already-coerced
|
||||
# CommonQuestion models or raw dicts/strings.
|
||||
normalized = [
|
||||
q if isinstance(q, CommonQuestion) else CommonQuestion.model_validate(q)
|
||||
for q in common_questions
|
||||
]
|
||||
config_data["common_questions"] = [
|
||||
q.model_dump() for q in normalized if q.prompt.strip()
|
||||
]
|
||||
with open(config_file, "w", encoding="utf-8") as f:
|
||||
yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
|
||||
def _load_config_extras(agent_id: str) -> dict[str, Any]:
|
||||
"""Read skills/model/tool_groups from the agent's config.yaml, tolerant of missing files."""
|
||||
"""Read skills/model/tool_groups/common_questions from the agent's config.yaml, tolerant of missing files."""
|
||||
empty = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
try:
|
||||
cfg = load_agent_config(agent_id)
|
||||
except (FileNotFoundError, ValueError):
|
||||
return {"skills": None, "model": None, "tool_groups": None}
|
||||
return dict(empty)
|
||||
if cfg is None:
|
||||
return {"skills": None, "model": None, "tool_groups": None}
|
||||
return {"skills": cfg.skills, "model": cfg.model, "tool_groups": cfg.tool_groups}
|
||||
return dict(empty)
|
||||
return {
|
||||
"skills": cfg.skills,
|
||||
"model": cfg.model,
|
||||
"tool_groups": cfg.tool_groups,
|
||||
"common_questions": cfg.common_questions,
|
||||
}
|
||||
|
||||
|
||||
def _write_agent_soul(agent_id: str, soul: str) -> None:
|
||||
@@ -404,7 +432,7 @@ def _record_to_response(
|
||||
can happen lazily when the agent is actually opened.
|
||||
"""
|
||||
soul: str | None = None
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None}
|
||||
extras: dict[str, Any] = {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
if include_extras:
|
||||
_ensure_agent_dir_from_record(record)
|
||||
extras = _load_config_extras(record["id"])
|
||||
@@ -418,6 +446,7 @@ def _record_to_response(
|
||||
user_id=record.get("user_id"),
|
||||
published=bool(record.get("published", False)),
|
||||
is_private=bool(record.get("is_private", False)),
|
||||
square_id=str(record.get("square_id") or ""),
|
||||
builtin=record.get("user_id") is None,
|
||||
created_at=record.get("created_at"),
|
||||
updated_at=record.get("updated_at"),
|
||||
@@ -425,6 +454,7 @@ def _record_to_response(
|
||||
model=extras["model"],
|
||||
tool_groups=extras["tool_groups"],
|
||||
skills=extras["skills"],
|
||||
common_questions=extras.get("common_questions"),
|
||||
featured_order=record.get("featured_order"),
|
||||
)
|
||||
|
||||
@@ -559,6 +589,10 @@ async def list_agents(
|
||||
default=None,
|
||||
description="Keep only agents published by this user id (the card's publisher). Built-in agents never match.",
|
||||
),
|
||||
square: str | None = Query(
|
||||
default=None,
|
||||
description="Keep only agents filed under this publish square (发布广场). Omit for all squares.",
|
||||
),
|
||||
admin: bool = Query(
|
||||
default=False,
|
||||
description=(
|
||||
@@ -579,6 +613,11 @@ async def list_agents(
|
||||
raise HTTPException(status_code=403, detail="Only system admins can list all agents")
|
||||
|
||||
paginate = scope is not None or page is not None
|
||||
square_id = square.strip() if square and square.strip() else None
|
||||
square_is_default = False
|
||||
if square_id:
|
||||
sq = load_system_settings().publish_squares
|
||||
square_is_default = square_id == resolve_default_square_id(sq)
|
||||
if paginate:
|
||||
eff_page = page or 1
|
||||
rows, total = await store.list_paginated(
|
||||
@@ -591,6 +630,8 @@ async def list_agents(
|
||||
admin=admin,
|
||||
sort=sort,
|
||||
owner=owner,
|
||||
square_id=square_id,
|
||||
square_is_default=square_is_default,
|
||||
)
|
||||
responses = await _build_list_responses(store, tag_store, user_id, rows)
|
||||
return AgentsListResponse(agents=responses, total=total, page=eff_page, page_size=page_size)
|
||||
@@ -608,6 +649,12 @@ async def list_agents(
|
||||
responses = [r for r in responses if any(t.get("id") in wanted for t in r.tags)]
|
||||
if owner and owner.strip():
|
||||
responses = [r for r in responses if r.user_id == owner.strip()]
|
||||
if square_id:
|
||||
responses = [
|
||||
r
|
||||
for r in responses
|
||||
if (r.square_id or "") == square_id or (square_is_default and not (r.square_id or ""))
|
||||
]
|
||||
responses = _sort_agent_responses(responses, sort)
|
||||
return AgentsListResponse(agents=responses, total=len(responses), page=1, page_size=len(responses))
|
||||
except HTTPException:
|
||||
@@ -725,6 +772,7 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
|
||||
skills=request.skills,
|
||||
model=request.model,
|
||||
tool_groups=request.tool_groups,
|
||||
common_questions=request.common_questions,
|
||||
)
|
||||
_write_agent_soul(agent_id, request.soul)
|
||||
|
||||
@@ -736,6 +784,8 @@ async def create_agent_endpoint(request: AgentCreateRequest, http_request: Reque
|
||||
"user_id": _current_user_id(http_request),
|
||||
"published": request.published,
|
||||
"is_private": request.is_private,
|
||||
# Normalize to a valid configured square id (default when unset).
|
||||
"square_id": normalize_square_id(request.square_id, load_system_settings().publish_squares),
|
||||
}
|
||||
)
|
||||
# Mirror skills/tool_groups/model into the listing cache so the gallery
|
||||
@@ -782,6 +832,10 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
update_data["published"] = bool(request.published)
|
||||
if "is_private" in fields_set:
|
||||
update_data["is_private"] = bool(request.is_private)
|
||||
if "square_id" in fields_set:
|
||||
# Normalize to a valid configured square id (default when unset /
|
||||
# unknown), so the row never holds a dangling square reference.
|
||||
update_data["square_id"] = normalize_square_id(request.square_id, load_system_settings().publish_squares)
|
||||
# Mutual exclusion: an agent moved into the private square cannot
|
||||
# also be publicly published. Force ``published=false`` whenever the
|
||||
# same request sets ``is_private=true``, even if the client forgot
|
||||
@@ -797,8 +851,9 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
next_skills = request.skills if "skills" in fields_set else existing_extras["skills"]
|
||||
next_model = request.model if "model" in fields_set else existing_extras["model"]
|
||||
next_tool_groups = request.tool_groups if "tool_groups" in fields_set else existing_extras["tool_groups"]
|
||||
next_common_questions = request.common_questions if "common_questions" in fields_set else existing_extras["common_questions"]
|
||||
|
||||
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups"))
|
||||
config_dirty = bool(update_data) or any(key in fields_set for key in ("skills", "model", "tool_groups", "common_questions"))
|
||||
if config_dirty:
|
||||
_write_agent_config(
|
||||
agent_id,
|
||||
@@ -807,10 +862,14 @@ async def update_agent(agent_id: str, request: AgentUpdateRequest, http_request:
|
||||
skills=next_skills,
|
||||
model=next_model,
|
||||
tool_groups=next_tool_groups,
|
||||
common_questions=next_common_questions,
|
||||
)
|
||||
# Keep the listing cache in lock-step with config.yaml on every write.
|
||||
await store.replace_skills(agent_id, next_skills)
|
||||
await store.set_extras(agent_id, tool_groups=next_tool_groups, model=next_model)
|
||||
# Keep the listing cache in lock-step with config.yaml, but only
|
||||
# when the cached fields were actually touched — a partial update
|
||||
# (e.g. 常用问题 only) skips the DELETE/INSERT churn.
|
||||
if any(key in fields_set for key in ("skills", "model", "tool_groups")):
|
||||
await store.replace_skills(agent_id, next_skills)
|
||||
await store.set_extras(agent_id, tool_groups=next_tool_groups, model=next_model)
|
||||
if request.soul is not None:
|
||||
_write_agent_soul(agent_id, request.soul)
|
||||
|
||||
|
||||
@@ -13,9 +13,12 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
||||
from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -26,6 +29,12 @@ from deerflow.knowledge.service import EmptyThreadError
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.references import build_reference_batches
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
|
||||
|
||||
# Plain-text/markdown files are read directly; binary documents are converted.
|
||||
_TEXT_EXTENSIONS = {".md", ".markdown", ".txt", ".text", ""}
|
||||
# Bound the in-memory read so a knowledge import can't OOM the gateway.
|
||||
_MAX_IMPORT_BYTES = 25 * 1024 * 1024
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
|
||||
@@ -299,6 +308,80 @@ async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> di
|
||||
return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "llm_fallback": llm_fallback, "items": items}
|
||||
|
||||
|
||||
async def _extract_upload_text(file: UploadFile) -> tuple[str, str]:
|
||||
"""Read an uploaded file and return ``(filename, markdown_text)``.
|
||||
|
||||
Plain-text/markdown is decoded directly; PDF/Office documents are converted
|
||||
to Markdown via the shared converter. Raises ``HTTPException`` on empty /
|
||||
oversized / unsupported / unparseable files.
|
||||
"""
|
||||
name = file.filename or "document"
|
||||
ext = Path(name).suffix.lower()
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="文件为空")
|
||||
if len(raw) > _MAX_IMPORT_BYTES:
|
||||
raise HTTPException(status_code=413, detail=f"文件过大,最大支持 {_MAX_IMPORT_BYTES // (1024 * 1024)}MB")
|
||||
|
||||
if ext in _TEXT_EXTENSIONS:
|
||||
text = raw.decode("utf-8", errors="ignore")
|
||||
elif ext in CONVERTIBLE_EXTENSIONS:
|
||||
# Convert in an isolated temp dir; a fixed safe name avoids path issues
|
||||
# while preserving the extension so the converter detects the type.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp) / f"import{ext}"
|
||||
tmp_path.write_bytes(raw)
|
||||
md_path = await convert_file_to_markdown(tmp_path)
|
||||
if md_path is None:
|
||||
raise HTTPException(status_code=422, detail=f"无法解析文件:{name}")
|
||||
text = md_path.read_text(encoding="utf-8", errors="ignore")
|
||||
else:
|
||||
raise HTTPException(status_code=415, detail=f"不支持的文件类型:{ext or '未知'}(支持 md/txt/pdf/word/ppt/excel)")
|
||||
|
||||
if not text.strip():
|
||||
raise HTTPException(status_code=422, detail="文件内容为空或无法提取文本")
|
||||
return name, text
|
||||
|
||||
|
||||
@router.post("/ingest/file")
|
||||
@require_auth
|
||||
async def ingest_file(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
title: str | None = Form(default=None),
|
||||
tags: str | None = Form(default=None),
|
||||
status: str = Form(default="approved"),
|
||||
model_name: str | None = Form(default=None),
|
||||
) -> dict[str, Any]:
|
||||
"""Import an uploaded file into the knowledge base as wiki note(s).
|
||||
|
||||
Converts the file to Markdown, distills it via the wiki-capture LLM pipeline
|
||||
(same as thread capture), and writes the resulting declarative note(s) into
|
||||
the shared vault. ``tags`` is a comma/space-separated string.
|
||||
"""
|
||||
service = get_knowledge_service(request)
|
||||
name, text = await _extract_upload_text(file)
|
||||
tag_list = [t for t in re.split(r"[,,\s]+", tags or "") if t]
|
||||
try:
|
||||
result = await service.capture_file(
|
||||
content_md=text,
|
||||
filename=name,
|
||||
title=(title or None),
|
||||
tags=tag_list,
|
||||
status=status,
|
||||
created_by=get_effective_user_id(),
|
||||
model_name=(model_name or None),
|
||||
)
|
||||
except EmptyThreadError:
|
||||
raise HTTPException(status_code=422, detail="文件内容无可沉淀的知识")
|
||||
return {
|
||||
"note": result["note"],
|
||||
"created": result["created"],
|
||||
"notes": result.get("notes", [result["note"]]),
|
||||
"llm_fallback": result.get("llm_fallback", False),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ingest/search")
|
||||
@require_auth
|
||||
async def ingest_search(body: IngestSearchRequest, request: Request) -> dict[str, Any]:
|
||||
@@ -488,6 +571,37 @@ async def get_graph(request: Request) -> dict[str, Any]:
|
||||
return await service.get_graph()
|
||||
|
||||
|
||||
class GraphBlacklistAddRequest(BaseModel):
|
||||
labels: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/graph/blacklist")
|
||||
@require_auth
|
||||
async def list_graph_blacklist(request: Request) -> dict[str, Any]:
|
||||
"""Return blacklisted graph node labels (hidden from the graph)."""
|
||||
service = get_knowledge_service(request)
|
||||
return {"items": await service.list_graph_blacklist()}
|
||||
|
||||
|
||||
@router.post("/graph/blacklist")
|
||||
@require_auth
|
||||
async def add_graph_blacklist(body: GraphBlacklistAddRequest, request: Request) -> dict[str, Any]:
|
||||
"""Blacklist node labels (entity names / note titles); they disappear from the graph."""
|
||||
service = get_knowledge_service(request)
|
||||
added = await service.add_graph_blacklist(body.labels, created_by=get_effective_user_id())
|
||||
return {"added": added, "items": await service.list_graph_blacklist()}
|
||||
|
||||
|
||||
@router.delete("/graph/blacklist/{entry_id}")
|
||||
@require_auth
|
||||
async def remove_graph_blacklist(entry_id: str, request: Request) -> dict[str, Any]:
|
||||
"""Remove a blacklist entry so the node shows up in the graph again."""
|
||||
service = get_knowledge_service(request)
|
||||
if not await service.remove_graph_blacklist(entry_id):
|
||||
raise HTTPException(status_code=404, detail="Blacklist entry not found")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/export/graph")
|
||||
@require_auth
|
||||
async def export_graph(request: Request) -> dict[str, Any]:
|
||||
|
||||
@@ -66,6 +66,7 @@ from app.gateway.roundtable_run_policy import merge_skill_stop_names, roundtable
|
||||
from app.gateway.routers._roundtable_seed import (
|
||||
COORDINATOR_AGENT_ID,
|
||||
REPORT_AGENT_ID,
|
||||
SUMMARY_AGENT_ID,
|
||||
ensure_roundtable_functional_agents,
|
||||
)
|
||||
from deerflow.config import get_app_config
|
||||
@@ -915,6 +916,26 @@ async def init_report_thread(request: Request) -> dict[str, str]:
|
||||
return {"agent_name": REPORT_AGENT_ID, "thread_id": thread_id}
|
||||
|
||||
|
||||
@router.post("/summary/init")
|
||||
async def init_summary_thread(request: Request) -> dict[str, str]:
|
||||
"""为 Step 3「总结报告」创建一个供方案总结报告智能体(``roundtable-summary``)
|
||||
运行的线程,返回 ``{"agent_name", "thread_id"}``。
|
||||
|
||||
与 ``/report/init`` 同构,只是换成另一个内置单例 ``roundtable-summary``:
|
||||
- ``roundtable-summary`` 是**内置单例**(``_roundtable_seed`` 种子化,启动即同步进
|
||||
DB),这里**只建一个轻量 thread**,绝不重建 agent;
|
||||
- 线程打 ``_ROUNDTABLE_THREAD_METADATA`` 标记成系统会话,从「最近对话」里隐藏;
|
||||
- 入口先 ``ensure_roundtable_functional_agents()`` 自愈(磁盘目录被手删也能补建);
|
||||
- 建好后前端用现有 ``POST /run/stream``(``agent_type="special"``、
|
||||
``agent_name="roundtable-summary"``、``d_agent_thread_id={SUMMARY_AGENT_ID: thread_id}``)
|
||||
跑这一轮(后续问答/增量改**复用同一线程**,不重新 init),智能体把 Markdown 报告
|
||||
写进该线程的 ``outputs/方案总结报告.md``,再由沙箱预览。
|
||||
"""
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
return {"agent_name": SUMMARY_AGENT_ID, "thread_id": thread_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 上游 LangGraph SSE 透传(intent / recommend / leader / special 共用)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1572,6 +1593,8 @@ async def _special_run(
|
||||
# 后台进程内网关用同一份 policy,保证「后台挂起」与「前端手动调用」传参一致。
|
||||
if agent_name == REPORT_AGENT_ID:
|
||||
role = "report"
|
||||
elif agent_name == SUMMARY_AGENT_ID:
|
||||
role = "summary"
|
||||
elif no_file:
|
||||
role = "seat_parallel"
|
||||
else:
|
||||
|
||||
@@ -62,13 +62,6 @@ def _anonymize(resp: AdminLeaderboardResponse) -> AdminLeaderboardResponse:
|
||||
user.email = _mask_email(user.email)
|
||||
user.user_id = hashed(user.user_id)
|
||||
|
||||
for bucket in (resp.recent_questions, resp.frequent_questions):
|
||||
for question in bucket:
|
||||
question.email = _mask_email(question.email)
|
||||
question.user_id = hashed(question.user_id)
|
||||
question.thread_id = ""
|
||||
question.run_id = ""
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,8 @@ router = APIRouter(prefix="/api/roundtable-drafts", tags=["roundtable-drafts"])
|
||||
|
||||
class DraftMetaResponse(BaseModel):
|
||||
id: str
|
||||
# 外部任务 id(无界嵌入抽屉传入);非空时草稿按 task_id 共享、不按 user 分权。
|
||||
task_id: str | None = None
|
||||
title: str = ""
|
||||
furthest_step: int = 1
|
||||
created_at: datetime | str | None = None
|
||||
@@ -62,6 +64,8 @@ class DraftCreateRequest(BaseModel):
|
||||
# Optional client-provided id so the browser can keep one stable id across
|
||||
# the create + subsequent updates without an extra round-trip.
|
||||
id: str | None = Field(default=None, max_length=64)
|
||||
# 外部任务 id(无界嵌入抽屉);设置后草稿按 task_id 共享、不按 user 分权。
|
||||
task_id: str | None = Field(default=None, max_length=128)
|
||||
title: str = Field(default="", max_length=512)
|
||||
furthest_step: int = Field(default=1, ge=1, le=3)
|
||||
step1: dict[str, Any] | None = None
|
||||
@@ -74,6 +78,7 @@ class DraftUpdateRequest(BaseModel):
|
||||
# so it never clobbers a manual rename.
|
||||
title: str | None = Field(default=None, max_length=512)
|
||||
furthest_step: int | None = Field(default=None, ge=1, le=3)
|
||||
task_id: str | None = Field(default=None, max_length=128)
|
||||
step1: dict[str, Any] | None = None
|
||||
step2: dict[str, Any] | None = None
|
||||
step3: dict[str, Any] | None = None
|
||||
@@ -165,6 +170,7 @@ async def create_draft(request: Request, body: DraftCreateRequest) -> DraftRespo
|
||||
user_id,
|
||||
{
|
||||
"id": (body.id or uuid4().hex),
|
||||
"task_id": (body.task_id or None),
|
||||
"title": body.title.strip(),
|
||||
"furthest_step": body.furthest_step,
|
||||
"step1": body.step1,
|
||||
@@ -175,6 +181,19 @@ async def create_draft(request: Request, body: DraftCreateRequest) -> DraftRespo
|
||||
return DraftResponse(**row)
|
||||
|
||||
|
||||
@router.get("/by-task/{task_id}/latest", response_model=DraftResponse)
|
||||
async def get_latest_draft_by_task(request: Request, task_id: str) -> DraftResponse:
|
||||
"""按外部任务 id 取最新一条草稿(**不按 user 分权**);无则 404。
|
||||
|
||||
无界嵌入抽屉打开时调用:若该任务已有聊天记录,回显最新一条。
|
||||
"""
|
||||
store = _get_store(request)
|
||||
row = await store.get_latest_draft_by_task(task_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="No draft for this task")
|
||||
return DraftResponse(**row)
|
||||
|
||||
|
||||
@router.get("/{draft_id}", response_model=DraftResponse)
|
||||
async def get_draft(request: Request, draft_id: str) -> DraftResponse:
|
||||
store = _get_store(request)
|
||||
|
||||
@@ -166,6 +166,7 @@ class SkillResponse(BaseModel):
|
||||
owner_user_id: str | None = Field(default=None, description="Custom skill owner (null = built-in or legacy)")
|
||||
owner_username: str | None = Field(default=None, description="Publisher username (email local part). Null for built-in skills.")
|
||||
published: bool = Field(default=False, description="Whether the owner has made this skill visible to other users")
|
||||
square_id: str = Field(default="", description="Publish square (发布广场) the skill is filed under once published. Empty string = the default square.")
|
||||
builtin: bool = Field(default=False, description="True when owner_user_id is null (no human owner)")
|
||||
detail: str | None = Field(default=None, description="Human-authored long-form detail text (Markdown), editable by owner/admin")
|
||||
display: dict[str, Any] | None = Field(default=None, description="Optional frontend display adapter")
|
||||
@@ -308,12 +309,14 @@ def _skill_to_response(
|
||||
builtin = skill.category != SkillCategory.CUSTOM
|
||||
detail: str | None = None
|
||||
name_zh: str | None = None
|
||||
square_id = ""
|
||||
if ownership is not None:
|
||||
owner_user_id = ownership.get("owner_user_id")
|
||||
published = bool(ownership.get("published", False))
|
||||
builtin = bool(ownership.get("builtin", owner_user_id is None))
|
||||
detail = ownership.get("detail")
|
||||
name_zh = ownership.get("name_zh")
|
||||
square_id = str(ownership.get("square_id") or "")
|
||||
|
||||
display = None if display_override is _UNSET else display_override
|
||||
if display_override is _UNSET:
|
||||
@@ -342,6 +345,7 @@ def _skill_to_response(
|
||||
source=entry.get("source", "upload"),
|
||||
owner_user_id=owner_user_id,
|
||||
published=published,
|
||||
square_id=square_id,
|
||||
builtin=builtin,
|
||||
detail=detail,
|
||||
display=display,
|
||||
@@ -772,6 +776,7 @@ async def list_skills(
|
||||
tag_store: TagStore = Depends(get_tag_store),
|
||||
search: str | None = Query(default=None, description="Case-insensitive fuzzy match on skill name or Chinese name (name_zh)"),
|
||||
tag_ids: list[str] | None = Query(default=None, description="Keep skills carrying any of these tags (OR)"),
|
||||
square: str | None = Query(default=None, description="Keep only skills filed under this publish square (发布广场). Omit for all squares."),
|
||||
sort: str = Query(
|
||||
default="time_desc",
|
||||
pattern="^(time_desc|time_asc|activity_desc|activity_asc|usage_desc|usage_asc|name_asc|name_desc)$",
|
||||
@@ -831,6 +836,16 @@ async def list_skills(
|
||||
)
|
||||
responses = await _attach_and_filter_tags(tag_store, responses, search, tag_ids)
|
||||
responses = await _attach_owner_usernames(responses)
|
||||
square_id = square.strip() if square and square.strip() else None
|
||||
if square_id:
|
||||
from deerflow.config.system_settings import load_system_settings, resolve_default_square_id
|
||||
|
||||
square_is_default = square_id == resolve_default_square_id(load_system_settings().publish_squares)
|
||||
responses = [
|
||||
r
|
||||
for r in responses
|
||||
if (r.square_id or "") == square_id or (square_is_default and not (r.square_id or ""))
|
||||
]
|
||||
responses = _sort_skill_responses(responses, sort)
|
||||
return SkillsListResponse(skills=responses)
|
||||
except HTTPException:
|
||||
@@ -1366,12 +1381,14 @@ class SkillPublishRequest(BaseModel):
|
||||
"""Request body for toggling a custom skill's published flag."""
|
||||
|
||||
published: bool = Field(..., description="Whether to publish (True) or unpublish (False)")
|
||||
square_id: str | None = Field(default=None, description="Publish square (发布广场) to file the skill under when publishing. Omit / empty = the default square.")
|
||||
|
||||
|
||||
class SkillPublishResponse(BaseModel):
|
||||
name: str
|
||||
owner_user_id: str | None
|
||||
published: bool
|
||||
square_id: str = ""
|
||||
|
||||
|
||||
@router.put(
|
||||
@@ -1393,10 +1410,18 @@ async def update_custom_skill_published(
|
||||
record, mode = await _get_editable_skill_or_raise(skill_store, skill_name, user_id, is_admin=is_admin)
|
||||
|
||||
was_published = bool(record.get("published", False))
|
||||
# When publishing, also file the skill into the chosen square (normalized
|
||||
# to a valid configured square id; default when unset). Only one square
|
||||
# per skill — a single column, so it can never be "published twice".
|
||||
update_fields: dict[str, Any] = {"published": body.published}
|
||||
if body.published:
|
||||
from deerflow.config.system_settings import load_system_settings, normalize_square_id
|
||||
|
||||
update_fields["square_id"] = normalize_square_id(body.square_id, load_system_settings().publish_squares)
|
||||
if mode == "admin":
|
||||
updated = await skill_store.update_admin(skill_name, {"published": body.published})
|
||||
updated = await skill_store.update_admin(skill_name, update_fields)
|
||||
else:
|
||||
updated = await skill_store.update(skill_name, user_id, {"published": body.published})
|
||||
updated = await skill_store.update(skill_name, user_id, update_fields)
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=500, detail="Failed to persist published state")
|
||||
|
||||
@@ -1419,6 +1444,7 @@ async def update_custom_skill_published(
|
||||
name=updated["name"],
|
||||
owner_user_id=updated.get("owner_user_id"),
|
||||
published=bool(updated.get("published", False)),
|
||||
square_id=str(updated.get("square_id") or ""),
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
@@ -2762,14 +2788,15 @@ class CuratorConfigResponse(BaseModel):
|
||||
|
||||
|
||||
class CuratorConfigRequest(BaseModel):
|
||||
"""Payload for updating skill_evolution overrides.
|
||||
"""Payload for updating user-facing curator settings in skill_evolution.json.
|
||||
|
||||
All fields are required so the frontend always sends a complete config;
|
||||
backend writes the entire payload to skill_evolution.json atomically.
|
||||
All other settings (enabled, timeouts) are global-only and controlled via config.yaml.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
|
||||
curator: CuratorSubConfig
|
||||
interval_hours: int = Field(..., ge=1, description="Hours between curator runs.")
|
||||
stale_after_days: int = Field(..., ge=1, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(..., ge=1, description="Days of inactivity before a stale skill is archived.")
|
||||
model_name: str | None = Field(default=None, description="Model for the curator LLM review; null = primary model.")
|
||||
|
||||
|
||||
class CuratorLogRecord(BaseModel):
|
||||
@@ -2785,11 +2812,7 @@ class CuratorLogsResponse(BaseModel):
|
||||
|
||||
|
||||
def _build_curator_config_response(evo) -> CuratorConfigResponse:
|
||||
"""Pure helper so both the GET route and the post-PUT refresh share one shape.
|
||||
|
||||
``evo`` is a ``SkillEvolutionConfig`` instance (already merged with per-user
|
||||
overrides if any).
|
||||
"""
|
||||
"""Pure helper so both the GET route and the post-PUT refresh share one shape."""
|
||||
return CuratorConfigResponse(
|
||||
enabled=evo.enabled,
|
||||
curator=CuratorSubConfig(
|
||||
@@ -2805,8 +2828,7 @@ def _build_curator_config_response(evo) -> CuratorConfigResponse:
|
||||
|
||||
@router.get("/curator/config", response_model=CuratorConfigResponse, summary="Get Skill Evolution Config")
|
||||
async def get_curator_config(request: Request) -> CuratorConfigResponse:
|
||||
"""Return the current curator config for the authenticated user (yaml
|
||||
defaults + that user's overrides)."""
|
||||
"""Return the current curator config for the authenticated user (yaml defaults + per-user overrides)."""
|
||||
try:
|
||||
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
|
||||
|
||||
@@ -2819,21 +2841,22 @@ async def get_curator_config(request: Request) -> CuratorConfigResponse:
|
||||
|
||||
@router.put("/curator/config", response_model=CuratorConfigResponse, summary="Update Skill Evolution Config")
|
||||
async def update_curator_config(body: CuratorConfigRequest, request: Request) -> CuratorConfigResponse:
|
||||
"""Persist per-user overrides to ``users/{user_id}/skill_evolution.json``
|
||||
and return the effective merged config.
|
||||
"""Persist the three user-facing curator thresholds to the per-user ``skill_evolution.json``.
|
||||
|
||||
The payload is validated against the ``SkillEvolutionConfig`` shape; unknown
|
||||
fields are rejected.
|
||||
Other settings (enabled, model, timeouts) are global and controlled via config.yaml.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
|
||||
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user, save_user_overrides
|
||||
|
||||
user_id = _current_user_id(request)
|
||||
payload = body.model_dump()
|
||||
# Round-trip through SkillEvolutionConfig to ensure persisted shape stays valid.
|
||||
SkillEvolutionConfig.model_validate(payload)
|
||||
save_user_overrides(user_id, payload)
|
||||
save_user_overrides(user_id, {
|
||||
"curator": {
|
||||
"interval_hours": body.interval_hours,
|
||||
"stale_after_days": body.stale_after_days,
|
||||
"archive_after_days": body.archive_after_days,
|
||||
"model_name": body.model_name,
|
||||
},
|
||||
})
|
||||
return _build_curator_config_response(load_skill_evolution_config_for_user(user_id))
|
||||
except HTTPException:
|
||||
raise
|
||||
|
||||
@@ -14,8 +14,12 @@ from deerflow.config import get_app_config
|
||||
from deerflow.config.system_settings import (
|
||||
LeaderboardSettings,
|
||||
PromptPrefixSettings,
|
||||
PublishSquare,
|
||||
PublishSquaresSettings,
|
||||
SystemSettings,
|
||||
list_publish_squares,
|
||||
load_system_settings,
|
||||
resolve_default_square_id,
|
||||
save_system_settings,
|
||||
)
|
||||
|
||||
@@ -219,6 +223,88 @@ async def update_private_square(body: PrivateSquareUpdateRequest, request: Reque
|
||||
)
|
||||
|
||||
|
||||
class PublishSquareItem(BaseModel):
|
||||
id: str = Field(..., description="Stable square id")
|
||||
name: str = Field(..., description="Display name / tab label")
|
||||
description: str = Field(default="", description="Optional description")
|
||||
|
||||
|
||||
class PublishSquaresResponse(BaseModel):
|
||||
"""The configured publish squares plus the resolved default.
|
||||
|
||||
Readable by any logged-in user — squares are pure category tabs with no
|
||||
access control, so the gallery needs the list to render its sub-tabs and
|
||||
the publish flow needs it to populate the square picker.
|
||||
"""
|
||||
|
||||
squares: list[PublishSquareItem] = Field(default_factory=list)
|
||||
default_square_id: str = Field(..., description="Square id used when a publisher picks none")
|
||||
|
||||
|
||||
class PublishSquaresUpdateRequest(BaseModel):
|
||||
squares: list[PublishSquareItem] | None = Field(default=None, description="Full ordered replacement list of squares")
|
||||
default_square_id: str | None = Field(default=None, description="Which square id is the publish default")
|
||||
|
||||
|
||||
def _publish_squares_response(settings: PublishSquaresSettings) -> PublishSquaresResponse:
|
||||
squares = list_publish_squares(settings)
|
||||
return PublishSquaresResponse(
|
||||
squares=[PublishSquareItem(id=s.id, name=s.name, description=s.description) for s in squares],
|
||||
default_square_id=resolve_default_square_id(settings),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/publish-squares", response_model=PublishSquaresResponse)
|
||||
async def get_publish_squares() -> PublishSquaresResponse:
|
||||
"""Return the configured publish squares (readable by any logged-in user)."""
|
||||
return _publish_squares_response(load_system_settings().publish_squares)
|
||||
|
||||
|
||||
@router.put("/publish-squares", response_model=PublishSquaresResponse)
|
||||
async def update_publish_squares(body: PublishSquaresUpdateRequest, request: Request) -> PublishSquaresResponse:
|
||||
"""Replace the publish-squares configuration. Admin only.
|
||||
|
||||
``squares`` is a full ordered replacement. Each square needs a non-empty
|
||||
name; a missing id is auto-generated from a slug of the name. Duplicate ids
|
||||
are de-duplicated (first wins). ``default_square_id`` must reference one of
|
||||
the surviving squares, else it falls back to the first.
|
||||
"""
|
||||
from uuid import uuid4
|
||||
|
||||
await _require_admin(request)
|
||||
|
||||
current = load_system_settings()
|
||||
fields = body.model_dump(exclude_unset=True)
|
||||
update: dict[str, object] = {}
|
||||
|
||||
if "squares" in fields and fields["squares"] is not None:
|
||||
normalized: list[PublishSquare] = []
|
||||
seen_ids: set[str] = set()
|
||||
for raw in body.squares or []:
|
||||
name = (raw.name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
sid = (raw.id or "").strip() or uuid4().hex[:8]
|
||||
if sid in seen_ids:
|
||||
continue
|
||||
seen_ids.add(sid)
|
||||
normalized.append(PublishSquare(id=sid, name=name, description=(raw.description or "").strip()))
|
||||
update["squares"] = normalized
|
||||
|
||||
if "default_square_id" in fields and fields["default_square_id"] is not None:
|
||||
update["default_square_id"] = str(fields["default_square_id"]).strip()
|
||||
|
||||
if update:
|
||||
updated = current.publish_squares.model_copy(update=update)
|
||||
# Re-anchor the default to a surviving square id.
|
||||
if updated.squares and updated.default_square_id not in {s.id for s in updated.squares}:
|
||||
updated = updated.model_copy(update={"default_square_id": updated.squares[0].id})
|
||||
saved = save_system_settings(current.model_copy(update={"publish_squares": updated}))
|
||||
else:
|
||||
saved = current
|
||||
return _publish_squares_response(saved.publish_squares)
|
||||
|
||||
|
||||
class AppearanceResponse(BaseModel):
|
||||
"""Effective system-wide appearance defaults."""
|
||||
|
||||
|
||||
@@ -173,6 +173,7 @@ class ThreadSearchRequest(BaseModel):
|
||||
limit: int = Field(default=100, ge=1, le=1000, description="Maximum results")
|
||||
offset: int = Field(default=0, ge=0, description="Pagination offset")
|
||||
status: str | None = Field(default=None, description="Filter by thread status")
|
||||
query: str | None = Field(default=None, description="Case-insensitive substring match on the thread title")
|
||||
|
||||
|
||||
class ThreadStateResponse(BaseModel):
|
||||
@@ -660,11 +661,19 @@ async def search_threads(body: ThreadSearchRequest, request: Request) -> list[Th
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
repo = get_thread_store(request)
|
||||
# System threads (scheduler runs, roundtable workers, …) carry
|
||||
# ``metadata.system=true`` and are machine-created — hide them from the
|
||||
# default conversation list so they don't crowd out real chats. A caller
|
||||
# that explicitly filters on ``thread_type``/``system`` (e.g. the studio
|
||||
# notebook list) opts out of the exclusion.
|
||||
explicit_system_filter = bool(body.metadata) and ("thread_type" in body.metadata or "system" in body.metadata)
|
||||
rows = await repo.search(
|
||||
metadata=body.metadata or None,
|
||||
status=body.status,
|
||||
limit=body.limit,
|
||||
offset=body.offset,
|
||||
exclude_system=not explicit_system_filter,
|
||||
query=body.query,
|
||||
)
|
||||
return [
|
||||
ThreadResponse(
|
||||
@@ -683,6 +692,32 @@ async def search_threads(body: ThreadSearchRequest, request: Request) -> list[Th
|
||||
]
|
||||
|
||||
|
||||
class ThreadCountResponse(BaseModel):
|
||||
"""Response model for the thread count endpoint."""
|
||||
|
||||
total: int = Field(description="Number of threads matching the filters")
|
||||
|
||||
|
||||
@router.post("/count", response_model=ThreadCountResponse)
|
||||
async def count_threads(body: ThreadSearchRequest, request: Request) -> ThreadCountResponse:
|
||||
"""Count threads matching the same filters as ``/threads/search``.
|
||||
|
||||
``limit``/``offset`` in the body are ignored. Backs the chats page
|
||||
pagination total ("共 N 条 / X 页") without fetching any rows.
|
||||
"""
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
repo = get_thread_store(request)
|
||||
explicit_system_filter = bool(body.metadata) and ("thread_type" in body.metadata or "system" in body.metadata)
|
||||
total = await repo.count(
|
||||
metadata=body.metadata or None,
|
||||
status=body.status,
|
||||
exclude_system=not explicit_system_filter,
|
||||
query=body.query,
|
||||
)
|
||||
return ThreadCountResponse(total=total)
|
||||
|
||||
|
||||
@router.patch("/{thread_id}", response_model=ThreadResponse)
|
||||
@require_permission("threads", "write", owner_check=True, require_existing=True)
|
||||
async def patch_thread(thread_id: str, body: ThreadPatchRequest, request: Request) -> ThreadResponse:
|
||||
|
||||
@@ -14,7 +14,7 @@ models:
|
||||
# display_name: DeepSeek Chat
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-chat
|
||||
# api_key: sk-1520028668cb4cc388ed8315ff1e856e
|
||||
# api_key: sk-3fee562f834a4c08bb0d471ef44cb445
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
@@ -23,11 +23,24 @@ models:
|
||||
# supports_vision: false
|
||||
# supports_thinking: true
|
||||
|
||||
# - name: zai-org/GLM-5-FP8
|
||||
# display_name: zai-org/GLM-5-FP8
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: zai-org/GLM-5-FP8
|
||||
# api_key: modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA
|
||||
# base_url: https://api.us-west-2.modal.direct/v1
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
|
||||
# - name: deepseek-v4-flash
|
||||
# display_name: DeepSeek V4 Flash
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: deepseek-v4-flash
|
||||
# api_key: sk-e1c22b7931dc446f8b67e57878edd105
|
||||
# api_key: sk-3fee562f834a4c08bb0d471ef44cb445
|
||||
# base_url: https://api.deepseek.com
|
||||
# request_timeout: 600.0
|
||||
# max_retries: 2
|
||||
@@ -90,6 +103,16 @@ models:
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
# - name: Qwen/Qwen3.6-27B
|
||||
# display_name: Qwen/Qwen3.6-27B
|
||||
# use: langchain_openai:ChatOpenAI
|
||||
# model: Qwen/Qwen3.6-27B
|
||||
# api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
# base_url: https://api.siliconflow.cn/v1/
|
||||
# max_tokens: 4096
|
||||
# temperature: 1.0
|
||||
# supports_vision: true
|
||||
# supports_thinking: true
|
||||
- name: deepseek-ai/DeepSeek-V4-Flash
|
||||
display_name: deepseek-ai/DeepSeek-V4-Flash
|
||||
use: langchain_openai:ChatOpenAI
|
||||
@@ -100,6 +123,36 @@ models:
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: Pro/MiniMaxAI/MiniMax-M2.5
|
||||
display_name: Pro/MiniMaxAI/MiniMax-M2.5
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: Pro/MiniMaxAI/MiniMax-M2.5
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: inclusionAI/Ling-flash-2.0
|
||||
display_name: inclusionAI/Ling-flash-2.0
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: inclusionAI/Ling-flash-2.0
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: deepseek-ai/DeepSeek-V4-Pro
|
||||
display_name: deepseek-ai/DeepSeek-V4-Pro
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-ai/DeepSeek-V4-Pro
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
# 识图(图片识别)配置。
|
||||
# - model_name:指定一个 models[] 中的模型名,作为“默认识图大模型”。view_image 工具会把
|
||||
# 图片交给该模型识别,并把识别出的文字结果返回给主对话模型 —— 这样即使主对话模型不支持
|
||||
@@ -122,26 +175,6 @@ uploads:
|
||||
auto_convert_documents: true
|
||||
pdf_converter: auto
|
||||
|
||||
# Chrome 插件「当前网页发送到 Deer」入口。默认关闭,打开后才启用
|
||||
# /api/browser-context/capture。它只保存当前页 Markdown 到线程 uploads,
|
||||
# 不会改变普通聊天、上传或工具调用流程。
|
||||
browser_context:
|
||||
enabled: true
|
||||
max_content_chars: 200000
|
||||
max_title_chars: 300
|
||||
max_url_chars: 2048
|
||||
filename_prefix: webpage
|
||||
route_base: /page/workspace
|
||||
task_result_timeout_seconds: 60
|
||||
task_poll_interval_seconds: 1
|
||||
task_lease_seconds: 120
|
||||
tool_result_max_chars: 60000
|
||||
# 交互式页面操作(browser_act:点击/输入/滚动/等待/跳转)。关闭后只保留只读抓取。
|
||||
actions_enabled: true
|
||||
action_result_timeout_seconds: 45
|
||||
max_observed_elements: 150
|
||||
observation_text_max_chars: 12000
|
||||
|
||||
title:
|
||||
enabled: true
|
||||
max_words: 6
|
||||
@@ -159,7 +192,6 @@ title:
|
||||
|
||||
tool_groups:
|
||||
- name: web
|
||||
- name: browser
|
||||
- name: file:read
|
||||
- name: file:write
|
||||
- name: bash
|
||||
@@ -175,14 +207,6 @@ tools:
|
||||
use: deerflow.community.jina_ai.tools:web_fetch_tool
|
||||
timeout: 15
|
||||
|
||||
- name: browser_fetch_page
|
||||
group: browser
|
||||
use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
|
||||
|
||||
- name: browser_act
|
||||
group: browser
|
||||
use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
|
||||
|
||||
- name: ls
|
||||
group: file:read
|
||||
use: deerflow.sandbox.tools:ls_tool
|
||||
@@ -216,12 +240,28 @@ tools:
|
||||
sandbox:
|
||||
use: deerflow.sandbox.local:LocalSandboxProvider
|
||||
allow_host_bash: true
|
||||
mounts:
|
||||
- host_path: C:\Users\iie\Desktop\deerflow-new\offline-backend-20260512\backend\knowledge\knowledge_base
|
||||
container_path: /mnt/knowledge
|
||||
read_only: true
|
||||
|
||||
skills:
|
||||
path: skills
|
||||
security_scan:
|
||||
enabled: false
|
||||
|
||||
skill_evolution:
|
||||
enabled: true
|
||||
curator:
|
||||
enabled: true # 每 24 小时自动清理 + 合并 skill
|
||||
interval_hours: 24
|
||||
review_worker:
|
||||
enabled: true # 被动复盘开关
|
||||
min_tool_calls: 5 # 触发阈值:本轮至少 5 次工具调用,与 prompt 中"≥5 步"硬条件一致
|
||||
max_messages: 60 # 传给 LLM 的最大消息条数
|
||||
timeout_seconds: 120 # 单次 review 超时(秒)
|
||||
model_name: # 留空则使用主模型
|
||||
|
||||
agents_api:
|
||||
enabled: true
|
||||
|
||||
@@ -251,7 +291,7 @@ auth_login:
|
||||
|
||||
database:
|
||||
backend: sqlite
|
||||
sqlite_dir: /app/backend/.deer-flow/data
|
||||
sqlite_dir: .deer-flow/data
|
||||
pool_size: 20 # 基础连接池(postgres/mysql 生效;sqlite 忽略)
|
||||
max_overflow: 20 # 高峰期可临时超出 pool_size 的额外连接数,真实上限 = pool_size + max_overflow
|
||||
pool_timeout: 30 # 池满时等待空闲连接的秒数,超时报错
|
||||
@@ -259,7 +299,7 @@ database:
|
||||
|
||||
checkpointer:
|
||||
type: sqlite
|
||||
connection_string: /app/backend/.deer-flow/data/deerflow.db
|
||||
connection_string: .deer-flow/data/deerflow.db
|
||||
|
||||
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
|
||||
# 三层覆盖优先级(高->低):
|
||||
@@ -344,39 +384,3 @@ log_system:
|
||||
enabled: true
|
||||
url: 'https://your-log-system.example.com'
|
||||
label: '日志系统'
|
||||
|
||||
# 产品内知识库 —— obsidian-wiki 兼容 vault。知识元数据写入数据库,正文
|
||||
# content_md 写入 Markdown vault(index.md/log.md/hot.md/.manifest.json/frontmatter/
|
||||
# wikilinks/wiki-export 图谱产物)。知识库为全局共享,不做 user_id 隔离,
|
||||
# 写入时记录 created_by / updated_by 作为审计。
|
||||
knowledge:
|
||||
enabled: true
|
||||
provider: obsidian_wiki
|
||||
vault_dir: data/knowledge/obsidian-vault
|
||||
project_name: zncm
|
||||
auto_ingest_enabled: false
|
||||
auto_ingest_min_chars: 300
|
||||
# 用大模型按 wiki-capture 规范提炼陈述式知识(而非堆砌对话原文)。
|
||||
# 关闭则回退到规则抽取;extract_model_name 留空时用 models[0]。
|
||||
llm_extract_enabled: true
|
||||
extract_model_name: ''
|
||||
extract_max_input_chars: 12000
|
||||
max_thread_messages: 80
|
||||
max_source_items: 20
|
||||
rag_enabled: false
|
||||
rag_limit: 5
|
||||
# 阶段三:写入前脱敏(token/密钥/手机号/身份证/邮箱);重复检测相似度阈值。
|
||||
sensitive_redaction_enabled: true
|
||||
dedup_similarity_threshold: 0.82
|
||||
export:
|
||||
enabled: true
|
||||
graph_html_enabled: true
|
||||
graph_json_enabled: true
|
||||
query:
|
||||
use_hot_md: true
|
||||
use_index_md: true
|
||||
use_frontmatter_scan: true
|
||||
embedding:
|
||||
enabled: false
|
||||
provider: local
|
||||
model: ''
|
||||
@@ -4,13 +4,26 @@ DeerFlow includes automatic conversation summarization to handle long conversati
|
||||
|
||||
## Overview
|
||||
|
||||
The summarization feature uses LangChain's `SummarizationMiddleware` to monitor conversation history and trigger summarization based on configurable thresholds. When activated, it:
|
||||
The summarization feature uses DeerFlow's `DeerFlowSummarizationMiddleware` (a subclass of LangChain's `SummarizationMiddleware`) to monitor conversation history and trigger summarization based on configurable thresholds. When activated, it:
|
||||
|
||||
1. Monitors message token counts in real-time
|
||||
2. Triggers summarization when thresholds are met
|
||||
3. Keeps recent messages intact while summarizing older exchanges
|
||||
4. Maintains AI/Tool message pairs together for context continuity
|
||||
5. Injects the summary back into the conversation
|
||||
5. Substitutes the summary for the older messages **in the model request only**
|
||||
|
||||
### Non-destructive (display-preserving) compression
|
||||
|
||||
> **Important:** DeerFlow compresses **transiently**, not destructively.
|
||||
|
||||
LangChain's stock `SummarizationMiddleware` runs in `before_model` and persists a `RemoveMessage(REMOVE_ALL_MESSAGES)` to the checkpoint — which permanently deletes the original Q&A from both the model context *and* the conversation the frontend renders. DeerFlow overrides this:
|
||||
|
||||
- `before_model` / `abefore_model` are **no-ops** (the destructive path is disabled).
|
||||
- Compression runs in `wrap_model_call` / `awrap_model_call`: for a single model call, the message list is replaced with `[summary, *recent]` via `request.override(messages=...)`. **Nothing is written back to state.**
|
||||
- The full original messages stay in the checkpoint, so the frontend keeps showing the user's real conversation. The summary message is named `summary` and flagged `hide_from_ui`, and because it only ever lives in the transient request it never reaches the frontend at all.
|
||||
- The generated summary is cached **per-thread** so we don't pay an extra LLM call on every model invocation. The cached summary is reused (no LLM call) until enough *new* messages accumulate to push the compressed context back over the trigger threshold (a sticky boundary mirroring the stock middleware's post-compaction behavior). The cache is bounded (oldest threads evicted first).
|
||||
|
||||
> **Legacy threads:** conversations compacted by an older (destructive) build already lost their original messages and carry a persisted `summary` message. The frontend hides any `name: "summary"` message, so the stray summary bubble no longer shows — but those already-discarded messages cannot be recovered.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -169,10 +182,7 @@ The default LangChain prompt instructs the model to:
|
||||
- Messages to summarize (older messages beyond the `keep` threshold)
|
||||
- Messages to preserve (recent messages within the `keep` threshold)
|
||||
4. **Summary Generation**: The model generates a concise summary of the older messages
|
||||
5. **Context Replacement**: The message history is updated:
|
||||
- All old messages are removed
|
||||
- A single summary message is added
|
||||
- Recent messages are preserved
|
||||
5. **Context Replacement (transient)**: The message list **handed to the model for this call** is rebuilt as `[summary, *recent]`. The persisted state is left untouched — old messages are **not** removed, so the frontend still renders the full original conversation.
|
||||
6. **AI/Tool Pair Protection**: The system ensures AI messages and their corresponding tool messages stay together
|
||||
7. **Skill Rescue**: Before the summary is generated, the most recently loaded skill files (tool results whose tool name is in `skill_file_read_tool_names` and whose target path is under `skills.container_path`) are lifted out of the summarization set and prepended to the preserved tail. Selection walks newest-first under three budgets: `preserve_recent_skill_count`, `preserve_recent_skill_tokens`, and `preserve_recent_skill_tokens_per_skill`. The triggering AIMessage and all of its paired ToolMessages move together so tool_call ↔ tool_result pairing stays intact.
|
||||
|
||||
@@ -298,8 +308,9 @@ The middleware intelligently preserves message context:
|
||||
### Code Structure
|
||||
|
||||
- **Configuration**: `packages/harness/deerflow/config/summarization_config.py`
|
||||
- **Integration**: `packages/harness/deerflow/agents/lead_agent/agent.py`
|
||||
- **Middleware**: Uses `langchain.agents.middleware.SummarizationMiddleware`
|
||||
- **Integration**: `packages/harness/deerflow/agents/lead_agent/agent.py` (`_create_summarization_middleware`)
|
||||
- **Middleware**: `packages/harness/deerflow/agents/middlewares/summarization_middleware.py` — `DeerFlowSummarizationMiddleware`, subclassing `langchain.agents.middleware.SummarizationMiddleware` but overriding compaction to be transient (`wrap_model_call`) instead of destructive (`before_model`)
|
||||
- **Tests**: `tests/test_summarization_middleware.py`
|
||||
|
||||
### Middleware Order
|
||||
|
||||
@@ -313,9 +324,9 @@ Summarization runs after ThreadData and Sandbox initialization but before Title
|
||||
|
||||
### State Management
|
||||
|
||||
- Summarization is stateless - configuration is loaded once at startup
|
||||
- Summaries are added as regular messages in the conversation history
|
||||
- The checkpointer persists the summarized history automatically
|
||||
- Configuration is loaded once at startup
|
||||
- Summarization is **non-destructive**: the summary replaces older messages only in the transient model request (`wrap_model_call`), never in persisted state. The checkpointer therefore keeps the full original conversation
|
||||
- The per-thread summary cache lives in the middleware instance (in-memory, bounded). It is a cost optimization only — a cold/stale cache simply re-summarizes, which is always correct
|
||||
|
||||
## Example Configurations
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"code-retrieval": { "enabled": true },
|
||||
"news-briefing": { "enabled": true },
|
||||
"workspace-search": { "enabled": true },
|
||||
"policy-knowledge": { "enabled": true },
|
||||
"policy-knowledge": { "enabled": false },
|
||||
"html-page-builder": { "enabled": true },
|
||||
"html-charts": { "enabled": true }
|
||||
}
|
||||
|
||||
+42
-9
@@ -34,17 +34,20 @@
|
||||
### 关键事件
|
||||
|
||||
**1987.11 开放赴中国大陆探亲**:
|
||||
|
||||
- 蒋经国政府宣布开放赴大陆探亲
|
||||
- 表面为人道措施,实质开启人员与资金流动
|
||||
- "探亲带带货"成为早期贸易雏形
|
||||
|
||||
**1990s 间接贸易兴起**:
|
||||
|
||||
- 贸易通过香港等第三地转口进行
|
||||
- 台商以个人或第三地公司名义登陆
|
||||
- 鞋业、成衣、玩具等劳力密集产业率先外移
|
||||
- 1992 邓小平南巡后,台商投资第一波高峰
|
||||
|
||||
**1996.09 戒急用忍政策**:
|
||||
|
||||
- 李登辉提出"戒急用忍"
|
||||
- 限制高科技与大型基础设施赴大陆投资
|
||||
- 试图在政治风险上升时踩刹车
|
||||
@@ -77,23 +80,27 @@
|
||||
### 关键事件
|
||||
|
||||
**2001.01 金马小三通**:
|
||||
|
||||
- 金门、马祖与福建沿海通航通商
|
||||
- 象征性大于实质经济意义
|
||||
- 但打破了"完全不直接通商"的禁区
|
||||
|
||||
**2001.11 两岸加入 WTO**:
|
||||
|
||||
- 台湾以"台澎金马个别关税领域"名义加入
|
||||
- 北京以"主权国家"身份加入
|
||||
- 加入 WTO 后,间接贸易的合法性基础松动
|
||||
- 台北面临"是否开放直航与直接投资"的压力
|
||||
|
||||
**2001-2004 积极开放有效管理**:
|
||||
|
||||
- 陈水扁政府调整"戒急用忍"为"积极开放有效管理"
|
||||
- 放宽投资上限(8千万新台币限制放宽)
|
||||
- 但 2006 后又收紧为"积极管理有效开放"
|
||||
|
||||
**2000s 中期 IT 产业大举西进**:
|
||||
- 笔记本电脑、半导体封装测试等中下游产业外移
|
||||
|
||||
- 空间电脑、半导体封装测试等中下游产业外移
|
||||
- 形成台湾接单、大陆生产、出口欧美的三角贸易结构
|
||||
- 台湾对大陆出口依存度突破 20%
|
||||
|
||||
@@ -125,11 +132,13 @@
|
||||
### 关键事件
|
||||
|
||||
**2008.07 周末包机与大陆观光客来台**:
|
||||
|
||||
- 两岸周末包机启动
|
||||
- 大陆观光客来台(首波仅限特定省市)
|
||||
- 大三通的前奏
|
||||
|
||||
**2008.12 大三通**:
|
||||
|
||||
- 海运直航、空运直航、直接通邮
|
||||
- 两岸经贸进入"直接"时代
|
||||
- 物流成本大幅下降,供应链整合加速
|
||||
@@ -137,6 +146,7 @@
|
||||
详见 `entities/people/ma_ying_jeou.md`。
|
||||
|
||||
**2010.06 ECFA 签订**:
|
||||
|
||||
- 两岸经济合作架构协议(ECFA)签署
|
||||
- 包含"早收清单"(539 项台湾货品降税)
|
||||
- 后续需谈判"货品贸易协议"与"服务贸易协议"
|
||||
@@ -145,6 +155,7 @@
|
||||
详见 `narratives/cross_strait/ecfa_legacy_debate.md`。
|
||||
|
||||
**2013.06 两岸服贸协议签署**:
|
||||
|
||||
- 服务贸易协议完成签署
|
||||
- 开放金融、医疗、电信等服务领域
|
||||
- 但需经立法院审查,尚未生效
|
||||
@@ -177,6 +188,7 @@
|
||||
### 关键事件
|
||||
|
||||
**2014.03-04 太阳花学运**:
|
||||
|
||||
- 反对服贸协议黑箱审查
|
||||
- 占领立法院 24 天
|
||||
- 服贸协议实质冻结
|
||||
@@ -185,11 +197,13 @@
|
||||
详见 `events/key_events/event_sunflower_2014.md`。
|
||||
|
||||
**2014.11 九合一选举**:
|
||||
|
||||
- 国民党大败
|
||||
- 经济整合路线的选举惩罚
|
||||
- 两岸政策从"积极整合"转向"保守防御"
|
||||
|
||||
**2015**:
|
||||
|
||||
- 两岸经贸谈判全面停滞
|
||||
- 台商投资持续但增速放缓
|
||||
- 青年创业成为新的交流形式(但规模有限)
|
||||
@@ -224,26 +238,31 @@
|
||||
### 关键事件
|
||||
|
||||
**2016.05 蔡英文上任**:
|
||||
|
||||
- 承认"九二会谈"但不承认"九二共识"
|
||||
- 北京中断官方沟通机制
|
||||
- 两岸经贸进入"冷对抗"状态
|
||||
|
||||
**2016-2020 新南向政策**:
|
||||
|
||||
- 鼓励台商投资东南亚与南亚
|
||||
- 对大陆出口依存度短暂下降后回升
|
||||
- 投资分散有限——2020 年对大陆投资仍占对外投资 40% 以上
|
||||
|
||||
**2018-2019 美中贸易战**:
|
||||
|
||||
- 台商回流与转单效应
|
||||
- 台商将部分产能移回台湾或转至东南亚
|
||||
- 但高阶制程仍在台湾,中低阶仍在中国大陆
|
||||
|
||||
**2020-2022 疫情与供应链重组**:
|
||||
|
||||
- 台湾对中国大陆出口依存度反而上升(因远程办公需求与大陆复工较快)
|
||||
- 2021 年对大陆出口依存度达 42.3%(历史新高)
|
||||
- 台积电等高阶半导体成为战略资产
|
||||
|
||||
**2022.08 裴洛西访台后大陆暂停部分农产品输入**:
|
||||
|
||||
- 柑橘类、冰鲜白带鱼、冻竹荚鱼等被暂停输入
|
||||
- 目标为台湾中南部农渔业
|
||||
- 经济制裁工具化的初步展现
|
||||
@@ -276,30 +295,36 @@
|
||||
### 关键事件
|
||||
|
||||
**2023.04 贸易壁垒调查**:
|
||||
|
||||
- 北京宣布对台湾 2455 项产品进行贸易壁垒调查
|
||||
- 涵盖农产品、五矿化工、纺织品等
|
||||
- 这是 ECFA 架构外的系统性施压
|
||||
|
||||
**2023.12 ECFA 部分中止**:
|
||||
|
||||
- 北京宣布中止 ECFA 早收清单中 12 项石化产品关税减让
|
||||
- 丙烯、聚碳酸酯等受影响
|
||||
- 明示与台湾选举结果的政治连动
|
||||
|
||||
**2024.01 台湾总统选举**:
|
||||
|
||||
- 赖清德当选
|
||||
- 北京在选举后继续 ECFA 调整
|
||||
|
||||
**2024.05-06 ECFA 第二波中止**:
|
||||
|
||||
- 中止 134 项产品关税减让(机械、纺织、汽车零组件等)
|
||||
- ECFA 早收清单 539 项中已有近 30% 被中止
|
||||
- 同时暂停台湾芒果等农产品输入
|
||||
|
||||
**2024.09-10 进一步施压**:
|
||||
|
||||
- 北京宣布对台 34 项农产品停止零关税
|
||||
- 涵盖水果、蔬菜、水产品
|
||||
- 精准打击中南部农渔业
|
||||
|
||||
**2024-2025 持续的"边打边留"**:
|
||||
|
||||
- 北京未全面废除 ECFA——保留部分杠杆
|
||||
- 高科技产业(半导体)未被直接制裁——避免自我伤害
|
||||
- 传统产业与农业成为主要打击目标
|
||||
@@ -328,14 +353,14 @@
|
||||
|
||||
### 变化一:政经分离的从可能到不可能
|
||||
|
||||
| 时期 | 台北的政经分离策略 | 北京的政经合一策略 | 结果 |
|
||||
|---|---|---|---|
|
||||
| 时期 | 台北的政经分离策略 | 北京的政经合一策略 | 结果 |
|
||||
| --------- | ------------------ | -------------------- | ------------------ |
|
||||
| 1987-1999 | 间接贸易、行政管制 | 默许民间交流以经促统 | 表面分离,实质连结 |
|
||||
| 2000-2007 | 积极开放有效管理 | 吸引台资建立依赖 | 管制失效,依赖加深 |
|
||||
| 2008-2013 | 先经后政 | 让利铺路 | 制度化整合 |
|
||||
| 2014-2015 | 阻断整合 | 让利策略失效 | 分离再确认 |
|
||||
| 2016-2022 | 经贸不扩展、新南向 | 有限度制裁 | 被动脱钩 |
|
||||
| 2023- | 防御性脱钩 | 以经逼政、边打边留 | 强制政经合一 |
|
||||
| 2000-2007 | 积极开放有效管理 | 吸引台资建立依赖 | 管制失效,依赖加深 |
|
||||
| 2008-2013 | 先经后政 | 让利铺路 | 制度化整合 |
|
||||
| 2014-2015 | 阻断整合 | 让利策略失效 | 分离再确认 |
|
||||
| 2016-2022 | 经贸不扩展、新南向 | 有限度制裁 | 被动脱钩 |
|
||||
| 2023- | 防御性脱钩 | 以经逼政、边打边留 | 强制政经合一 |
|
||||
|
||||
**分析含义**:政经分离在两岸关系中**结构性不可能**——因为北京始终将经济作为政治工具,而台北无法单方面维持分离。
|
||||
|
||||
@@ -370,26 +395,31 @@
|
||||
分析两岸经贸关系,**持续观察以下指标**:
|
||||
|
||||
### 指标一:ECFA 存续状态
|
||||
|
||||
- 早收清单被中止的比例
|
||||
- 北京是否全面废除 ECFA
|
||||
- 台北是否主动退出 ECFA
|
||||
|
||||
### 指标二:贸易依存度变化
|
||||
|
||||
- 台湾对大陆(含香港)出口占总出口比例
|
||||
- 关键产业(石化、机械、电子零组件)的出口结构变化
|
||||
- 农产品出口市场的分散程度
|
||||
|
||||
### 指标三:投资流向
|
||||
|
||||
- 台商对大陆投资金额与件数
|
||||
- 台商回台投资金额与产业别
|
||||
- 新南向国家的台商投资趋势
|
||||
|
||||
### 指标四:制裁与反制裁的升级/降级
|
||||
|
||||
- 北京制裁的产业范围与力度
|
||||
- 台北的反制措施(如对大陆产品的限制)
|
||||
- 双方是否在 WTO 框架内解决争端
|
||||
|
||||
### 指标五:高科技供应链的连结与脱钩
|
||||
|
||||
- 半导体设备与技术的出口管制
|
||||
- 台积电等关键企业在两岸的布局
|
||||
- 美国科技管制对两岸经贸的间接影响
|
||||
@@ -399,16 +429,19 @@
|
||||
## 未来观察点(待追踪事件)
|
||||
|
||||
### 2025-2026 ECFA 的最终走向
|
||||
|
||||
- 北京是否进一步中止早收清单项目
|
||||
- ECFA 是否名存实亡或正式终止
|
||||
- 台湾传统产业的转型与淘汰
|
||||
|
||||
### 2028 前后的经贸政策辩论
|
||||
|
||||
- 各候选人的两岸经贸政见
|
||||
- "重新谈判"vs"彻底脱钩"的路线竞争
|
||||
- 北京是否以经济让利介入选举
|
||||
|
||||
### 非红供应链的成型程度
|
||||
|
||||
- 台湾、东南亚、印度的供应链替代能力
|
||||
- 台商在大陆的"在地化"程度(是否可被替代)
|
||||
- 关键原材料的替代来源
|
||||
@@ -502,4 +535,4 @@ ECFA 只是关税减让框架,中止 ECFA 不等于停止贸易——**最惠
|
||||
|
||||
last_updated: 2026-05-08
|
||||
|
||||
**注**:本档案为活档案——需随事件持续更新,特别是 ECFA 调整与供应链重组的动态。
|
||||
**注**:本档案为活档案——需随事件持续更新,特别是 ECFA 调整与供应链重组的动态。
|
||||
@@ -0,0 +1,124 @@
|
||||
# Knowledge Base Routing
|
||||
|
||||
This file is the lightweight session-start routing memory for the policy knowledge base.
|
||||
|
||||
It may be injected at conversation start. It does not replace the caller skill. The caller skill owns the final workflow and output format; this file only points to the smallest useful knowledge entry point.
|
||||
|
||||
## Root
|
||||
|
||||
Knowledge root:
|
||||
|
||||
```text
|
||||
/mnt/knowledge
|
||||
```
|
||||
|
||||
Repository source path:
|
||||
|
||||
```text
|
||||
offline-backend-20260512/backend/knowledge/knowledge_base
|
||||
```
|
||||
|
||||
## Core Principles
|
||||
|
||||
Use the knowledge base as a reusable substrate:
|
||||
|
||||
1. Use this file to choose the first manifest, rule, or routing file.
|
||||
2. Read only the files needed for the current task.
|
||||
3. Do not force the caller skill into a full policy-analysis workflow unless the caller task needs it.
|
||||
4. Prefer manifests before reading many files.
|
||||
5. Follow the caller skill's output structure; use knowledge files only as supporting evidence.
|
||||
|
||||
## Scope
|
||||
|
||||
Use this knowledge base for Taiwan elections, Cross-Strait relations, US/Japan-Taiwan relations, political rhetoric, public-opinion events, policy signals, party dynamics, influence analysis, and scenario planning.
|
||||
|
||||
Do not use it for unrelated technical, lifestyle, economic, legal, medical, or private-person questions. Do not use it to produce political propaganda, campaign messaging, attacks, doxxing, private-communication analysis, or unverified allegations.
|
||||
|
||||
For unclear tasks, ask a focused clarification question before reading many files.
|
||||
|
||||
## Core Rules
|
||||
|
||||
Read these rule files only when needed:
|
||||
|
||||
```text
|
||||
rules/knowledge_usage.md
|
||||
rules/missing_and_placeholder.md
|
||||
rules/citation_and_evidence.md
|
||||
rules/realtime_news_usage.md
|
||||
rules/neutrality_and_perspective.md
|
||||
```
|
||||
|
||||
When to read:
|
||||
|
||||
- `rules/knowledge_usage.md`: before using knowledge files for a non-trivial task, or when unsure how much to read.
|
||||
- `rules/missing_and_placeholder.md`: when a file is missing, marked `[占位]`, or the knowledge base appears incomplete.
|
||||
- `rules/citation_and_evidence.md`: when the answer uses knowledge-base material as evidence.
|
||||
- `rules/realtime_news_usage.md`: when the task needs recent events, news facts, public-opinion developments, or source verification.
|
||||
- `rules/neutrality_and_perspective.md`: when the task involves political controversy, competing claims, attribution, blame, legitimacy, or sensitive issues.
|
||||
|
||||
## Directory Index
|
||||
|
||||
Use `_MANIFEST.md` files as directory-level indexes before reading many files.
|
||||
|
||||
```text
|
||||
entities/_MANIFEST.md # people, parties, institutions, media, foreign actors
|
||||
narratives/_MANIFEST.md # political narratives and discourse frames
|
||||
events/_MANIFEST.md # key events, timelines, recurring patterns
|
||||
primitives/_MANIFEST.md # concepts, frameworks, methods
|
||||
playbooks/_MANIFEST.md # analysis methods
|
||||
reference/_MANIFEST.md # reference tables and standards
|
||||
data_pointers/_MANIFEST.md # external/current material notes
|
||||
routing/_MANIFEST.md # legacy detailed routing files
|
||||
```
|
||||
|
||||
## Route By Task
|
||||
|
||||
Start with these broad entry points, then narrow through manifests:
|
||||
|
||||
```text
|
||||
Background or fact lookup -> reference/_MANIFEST.md, events/_MANIFEST.md, entities/_MANIFEST.md
|
||||
Speech or rhetoric analysis -> playbooks/_MANIFEST.md, narratives/_MANIFEST.md, reference/_MANIFEST.md
|
||||
Cross-Strait signal analysis -> playbooks/_MANIFEST.md, narratives/_MANIFEST.md, events/_MANIFEST.md
|
||||
US/Taiwan/international signal -> playbooks/_MANIFEST.md, narratives/_MANIFEST.md, entities/foreign_actors/_MANIFEST.md
|
||||
Election or party analysis -> playbooks/_MANIFEST.md, entities/_MANIFEST.md, narratives/domestic/_MANIFEST.md
|
||||
Narrative or influence analysis -> playbooks/_MANIFEST.md, narratives/_MANIFEST.md, primitives/_MANIFEST.md
|
||||
Prediction or scenarios -> playbooks/_MANIFEST.md, events/_MANIFEST.md, narratives/_MANIFEST.md
|
||||
```
|
||||
|
||||
Do not use a playbook for simple background lookup unless the user asks for analysis, interpretation, comparison, prediction, or attribution.
|
||||
|
||||
For multi-aspect tasks, read `playbooks/multi_playbook_orchestration.md` only after confirming that one playbook is not enough.
|
||||
|
||||
## Legacy Detailed Routing
|
||||
|
||||
If this lightweight routing is not specific enough, use the legacy routing files:
|
||||
|
||||
```text
|
||||
routing/by_question_type.md
|
||||
routing/by_topic.md
|
||||
routing/by_actor.md
|
||||
routing/by_event.md
|
||||
routing/cross_reference_rules.md
|
||||
```
|
||||
|
||||
These are more detailed and may point to specific entities, narratives, events, primitives, and playbooks. Use them only when manifests are not specific enough.
|
||||
|
||||
## Realtime Or Recent Facts
|
||||
|
||||
The knowledge base may be stale. For recent events, news facts, public-opinion developments, policy updates, or source verification:
|
||||
|
||||
1. Read `rules/realtime_news_usage.md`.
|
||||
2. Use `bm25_search_news` if available.
|
||||
3. Keep knowledge-base background separate from current news evidence.
|
||||
|
||||
## Minimal Use Pattern
|
||||
|
||||
```text
|
||||
1. Identify whether the caller task needs knowledge-base support.
|
||||
2. Read the relevant rule file if needed.
|
||||
3. Use this routing file or a `_MANIFEST.md` to pick a small set of files.
|
||||
4. Read only those files.
|
||||
5. If a playbook is useful, read it; otherwise use knowledge files as supporting material.
|
||||
6. Cite specific files when using knowledge-base material.
|
||||
7. Return control to the caller skill's workflow and output format.
|
||||
```
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
# Citation And Evidence Rules
|
||||
|
||||
Use this file when knowledge-base material supports an answer, analysis, decision, or recommendation.
|
||||
|
||||
## Core Rule
|
||||
|
||||
When using knowledge-base content, cite or name the specific file. Do not write only "the knowledge base says".
|
||||
|
||||
## Evidence Types
|
||||
|
||||
Keep these categories separate:
|
||||
|
||||
- **Knowledge-base background**: local Markdown files under `/mnt/knowledge`.
|
||||
- **Current news evidence**: results from `bm25_search_news` or user-provided current materials.
|
||||
- **User-provided material**: documents, prompts, screenshots, URLs, or files supplied in the conversation.
|
||||
- **Model inference**: reasoning based on the above sources.
|
||||
|
||||
## How To Refer To Knowledge Files
|
||||
|
||||
For concise answers, a file-level reference is enough:
|
||||
|
||||
```markdown
|
||||
依据 `entities/parties/dpp_overview.md` 和 `events/key_events/event_2024_election_three_way.md`,...
|
||||
```
|
||||
|
||||
For higher-stakes or contested claims, be more specific:
|
||||
|
||||
```markdown
|
||||
这一点来自 `playbooks/uncertainty_communication_protocol.md` 中关于信心度表达的规则。
|
||||
```
|
||||
|
||||
## Do Not Over-Cite
|
||||
|
||||
Do not turn every sentence into a citation. Cite when the file materially supports:
|
||||
|
||||
- A factual statement.
|
||||
- A definition.
|
||||
- A timeline point.
|
||||
- A framework choice.
|
||||
- A confidence or uncertainty judgment.
|
||||
- A disputed interpretation.
|
||||
|
||||
## Claims And Support
|
||||
|
||||
Use this discipline:
|
||||
|
||||
```text
|
||||
Fact -> needs a file, news source, or user material.
|
||||
Context -> can use knowledge-base background.
|
||||
Inference -> must be labeled as inference.
|
||||
Forecast -> must include uncertainty.
|
||||
Gap -> must be explicitly marked.
|
||||
```
|
||||
|
||||
## Conflict Handling
|
||||
|
||||
If files conflict:
|
||||
|
||||
1. State the conflict.
|
||||
2. Prefer more specific, more recent, or higher-credibility material.
|
||||
3. If recency matters, use `bm25_search_news`.
|
||||
4. Mark residual uncertainty.
|
||||
|
||||
## Realtime News Links
|
||||
|
||||
For news-search results, use visible Markdown links in the final answer when available. Do not replace inline evidence with a bare source list at the end.
|
||||
@@ -0,0 +1,75 @@
|
||||
# Knowledge Usage Rules
|
||||
|
||||
Use this file when a caller skill needs material from the policy knowledge base.
|
||||
|
||||
## Role Of The Knowledge Base
|
||||
|
||||
The knowledge base is a substrate for other skills. It provides:
|
||||
|
||||
- File navigation.
|
||||
- Background concepts.
|
||||
- Actor, event, narrative, and policy context.
|
||||
- Optional domain playbooks.
|
||||
- Evidence and source pointers.
|
||||
|
||||
It does not automatically own the final answer structure. If the caller skill has its own workflow or output format, follow the caller skill and use knowledge-base files as supporting material.
|
||||
|
||||
## Reading Discipline
|
||||
|
||||
1. Start from `routing.md`, a relevant `_MANIFEST.md`, or a legacy routing file.
|
||||
2. Read only files that are relevant to the current task.
|
||||
3. Do not read the whole knowledge base.
|
||||
4. For large directories, read `_MANIFEST.md` before opening individual files.
|
||||
5. For long files, read the relevant section when possible instead of consuming the entire file.
|
||||
6. Prefer existing local files and local helper APIs over ad hoc guessing.
|
||||
|
||||
## When To Read A Playbook
|
||||
|
||||
Read a playbook when the task needs a domain-specific analysis method, such as:
|
||||
|
||||
- Signal interpretation.
|
||||
- Scenario planning.
|
||||
- Narrative attribution.
|
||||
- Party strategy analysis.
|
||||
- Escalation risk assessment.
|
||||
- Election or voter-flow analysis.
|
||||
|
||||
Do not force a playbook for simple background lookup or when the caller skill already provides a complete method.
|
||||
|
||||
## When To Read Knowledge Files Without A Playbook
|
||||
|
||||
Use knowledge files directly when the caller only needs:
|
||||
|
||||
- A factual background note.
|
||||
- Actor context.
|
||||
- Event timeline context.
|
||||
- A narrative definition.
|
||||
- A concept or framework definition.
|
||||
- A reference table.
|
||||
|
||||
In these cases, read the relevant `_MANIFEST.md` and concrete files, then return the information in the caller skill's format.
|
||||
|
||||
## File Priority
|
||||
|
||||
When unsure, use this order:
|
||||
|
||||
```text
|
||||
routing.md
|
||||
-> rules/*.md as needed
|
||||
-> relevant _MANIFEST.md
|
||||
-> relevant playbook if method is needed
|
||||
-> concrete entities / narratives / events / primitives / reference files
|
||||
```
|
||||
|
||||
## Separation From Current Events
|
||||
|
||||
Knowledge files may be stale. Treat them as background and analytical context. Use `bm25_search_news` or user-provided current materials for recent facts.
|
||||
|
||||
## Output Integration
|
||||
|
||||
When using knowledge-base material:
|
||||
|
||||
- Mention the specific file when it materially supports a claim.
|
||||
- Do not say only "the knowledge base says".
|
||||
- Preserve the caller skill's final format.
|
||||
- Mark uncertainty if the knowledge base does not cover a key point.
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
# Missing And Placeholder Handling
|
||||
|
||||
Use this file when a referenced knowledge file is missing, marked `[占位]`, incomplete, or not specific enough for the task.
|
||||
|
||||
## Core Rule
|
||||
|
||||
Do not invent missing knowledge-base content.
|
||||
|
||||
If a file does not exist or is marked `[占位]`, treat that part of the knowledge base as unavailable.
|
||||
|
||||
## What To Do
|
||||
|
||||
1. Stop trying to repeatedly read the same missing file.
|
||||
2. Check the nearest `_MANIFEST.md` or related routing file for alternatives.
|
||||
3. Use adjacent existing files only when they genuinely cover the issue.
|
||||
4. Clearly state what the knowledge base does not cover.
|
||||
5. Mark uncertainty when the missing material affects the answer.
|
||||
6. If current facts are needed, use `bm25_search_news` or user-provided materials instead of guessing.
|
||||
|
||||
## Recommended Wording
|
||||
|
||||
```markdown
|
||||
知识库中关于【具体议题/人物/事件】的专门文件尚未建设;以下仅基于已存在的相关文件,结论置信度相应降低。
|
||||
```
|
||||
|
||||
or:
|
||||
|
||||
```markdown
|
||||
The knowledge base does not currently cover this point directly. I am using adjacent background files only as context, not as direct evidence.
|
||||
```
|
||||
|
||||
## Do Not
|
||||
|
||||
- Do not fabricate the contents of a missing file.
|
||||
- Do not cite a missing or placeholder file as evidence.
|
||||
- Do not hide the gap if it matters to the conclusion.
|
||||
- Do not treat adjacent files as direct support unless they directly address the point.
|
||||
|
||||
## Fallback Pattern
|
||||
|
||||
```text
|
||||
1. Identify the missing or placeholder file.
|
||||
2. Read the directory _MANIFEST.md.
|
||||
3. Read one or two adjacent files only if relevant.
|
||||
4. Mark the coverage gap.
|
||||
5. Continue with reduced confidence or ask the caller/user for more material.
|
||||
```
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
# Neutrality And Perspective Rules
|
||||
|
||||
Use this file when a task involves political controversy, competing claims, partisan conflict, attribution, blame, legitimacy, public opinion, or sensitive cross-Strait and international issues.
|
||||
|
||||
## Core Rule
|
||||
|
||||
Keep analysis separate from advocacy. Explain positions, incentives, evidence, and uncertainty without endorsing a political side or turning the answer into persuasion, mobilization, attack, or campaign messaging.
|
||||
|
||||
## Neutral Language
|
||||
|
||||
- Use descriptive terms instead of evaluative labels.
|
||||
- Attribute contested claims to the actor making them.
|
||||
- Quote loaded language only when it is evidence; do not adopt it as the analyst's voice.
|
||||
- Avoid moralizing words such as "good", "bad", "correct", "betrayal", "patriotic", or "evil" unless describing a quoted claim.
|
||||
- Do not write slogans, attack lines, campaign copy, or mobilizing calls to action.
|
||||
|
||||
## Multiple Perspectives
|
||||
|
||||
For contested issues, include the materially relevant perspectives needed to understand the dispute. Depending on the task, these may include:
|
||||
|
||||
- Major parties or factions.
|
||||
- Government institutions.
|
||||
- Beijing, Taipei, Washington, Tokyo, or other foreign actors.
|
||||
- Media, civil society, voters, affected communities, or policy experts.
|
||||
|
||||
Do not force artificial balance when evidence quality is uneven. If one side has stronger evidence, say so and explain why.
|
||||
|
||||
## Facts, Interpretations, And Inference
|
||||
|
||||
Keep these layers distinct:
|
||||
|
||||
```text
|
||||
Fact -> what is directly supported by sources.
|
||||
Position -> what an actor says or claims.
|
||||
Interpretation -> how different actors frame the issue.
|
||||
Inference -> the analyst's reasoning from evidence.
|
||||
Forecast -> a forward-looking judgment with uncertainty.
|
||||
```
|
||||
|
||||
Do not present an actor's framing as fact. Do not present inference as direct evidence.
|
||||
|
||||
## Sensitive Claims
|
||||
|
||||
For allegations, attribution, intent, coordination, corruption, foreign influence, or personal conduct:
|
||||
|
||||
1. Require explicit evidence from knowledge files, current news, official records, or user-provided materials.
|
||||
2. Identify who made the claim and what evidence supports it.
|
||||
3. Mark uncertainty when evidence is incomplete, indirect, contested, or outdated.
|
||||
4. Avoid naming private persons unless the task and evidence justify it.
|
||||
|
||||
## Minimal Output Check
|
||||
|
||||
Before answering, check:
|
||||
|
||||
- Are facts, positions, interpretations, and inferences separated?
|
||||
- Are contested claims attributed?
|
||||
- Are materially relevant perspectives included?
|
||||
- Is uncertainty marked where evidence is limited?
|
||||
- Is the answer analytical rather than persuasive or mobilizing?
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
# Realtime News Usage
|
||||
|
||||
Use this file when a task needs recent facts, current public-opinion developments, event timelines, policy updates, or source verification.
|
||||
|
||||
## Core Rule
|
||||
|
||||
The knowledge base is background context. It may be stale. Current facts require current material.
|
||||
|
||||
If `bm25_search_news` is available, use it for:
|
||||
|
||||
- Recent events.
|
||||
- News facts.
|
||||
- Public-opinion controversies.
|
||||
- People or institution updates.
|
||||
- Policy developments.
|
||||
- Source checking.
|
||||
- Timeline gaps.
|
||||
- Conflicting claims.
|
||||
|
||||
## How To Search
|
||||
|
||||
Start broad only when the issue is unclear:
|
||||
|
||||
```text
|
||||
core subject + controversy / latest / recent progress / public opinion
|
||||
```
|
||||
|
||||
Then narrow each follow-up search using concrete entities or clues from previous results:
|
||||
|
||||
- Dates.
|
||||
- Institutions.
|
||||
- People.
|
||||
- Places.
|
||||
- Platform names.
|
||||
- Event names.
|
||||
- Official responses.
|
||||
- Allegations or counter-claims.
|
||||
- Specific policy names.
|
||||
|
||||
Do not repeat broad synonym searches when the first result set already provides bridge terms.
|
||||
|
||||
## Combining News With Knowledge Files
|
||||
|
||||
Use this separation:
|
||||
|
||||
```text
|
||||
Knowledge files -> background, frameworks, prior patterns, actor context.
|
||||
News results -> current facts, dates, latest developments, source evidence.
|
||||
Inference -> your reasoning over both.
|
||||
```
|
||||
|
||||
Do not treat news commentary as knowledge-base truth. Treat it as evidence that still needs source-quality assessment.
|
||||
|
||||
## When To Continue Searching
|
||||
|
||||
Continue searching when:
|
||||
|
||||
- The timeline has gaps.
|
||||
- Key actors are unclear.
|
||||
- Sources conflict.
|
||||
- Only one side is represented.
|
||||
- The result set is old but the user asks about current status.
|
||||
- A claim requires verification.
|
||||
|
||||
## Output
|
||||
|
||||
When using news results:
|
||||
|
||||
- Link facts to visible Markdown links when URLs are available.
|
||||
- Reuse the same link for repeated references to the same article.
|
||||
- If no URL is available, name the title, source, and date.
|
||||
- Keep news evidence separate from knowledge-base file references.
|
||||
+55
-5
@@ -109,20 +109,42 @@ async def _search_notebook(
|
||||
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
|
||||
import httpx # 懒导入:仅在笔记模式下才需要
|
||||
|
||||
# 入参全 dump(token 只打长度,不打明文,防止泄漏到日志)
|
||||
logger.info(
|
||||
"[notebook-search] called keywords=%s search_space_id=%s source_ids=%s token_len=%d",
|
||||
keywords,
|
||||
search_space_id,
|
||||
source_ids,
|
||||
len(token) if token else 0,
|
||||
)
|
||||
|
||||
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
|
||||
if not base:
|
||||
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
|
||||
# 改成 error 级别,warning 在很多部署里默认会被过滤掉看不到
|
||||
logger.error(
|
||||
"[notebook-search] DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search. "
|
||||
"请在后端启动环境里设这个环境变量(指向笔记后端根地址,例如 http://localhost:8003)。"
|
||||
)
|
||||
return []
|
||||
|
||||
full_url = f"{base}{_NOTEBOOK_SEARCH_PATH}"
|
||||
logger.info("[notebook-search] base=%s full_url=%s", base, full_url)
|
||||
|
||||
results: list[Material] = []
|
||||
mat_idx = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
|
||||
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
|
||||
"query": kw,
|
||||
"source_ids": source_ids,
|
||||
"search_space_id": search_space_id,
|
||||
"top_k": _NOTEBOOK_TOP_K,
|
||||
})
|
||||
try:
|
||||
# /mentioned-documents/search 接口无需鉴权,不传 Authorization header
|
||||
resp = await client.post(
|
||||
f"{base}{_NOTEBOOK_SEARCH_PATH}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
full_url,
|
||||
json={
|
||||
"query": kw,
|
||||
"source_ids": source_ids,
|
||||
@@ -130,10 +152,29 @@ async def _search_notebook(
|
||||
"top_k": _NOTEBOOK_TOP_K,
|
||||
},
|
||||
)
|
||||
logger.info(
|
||||
"[notebook-search] kw=%r http_status=%d body_len=%d",
|
||||
kw,
|
||||
resp.status_code,
|
||||
len(resp.content or b""),
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
# 把错误响应体打出来,方便定位 401 / 422 等
|
||||
logger.error(
|
||||
"[notebook-search] kw=%r HTTP %d response: %s",
|
||||
kw,
|
||||
resp.status_code,
|
||||
(resp.text or "")[:1000],
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
# 解析 documents 列表,每个 chunk 作为独立素材
|
||||
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
|
||||
logger.info(
|
||||
"[notebook-search] kw=%r parsed documents=%d",
|
||||
kw,
|
||||
len(documents or []),
|
||||
)
|
||||
for doc in documents or []:
|
||||
doc_title: str = doc.get("title") or "笔记文档"
|
||||
doc_url: str = doc.get("url") or ""
|
||||
@@ -154,10 +195,11 @@ async def _search_notebook(
|
||||
url=doc_url,
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning("notebook search kw=%r failed: %s", kw, e)
|
||||
logger.warning("[notebook-search] kw=%r failed: %s", kw, e, exc_info=True)
|
||||
except Exception as e:
|
||||
logger.warning("notebook search client error: %s", e)
|
||||
logger.warning("[notebook-search] client error: %s", e, exc_info=True)
|
||||
|
||||
logger.info("[notebook-search] done — returning %d materials", len(results))
|
||||
return results
|
||||
|
||||
|
||||
@@ -250,6 +292,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
|
||||
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
|
||||
notebook_materials: list[Material] = []
|
||||
|
||||
if notebook_search_space_id:
|
||||
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
|
||||
notebook_token: str = cfg.get("notebook_token") or ""
|
||||
@@ -369,6 +412,13 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
|
||||
all_results_nested: list = []
|
||||
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
|
||||
elif material_source == "notebook" and notebook_search_space_id:
|
||||
# ── 纯笔记模式 ──
|
||||
# 用户在表单选了「我的空间」作为素材来源 → 跳过 web/intranet 检索,
|
||||
# 仅使用 Step 1.5 已拉取的 notebook_materials。
|
||||
# 与 UI 文案"从空间中检索素材"对齐。
|
||||
_emit(f"仅使用笔记素材,跳过 web/intranet 检索({len(notebook_materials)} 条)")
|
||||
all_results_nested = []
|
||||
else:
|
||||
# ── 常规 web/intranet 模式 ──
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
+39
-7
@@ -20,7 +20,7 @@ from deerflow.agents.middlewares.token_usage_middleware import TokenUsageMiddlew
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
|
||||
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config.agents_config import load_agent_config, validate_agent_id
|
||||
from deerflow.config.agents_config import load_agent_config, load_agent_soul, validate_agent_id
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
@@ -331,6 +331,13 @@ def _build_middlewares(
|
||||
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
|
||||
middlewares.append(SubagentLimitMiddleware(max_concurrent=max_concurrent_subagents))
|
||||
|
||||
# SkillReviewMiddleware — trigger background skill review after each turn.
|
||||
# Gated by skill_evolution.enabled + skill_evolution.review_worker.enabled.
|
||||
if resolved_app_config.skill_evolution.enabled and resolved_app_config.skill_evolution.review_worker.enabled:
|
||||
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
|
||||
|
||||
middlewares.append(SkillReviewMiddleware())
|
||||
|
||||
# LoopDetectionMiddleware — detect and break repetitive tool call loops
|
||||
middlewares.append(LoopDetectionMiddleware())
|
||||
|
||||
@@ -535,11 +542,30 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
|
||||
default_tools = [notebook_search_tool] + default_tools
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
|
||||
tools=default_tools,
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
|
||||
system_prompt=apply_prompt_template(
|
||||
# Custom agents are steered ONLY by their own SOUL.md — no generic
|
||||
# super-agent system template and no personal memory injection. We branch to
|
||||
# the agent-only prompt builder whenever a custom agent config carries a
|
||||
# non-empty SOUL.md. The plain default lead agent (no agent_config) and
|
||||
# notebook mode keep the full template. See apply_agent_only_prompt_template.
|
||||
has_agent_soul = bool(agent_config) and bool(load_agent_soul(agent_id))
|
||||
use_agent_only_prompt = has_agent_soul and not is_notebook_mode
|
||||
|
||||
if use_agent_only_prompt:
|
||||
from deerflow.agents.lead_agent.prompt import apply_agent_only_prompt_template
|
||||
|
||||
system_prompt = apply_agent_only_prompt_template(
|
||||
agent_id=agent_id,
|
||||
agent_name=agent_display_name,
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
available_skills=(set(agent_config.skills or []) if agent_config else None),
|
||||
app_config=resolved_app_config,
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
writing_mode=writing_mode,
|
||||
writing_artifact_path=writing_artifact_path,
|
||||
)
|
||||
else:
|
||||
system_prompt = apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
agent_id=agent_id,
|
||||
@@ -552,6 +578,12 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
),
|
||||
)
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config),
|
||||
tools=default_tools,
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config),
|
||||
system_prompt=system_prompt,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
+138
-22
@@ -615,6 +615,127 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
|
||||
"""
|
||||
|
||||
|
||||
# Minimal system prompt for custom agents that should be steered ONLY by their
|
||||
# own SOUL.md — no generic "super agent" persona scaffolding (role line, thinking
|
||||
# style, clarification system, citation rules, response-style / critical-reminder
|
||||
# defaults) and no personal memory injection. Only the mechanical sections an
|
||||
# agent still needs to actually use its configured skills/tools are kept.
|
||||
AGENT_ONLY_PROMPT_TEMPLATE = """{soul}
|
||||
{network_policy_section}{skills_section}
|
||||
|
||||
{deferred_tools_section}
|
||||
|
||||
{writing_mode_section}
|
||||
|
||||
{subagent_section}
|
||||
|
||||
<working_directory existed="true">
|
||||
- User uploads: `/mnt/user-data/uploads` - Files uploaded by the user (automatically listed in context)
|
||||
- User workspace: `/mnt/user-data/workspace` - Working directory for temporary files
|
||||
- Output files: `/mnt/user-data/outputs` - Final deliverables must be saved here
|
||||
|
||||
**File Management:**
|
||||
- Uploaded files are automatically listed in the <uploaded_files> section before each request
|
||||
- Use `read_file` tool to read uploaded files using their paths from the list
|
||||
- For PDF, PPT, Excel, and Word files, converted Markdown versions (*.md) are available alongside originals
|
||||
- Treat `/mnt/user-data/workspace` as your default current working directory for coding and file-editing tasks
|
||||
- Final deliverables must be copied to `/mnt/user-data/outputs` and presented using `present_files` tool
|
||||
{acp_section}
|
||||
</working_directory>
|
||||
"""
|
||||
|
||||
|
||||
def apply_agent_only_prompt_template(
|
||||
*,
|
||||
agent_id: str | None,
|
||||
agent_name: str | None,
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
rag_mode_enabled: bool = False,
|
||||
writing_mode: bool = False,
|
||||
writing_artifact_path: str | None = None,
|
||||
) -> str:
|
||||
"""Build a custom agent's system prompt from ONLY its SOUL.md plus the
|
||||
mechanical sections required for its configured skills/tools to work.
|
||||
|
||||
Unlike :func:`apply_prompt_template`, this deliberately omits:
|
||||
- the generic ``<role>`` "internal super agent" framing,
|
||||
- ``<thinking_style>`` / ``<clarification_system>`` / ``<response_style>`` /
|
||||
``<citations>`` / ``<critical_reminders>`` behavioural defaults,
|
||||
- the personal ``<memory>`` injection (USER.md / MEMORY.md).
|
||||
|
||||
The agent's SOUL.md is therefore the sole source of persona and behaviour.
|
||||
Skills, deferred tools, the subagent orchestration section (when enabled) and
|
||||
the working-directory contract are kept because they are capability wiring,
|
||||
not persona — dropping them would silently break agents that rely on skills
|
||||
or file output.
|
||||
"""
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
soul = get_agent_soul(storage_agent_id)
|
||||
|
||||
skills_section = get_skills_prompt_section(
|
||||
available_skills,
|
||||
app_config=app_config,
|
||||
display_enabled=rag_mode_enabled,
|
||||
)
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
|
||||
|
||||
n = max_concurrent_subagents
|
||||
subagent_section = _build_subagent_section(n, app_config=app_config) if subagent_enabled else ""
|
||||
|
||||
current_writing_artifact = (
|
||||
f"\n**Current Markdown Artifact**\n- Continue editing this existing Markdown file unless the user explicitly asks for a new document: `{writing_artifact_path}`\n"
|
||||
if writing_artifact_path
|
||||
else ""
|
||||
)
|
||||
writing_mode_section = (
|
||||
f"""<writing_mode>
|
||||
Writing mode is enabled for this run.
|
||||
- Your final deliverable MUST be a Markdown file saved under `/mnt/user-data/outputs/` (e.g. `/mnt/user-data/outputs/report.md`).
|
||||
- If a current Markdown artifact is provided below, revise that file rather than creating a new one unless the user explicitly asks for a separate document.
|
||||
- After writing the file, call `present_files` with the Markdown file path.
|
||||
{current_writing_artifact}</writing_mode>
|
||||
"""
|
||||
if writing_mode
|
||||
else ""
|
||||
)
|
||||
|
||||
acp_section = _build_acp_section(app_config=app_config)
|
||||
custom_mounts_section = _build_custom_mounts_section(app_config=app_config)
|
||||
acp_and_mounts_section = "\n".join(section for section in (acp_section, custom_mounts_section) if section)
|
||||
|
||||
network_policy_section = _build_network_policy_section()
|
||||
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering per search term, tool, or skill.
|
||||
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
|
||||
- Do not output `<skill-display .../>` in RAG/reference mode; the frontend renders the right reference panel.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
prompt = AGENT_ONLY_PROMPT_TEMPLATE.format(
|
||||
soul=soul,
|
||||
skills_section=skills_section,
|
||||
deferred_tools_section=deferred_tools_section,
|
||||
writing_mode_section=writing_mode_section,
|
||||
subagent_section=subagent_section,
|
||||
network_policy_section=network_policy_section,
|
||||
acp_section=acp_and_mounts_section,
|
||||
)
|
||||
|
||||
prompt = prompt + rag_mode_section + f"\n<current_date>{datetime.now().strftime('%Y-%m-%d, %A')}</current_date>"
|
||||
return prompt
|
||||
|
||||
|
||||
def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None, injection_enabled: bool = True) -> str:
|
||||
"""组装注入系统提示词的记忆上下文 (Memory V2)。
|
||||
|
||||
@@ -637,28 +758,23 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
|
||||
from deerflow.config.memory_config import get_effective_memory_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
if not config.enabled or not config.injection.enabled:
|
||||
return ""
|
||||
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=agent_name or "default",
|
||||
config=config,
|
||||
)
|
||||
user_id = get_effective_user_id()
|
||||
config = get_effective_memory_config(user_id)
|
||||
|
||||
manager = build_memory_manager(
|
||||
user_id=user_id,
|
||||
agent_id=agent_name or "default",
|
||||
config=config,
|
||||
app_config=app_config,
|
||||
)
|
||||
# 只取静态块 (builtin 的 frozen snapshot + Hindsight 的模式说明)。
|
||||
# Hindsight 的动态召回需要当前 query,走 MemoryMiddleware.before_model。
|
||||
memory_content = manager.build_system_prompt()
|
||||
|
||||
if not memory_content.strip():
|
||||
return ""
|
||||
|
||||
return f"""<memory>
|
||||
{memory_content}
|
||||
</memory>
|
||||
"""
|
||||
context_content = manager.build_system_prompt()
|
||||
|
||||
if not context_content.strip():
|
||||
return ""
|
||||
|
||||
return f"{context_content}\n"
|
||||
except Exception:
|
||||
logger.exception("Failed to load memory context")
|
||||
return ""
|
||||
@@ -1078,7 +1194,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
|
||||
<notebook_mode>
|
||||
## 角色
|
||||
你是一个笔记本 AI 助手,**只能**基于笔记本文档来源回答用户问题,禁止使用任何其他知识。
|
||||
你是一个空间 AI 助手,**只能**基于空间文档来源回答用户问题,禁止使用任何其他知识。
|
||||
|
||||
## 可用知识来源
|
||||
{source_line}
|
||||
@@ -1104,7 +1220,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- 未调用 `search_notebook_sources` 就直接回答
|
||||
- 引用不在检索结果中的 chunk_id
|
||||
- 省略引用标记
|
||||
- 使用笔记本来源之外的知识
|
||||
- 使用空间来源之外的知识
|
||||
</notebook_mode>"""
|
||||
|
||||
return prompt
|
||||
+20
-5
@@ -32,6 +32,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
#: 内建 provider 的固定名字。
|
||||
BUILTIN_PROVIDER_NAME = "builtin"
|
||||
KNOWLEDGE_PROVIDER_NAME = "knowledge"
|
||||
_INTERNAL_PROVIDER_NAMES = {BUILTIN_PROVIDER_NAME, KNOWLEDGE_PROVIDER_NAME}
|
||||
|
||||
|
||||
class MemoryManager:
|
||||
@@ -49,11 +51,11 @@ class MemoryManager:
|
||||
内建 provider (``name == "builtin"``) 始终接受。外部 provider 至多 1 个,
|
||||
第二个会被拒绝并 log 警告。返回是否注册成功。
|
||||
"""
|
||||
is_builtin = provider.name == BUILTIN_PROVIDER_NAME
|
||||
if not is_builtin:
|
||||
is_internal = provider.name in _INTERNAL_PROVIDER_NAMES
|
||||
if not is_internal:
|
||||
if self._has_external:
|
||||
existing = next(
|
||||
(p.name for p in self._providers if p.name != BUILTIN_PROVIDER_NAME),
|
||||
(p.name for p in self._providers if p.name not in _INTERNAL_PROVIDER_NAMES),
|
||||
"unknown",
|
||||
)
|
||||
logger.warning(
|
||||
@@ -220,6 +222,7 @@ def build_memory_manager(
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
config: Any = None,
|
||||
app_config: Any = None,
|
||||
) -> MemoryManager:
|
||||
"""按生效配置组装并初始化一个 MemoryManager。
|
||||
|
||||
@@ -246,7 +249,9 @@ def build_memory_manager(
|
||||
manager = MemoryManager()
|
||||
base_dir = str(get_paths().base_dir)
|
||||
|
||||
if getattr(config, "version", "v2") == "v1":
|
||||
memory_enabled = bool(getattr(config, "enabled", True) and getattr(config.injection, "enabled", True))
|
||||
|
||||
if memory_enabled and getattr(config, "version", "v2") == "v1":
|
||||
# V1 模式:原版 DeerFlow memory.json + LLM 自动提取
|
||||
from deerflow.agents.memory.providers.v1_builtin import V1BuiltinProvider
|
||||
|
||||
@@ -260,7 +265,7 @@ def build_memory_manager(
|
||||
token_budget=v1_cfg.token_budget,
|
||||
)
|
||||
)
|
||||
else:
|
||||
elif memory_enabled:
|
||||
# V2 模式:USER.md / MEMORY.md + 可选 Hindsight
|
||||
from deerflow.agents.memory.providers.builtin import BuiltinFileProvider
|
||||
|
||||
@@ -286,11 +291,21 @@ def build_memory_manager(
|
||||
except Exception as exc: # noqa: BLE001 - Hindsight 接入失败不能阻断记忆系统
|
||||
logger.warning("加载 Hindsight provider 失败,退回纯 builtin:%s", exc)
|
||||
|
||||
try:
|
||||
from deerflow.agents.memory.providers.knowledge import KnowledgeProvider
|
||||
|
||||
knowledge = KnowledgeProvider(app_config=app_config)
|
||||
if knowledge.is_available():
|
||||
manager.add_provider(knowledge)
|
||||
except Exception as exc: # noqa: BLE001 - 知识库路由失败不能阻断对话
|
||||
logger.warning("加载 Knowledge provider 失败:%s", exc)
|
||||
|
||||
manager.initialize_all(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
base_dir=base_dir,
|
||||
app_config=app_config,
|
||||
)
|
||||
return manager
|
||||
|
||||
|
||||
+4
-1
@@ -227,7 +227,10 @@ class BuiltinFileProvider(MemoryProvider):
|
||||
保持系统提示词稳定,利于 prefix cache。
|
||||
"""
|
||||
blocks = [b for b in (self._snapshot.get("user"), self._snapshot.get("memory")) if b]
|
||||
return "\n\n".join(blocks)
|
||||
content = "\n\n".join(blocks)
|
||||
if not content:
|
||||
return ""
|
||||
return f"<memory>\n{content}\n</memory>"
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
"""builtin 的 ``memory`` 工具以 LangChain BaseTool 形式独立注册,
|
||||
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
"""Read-only provider for the mounted policy knowledge base.
|
||||
|
||||
This provider keeps the current knowledge-base behaviour: inject the lightweight
|
||||
``routing.md`` guide and let the model decide when to read concrete files. It
|
||||
does not automatically retrieve snippets and never writes to the knowledge base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.agents.memory.provider import MemoryProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _read_routing_file(host_path: str) -> str:
|
||||
path = Path(host_path)
|
||||
if not path.is_file():
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
class KnowledgeProvider(MemoryProvider):
|
||||
"""Injects the knowledge routing guide as a read-only context provider."""
|
||||
|
||||
def __init__(self, *, app_config: Any | None = None) -> None:
|
||||
self._app_config = app_config
|
||||
self._routing_host_path: Path | None = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "knowledge"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
thread_id: str | None = None,
|
||||
base_dir: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
app_config = self._app_config or kwargs.get("app_config")
|
||||
if app_config is None:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
app_config = get_app_config()
|
||||
|
||||
self._routing_host_path = None
|
||||
for mount in app_config.sandbox.mounts or []:
|
||||
if mount.container_path.rstrip("/") == "/mnt/knowledge":
|
||||
self._routing_host_path = Path(mount.host_path) / "routing.md"
|
||||
break
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
if self._routing_host_path is None:
|
||||
return ""
|
||||
|
||||
content = _read_routing_file(str(self._routing_host_path.resolve()))
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
logger.debug("Injected knowledge routing memory from %s", self._routing_host_path)
|
||||
return f"""<knowledge_routing_memory source="/mnt/knowledge/routing.md">
|
||||
{content}
|
||||
</knowledge_routing_memory>"""
|
||||
|
||||
def prefetch(self, query: str, *, thread_id: str = "") -> str:
|
||||
return ""
|
||||
|
||||
def sync_turn(
|
||||
self,
|
||||
user_content: str,
|
||||
assistant_content: str,
|
||||
*,
|
||||
thread_id: str = "",
|
||||
turn_index: int | None = None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def get_tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
__all__ = ["KnowledgeProvider"]
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""SkillReviewMiddleware — triggers background skill review after each agent turn.
|
||||
|
||||
Hooks into ``after_agent``: once the lead agent graph has finished a full turn,
|
||||
this middleware counts the tool calls made and, if the threshold is met, hands
|
||||
the conversation messages to the SkillReviewWorker as a background asyncio task.
|
||||
|
||||
No blocking I/O or LLM calls happen in-band — the review runs fully in the
|
||||
background and never delays the HTTP response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.skills.review_worker import maybe_start_review
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillReviewMiddlewareState(AgentState):
|
||||
"""Minimal state schema — only needs ``messages``."""
|
||||
pass
|
||||
|
||||
|
||||
class SkillReviewMiddleware(AgentMiddleware[SkillReviewMiddlewareState]):
|
||||
"""Dispatch a background skill review after every substantial agent turn.
|
||||
|
||||
'Substantial' is defined by ``skill_evolution.review_worker.min_tool_calls``
|
||||
(default 5). Turns below the threshold are silently skipped to avoid
|
||||
wasting LLM tokens on trivial exchanges.
|
||||
"""
|
||||
|
||||
state_schema = SkillReviewMiddlewareState
|
||||
|
||||
@override
|
||||
def after_agent(self, state: Any, runtime: Runtime) -> None:
|
||||
"""Fire background review; never raises — failures are logged only."""
|
||||
try:
|
||||
app_config = get_app_config()
|
||||
messages = state.get("messages", []) if hasattr(state, "get") else []
|
||||
thread_id: str = runtime.context.get("thread_id", "") if runtime.context else ""
|
||||
if not thread_id:
|
||||
return
|
||||
user_id = get_effective_user_id()
|
||||
maybe_start_review(thread_id, messages, app_config, user_id)
|
||||
except Exception:
|
||||
logger.exception("SkillReviewMiddleware.after_agent: unexpected error (ignored)")
|
||||
+193
-45
@@ -1,21 +1,41 @@
|
||||
"""Summarization middleware extensions for DeerFlow."""
|
||||
"""Summarization middleware extensions for DeerFlow.
|
||||
|
||||
Unlike LangChain's stock ``SummarizationMiddleware`` (which runs in
|
||||
``before_model`` and **persists** a destructive ``RemoveMessage(REMOVE_ALL_MESSAGES)``
|
||||
to the checkpoint — wiping the original Q&A from both the model context *and*
|
||||
the conversation the frontend renders), this middleware compresses the history
|
||||
**transiently in ``wrap_model_call``**. The full original messages stay in state
|
||||
(so the UI keeps showing the user's real conversation); only the message list
|
||||
handed to the model for a single call is replaced with ``[summary, *recent]``.
|
||||
|
||||
The generated summary is cached per-thread so we don't pay an extra LLM call on
|
||||
every model invocation: once a thread is summarized, the cached summary is reused
|
||||
until enough *new* messages accumulate to push the compressed context back over
|
||||
the trigger threshold (sticky boundary, mirroring the stock middleware's
|
||||
post-compaction behavior).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Collection
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Collection
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol, override, runtime_checkable
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import SummarizationMiddleware
|
||||
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, RemoveMessage, ToolMessage
|
||||
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, ToolMessage
|
||||
from langgraph.config import get_config
|
||||
from langgraph.graph.message import REMOVE_ALL_MESSAGES
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Upper bound on the per-thread summary cache so a long-lived process serving
|
||||
# many threads can't grow it without limit. Oldest entries are evicted first.
|
||||
_SUMMARY_CACHE_MAX_THREADS = 512
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SummarizationEvent:
|
||||
@@ -95,6 +115,22 @@ class _SkillBundle:
|
||||
skill_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SummaryCacheEntry:
|
||||
"""A cached, reusable summary for one thread.
|
||||
|
||||
``summarized_ids`` are the ids of the original messages folded into
|
||||
``summary_text``. The cache is only reusable while every one of those ids is
|
||||
still present in the live message list and no AIMessage/ToolMessage pair was
|
||||
split when the boundary was drawn (``reusable``) — otherwise filtering by id
|
||||
could orphan a tool result, so we fall back to re-summarizing.
|
||||
"""
|
||||
|
||||
summarized_ids: frozenset[str]
|
||||
summary_text: str
|
||||
reusable: bool
|
||||
|
||||
|
||||
class DeerFlowSummarizationMiddleware(SummarizationMiddleware):
|
||||
"""Summarization middleware with pre-compression hook dispatch and skill rescue."""
|
||||
|
||||
@@ -116,69 +152,181 @@ class DeerFlowSummarizationMiddleware(SummarizationMiddleware):
|
||||
self._preserve_recent_skill_count = max(0, preserve_recent_skill_count)
|
||||
self._preserve_recent_skill_tokens = max(0, preserve_recent_skill_tokens)
|
||||
self._preserve_recent_skill_tokens_per_skill = max(0, preserve_recent_skill_tokens_per_skill)
|
||||
# Per-thread cache of the most recent summary: thread_id -> (summarized message ids, summary text).
|
||||
# Lets us reuse a summary across model calls instead of re-running the LLM every invocation.
|
||||
self._summary_cache: OrderedDict[str, _SummaryCacheEntry] = OrderedDict()
|
||||
|
||||
@override
|
||||
def before_model(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
return self._maybe_summarize(state, runtime)
|
||||
"""Disable the stock middleware's destructive ``before_model`` compaction.
|
||||
|
||||
Compression now happens transiently in :meth:`wrap_model_call`, so the
|
||||
persisted conversation (and the frontend's view of it) is never mutated.
|
||||
"""
|
||||
return None
|
||||
|
||||
@override
|
||||
async def abefore_model(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
return await self._amaybe_summarize(state, runtime)
|
||||
"""Async counterpart of :meth:`before_model` — also a no-op."""
|
||||
return None
|
||||
|
||||
def _maybe_summarize(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
messages = state["messages"]
|
||||
self._ensure_message_ids(messages)
|
||||
# ---- wrap_model_call: transient, display-preserving compression ----------
|
||||
|
||||
total_tokens = self.token_counter(messages)
|
||||
if not self._should_summarize(messages, total_tokens):
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
compressed = self._compressed_messages(request)
|
||||
if compressed is not None:
|
||||
request = request.override(messages=compressed)
|
||||
return handler(request)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
compressed = await self._acompressed_messages(request)
|
||||
if compressed is not None:
|
||||
request = request.override(messages=compressed)
|
||||
return await handler(request)
|
||||
|
||||
def _compressed_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
|
||||
"""Build the compressed message list for this call (sync summary path).
|
||||
|
||||
Returns ``None`` when no compression is needed (the request is passed
|
||||
through untouched).
|
||||
"""
|
||||
plan = self._plan_compression(request)
|
||||
if plan is None:
|
||||
return None
|
||||
|
||||
cutoff_index = self._determine_cutoff_index(messages)
|
||||
if cutoff_index <= 0:
|
||||
return None
|
||||
|
||||
messages_to_summarize, preserved_messages = self._partition_with_skill_rescue(messages, cutoff_index)
|
||||
self._fire_hooks(messages_to_summarize, preserved_messages, runtime)
|
||||
action, payload = plan
|
||||
if action == "reuse":
|
||||
return payload # type: ignore[return-value]
|
||||
# action == "summarize"
|
||||
messages_to_summarize, preserved, thread_id = payload # type: ignore[misc]
|
||||
self._fire_hooks(messages_to_summarize, preserved, getattr(request, "runtime", None))
|
||||
summary = self._create_summary(messages_to_summarize)
|
||||
new_messages = self._build_new_messages(summary)
|
||||
return self._finish_summarize(messages_to_summarize, preserved, summary, thread_id)
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
RemoveMessage(id=REMOVE_ALL_MESSAGES),
|
||||
*new_messages,
|
||||
*preserved_messages,
|
||||
]
|
||||
}
|
||||
async def _acompressed_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
|
||||
"""Async counterpart of :meth:`_compressed_messages`."""
|
||||
plan = self._plan_compression(request)
|
||||
if plan is None:
|
||||
return None
|
||||
action, payload = plan
|
||||
if action == "reuse":
|
||||
return payload # type: ignore[return-value]
|
||||
messages_to_summarize, preserved, thread_id = payload # type: ignore[misc]
|
||||
self._fire_hooks(messages_to_summarize, preserved, getattr(request, "runtime", None))
|
||||
summary = await self._acreate_summary(messages_to_summarize)
|
||||
return self._finish_summarize(messages_to_summarize, preserved, summary, thread_id)
|
||||
|
||||
async def _amaybe_summarize(self, state: AgentState, runtime: Runtime) -> dict | None:
|
||||
messages = state["messages"]
|
||||
def _plan_compression(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
) -> tuple[str, Any] | None:
|
||||
"""Decide what to do for this model call, doing **no** LLM work.
|
||||
|
||||
Returns one of:
|
||||
|
||||
- ``None`` — pass the request through unchanged (no compression needed).
|
||||
- ``("reuse", compressed_messages)`` — reuse the cached summary.
|
||||
- ``("summarize", (messages_to_summarize, preserved, thread_id))`` — the
|
||||
caller must run the (a)sync summarizer, then call
|
||||
:meth:`_finish_summarize`.
|
||||
"""
|
||||
messages = list(getattr(request, "messages", None) or [])
|
||||
if not messages:
|
||||
return None
|
||||
self._ensure_message_ids(messages)
|
||||
thread_id = self._resolve_request_thread_id(request)
|
||||
|
||||
total_tokens = self.token_counter(messages)
|
||||
if not self._should_summarize(messages, total_tokens):
|
||||
cached = self._summary_cache.get(thread_id) if thread_id else None
|
||||
present_ids = {m.id for m in messages}
|
||||
if cached is not None and cached.reusable and cached.summarized_ids <= present_ids:
|
||||
preserved = [m for m in messages if m.id not in cached.summarized_ids]
|
||||
effective = [self._summary_message(cached.summary_text), *preserved]
|
||||
if not self._should_summarize(effective, self.token_counter(effective)):
|
||||
# Refresh LRU recency and reuse without an LLM call.
|
||||
self._summary_cache.move_to_end(thread_id)
|
||||
return ("reuse", effective)
|
||||
# New messages have pushed the compressed context back over the
|
||||
# threshold — fall through and re-summarize a larger prefix.
|
||||
|
||||
if not self._should_summarize(messages, self.token_counter(messages)):
|
||||
return None
|
||||
|
||||
cutoff_index = self._determine_cutoff_index(messages)
|
||||
if cutoff_index <= 0:
|
||||
return None
|
||||
|
||||
messages_to_summarize, preserved_messages = self._partition_with_skill_rescue(messages, cutoff_index)
|
||||
self._fire_hooks(messages_to_summarize, preserved_messages, runtime)
|
||||
summary = await self._acreate_summary(messages_to_summarize)
|
||||
new_messages = self._build_new_messages(summary)
|
||||
messages_to_summarize, preserved = self._partition_with_skill_rescue(messages, cutoff_index)
|
||||
if not messages_to_summarize:
|
||||
return None
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
RemoveMessage(id=REMOVE_ALL_MESSAGES),
|
||||
*new_messages,
|
||||
*preserved_messages,
|
||||
]
|
||||
}
|
||||
return ("summarize", (messages_to_summarize, preserved, thread_id))
|
||||
|
||||
def _finish_summarize(
|
||||
self,
|
||||
messages_to_summarize: list[AnyMessage],
|
||||
preserved: list[AnyMessage],
|
||||
summary: str,
|
||||
thread_id: str | None,
|
||||
) -> list[AnyMessage]:
|
||||
"""Assemble the compressed message list and update the per-thread cache."""
|
||||
compressed = [self._summary_message(summary), *preserved]
|
||||
|
||||
if thread_id:
|
||||
summarized_ids = frozenset(m.id for m in messages_to_summarize if m.id is not None)
|
||||
preserved_ids = {m.id for m in preserved if m.id is not None}
|
||||
# A non-empty overlap means skill rescue split an AIMessage across the
|
||||
# boundary; reconstructing ``preserved`` by id filtering later could
|
||||
# orphan a tool result, so mark the entry non-reusable (re-summarize).
|
||||
reusable = summarized_ids.isdisjoint(preserved_ids)
|
||||
self._store_cache_entry(
|
||||
thread_id,
|
||||
_SummaryCacheEntry(summarized_ids=summarized_ids, summary_text=summary, reusable=reusable),
|
||||
)
|
||||
|
||||
return compressed
|
||||
|
||||
def _store_cache_entry(self, thread_id: str, entry: _SummaryCacheEntry) -> None:
|
||||
"""Insert/refresh a cache entry, evicting the oldest thread if over capacity."""
|
||||
self._summary_cache[thread_id] = entry
|
||||
self._summary_cache.move_to_end(thread_id)
|
||||
while len(self._summary_cache) > _SUMMARY_CACHE_MAX_THREADS:
|
||||
self._summary_cache.popitem(last=False)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_request_thread_id(request: ModelRequest) -> str | None:
|
||||
"""Best-effort thread id from the model request's runtime."""
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
return _resolve_thread_id(runtime)
|
||||
|
||||
def _summary_message(self, summary: str) -> HumanMessage:
|
||||
"""Build the summary message handed to the model.
|
||||
|
||||
It is named ``summary`` and flagged ``hide_from_ui`` defensively. Because
|
||||
it only ever lives in the transient model request (never in persisted
|
||||
state), the frontend never receives it — the user keeps seeing the
|
||||
original conversation.
|
||||
"""
|
||||
return HumanMessage(
|
||||
content=f"Here is a summary of the conversation to date:\n\n{summary}",
|
||||
name="summary",
|
||||
additional_kwargs={"hide_from_ui": True, "lc_source": "summarization"},
|
||||
)
|
||||
|
||||
@override
|
||||
def _build_new_messages(self, summary: str) -> list[HumanMessage]:
|
||||
"""Override the base implementation to let the human message with the special name 'summary'.
|
||||
And this message will be ignored to display in the frontend, but still can be used as context for the model.
|
||||
"""
|
||||
return [HumanMessage(content=f"Here is a summary of the conversation to date:\n\n{summary}", name="summary")]
|
||||
"""Kept for backwards compatibility; delegates to :meth:`_summary_message`."""
|
||||
return [self._summary_message(summary)]
|
||||
|
||||
def _partition_with_skill_rescue(
|
||||
self,
|
||||
|
||||
+1
-1
@@ -107,7 +107,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
|
||||
content=last_message.content,
|
||||
id=last_message.id,
|
||||
name=last_message.name,
|
||||
additional_kwargs={**last_message.additional_kwargs, "run_id": runtime.context.get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
|
||||
additional_kwargs={**last_message.additional_kwargs, "run_id": (runtime.context or {}).get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@@ -7,7 +7,7 @@ import threading
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, model_validator
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
@@ -56,6 +56,32 @@ def validate_agent_name(name: str | None) -> str | None:
|
||||
return validate_agent_id(name)
|
||||
|
||||
|
||||
class CommonQuestion(BaseModel):
|
||||
"""A recommended question shown on the agent chat home / sidebar.
|
||||
|
||||
``title`` is the short label rendered on the question chip; ``prompt`` is the
|
||||
full message actually sent when the user clicks it. Splitting the two lets an
|
||||
admin keep the chip terse while still sending a richly-worded question.
|
||||
|
||||
Legacy configs stored questions as bare strings (content only). The
|
||||
before-validator coerces such a string into ``{title, prompt}`` so old
|
||||
config.yaml files keep loading without a migration — the title falls back to
|
||||
a truncated copy of the content.
|
||||
"""
|
||||
|
||||
title: str = ""
|
||||
prompt: str = ""
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _coerce_legacy_string(cls, value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
title = text if len(text) <= 40 else text[:40].rstrip() + "…"
|
||||
return {"title": title, "prompt": text}
|
||||
return value
|
||||
|
||||
|
||||
class AgentConfig(BaseModel):
|
||||
"""Configuration for a custom agent."""
|
||||
|
||||
@@ -69,6 +95,11 @@ class AgentConfig(BaseModel):
|
||||
# - [] (explicit empty list): no skills
|
||||
# - ["skill1", "skill2"]: load only the specified skills
|
||||
skills: list[str] | None = None
|
||||
# Frequently-asked questions ("常用问题") shown in the agent chat sidebar.
|
||||
# Display/UX metadata only — the runtime never reads these. None = none set.
|
||||
# Accepts both the legacy ``list[str]`` shape and the new ``{title, prompt}``
|
||||
# objects (see CommonQuestion).
|
||||
common_questions: list[CommonQuestion] | None = None
|
||||
|
||||
|
||||
def load_agent_config(agent_id: str | None) -> AgentConfig | None:
|
||||
|
||||
+39
-30
@@ -1,30 +1,39 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class CuratorConfig(BaseModel):
|
||||
"""Configuration for the background skill curator."""
|
||||
|
||||
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
|
||||
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
||||
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
||||
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
|
||||
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
|
||||
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
|
||||
|
||||
|
||||
class SkillEvolutionConfig(BaseModel):
|
||||
"""Configuration for agent-managed skill evolution."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether the agent can create and modify skills under skills/custom.",
|
||||
)
|
||||
curator: CuratorConfig = Field(
|
||||
default_factory=CuratorConfig,
|
||||
description="Configuration for the background skill curator.",
|
||||
)
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ReviewWorkerConfig(BaseModel):
|
||||
"""Configuration for the post-turn background skill review worker."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the review worker is active.")
|
||||
min_tool_calls: int = Field(default=5, description="Minimum tool calls in a turn to trigger review. Matches the prompt's '≥5 steps' hard condition so simple queries never reach the LLM reviewer.")
|
||||
max_messages: int = Field(default=60, description="Max messages passed to the review LLM.")
|
||||
model_name: str | None = Field(default=None, description="Model to use for review; defaults to primary model.")
|
||||
timeout_seconds: int = Field(default=120, description="Overall timeout for a single review run.")
|
||||
|
||||
|
||||
class CuratorConfig(BaseModel):
|
||||
"""Configuration for the background skill curator."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
||||
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
||||
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
||||
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
|
||||
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
|
||||
|
||||
|
||||
class SkillEvolutionConfig(BaseModel):
|
||||
"""Configuration for agent-managed skill evolution."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether the agent can create and modify skills under skills/custom.",
|
||||
)
|
||||
curator: CuratorConfig = Field(
|
||||
default_factory=CuratorConfig,
|
||||
description="Configuration for the background skill curator.",
|
||||
)
|
||||
review_worker: ReviewWorkerConfig = Field(
|
||||
default_factory=ReviewWorkerConfig,
|
||||
description="Configuration for the post-turn background skill review worker.",
|
||||
)
|
||||
+19
-2
@@ -135,8 +135,18 @@ def resolve_user_path(user_id: str) -> Path:
|
||||
return get_paths().user_dir(user_id) / _FILENAME
|
||||
|
||||
|
||||
# Only these curator sub-fields may be overridden per user.
|
||||
# All other fields (enabled, model_name, timeouts, etc.) are global-only.
|
||||
_USER_ALLOWED_CURATOR_KEYS = {"interval_hours", "stale_after_days", "archive_after_days", "model_name"}
|
||||
|
||||
|
||||
def load_user_overrides(user_id: str) -> dict[str, Any] | None:
|
||||
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid)."""
|
||||
"""Read the per-user overrides file for ``user_id`` (None if missing/invalid).
|
||||
|
||||
Only the three curator threshold keys are honoured; any other keys
|
||||
(e.g. legacy ``enabled`` or ``model_name``) are silently stripped so that
|
||||
old files written by previous versions cannot override global-only settings.
|
||||
"""
|
||||
path = resolve_user_path(user_id)
|
||||
if not path.exists():
|
||||
return None
|
||||
@@ -154,7 +164,14 @@ def load_user_overrides(user_id: str) -> dict[str, Any] | None:
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("User skill_evolution overrides at %s must be a JSON object", path)
|
||||
return None
|
||||
return data
|
||||
# Strip top-level keys that are not user-configurable (e.g. legacy "enabled").
|
||||
curator_raw = data.get("curator")
|
||||
if not isinstance(curator_raw, dict):
|
||||
return None
|
||||
filtered_curator = {k: v for k, v in curator_raw.items() if k in _USER_ALLOWED_CURATOR_KEYS}
|
||||
if not filtered_curator:
|
||||
return None
|
||||
return {"curator": filtered_curator}
|
||||
|
||||
|
||||
def save_user_overrides(user_id: str, payload: dict[str, Any]) -> Path:
|
||||
|
||||
@@ -86,6 +86,75 @@ class PrivateSquareSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
#: Id of the always-present built-in default square (used when the admin has
|
||||
#: configured no squares, or when a published item picks none).
|
||||
DEFAULT_SQUARE_ID = "default"
|
||||
|
||||
|
||||
class PublishSquare(BaseModel):
|
||||
"""One admin-configured publish square (发布广场).
|
||||
|
||||
A pure organizational category that published agents/skills can be filed
|
||||
under — unlike the password-gated :class:`PrivateSquareSettings`, a square
|
||||
has **no access control**: every logged-in user can browse every square.
|
||||
Each agent/skill belongs to at most one square (a single ``square_id``
|
||||
field on the row), so an item can never be "published twice".
|
||||
"""
|
||||
|
||||
id: str = Field(..., description="Stable square id (slug/uuid); used as agents.square_id / skills.square_id")
|
||||
name: str = Field(..., description="Display name shown as the square's tab label")
|
||||
description: str = Field(default="", description="Optional description shown in the admin config / tab hint")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class PublishSquaresSettings(BaseModel):
|
||||
"""Admin-configurable list of publish squares.
|
||||
|
||||
``squares`` is display-ordered. ``default_square_id`` selects the square a
|
||||
publisher lands in when they do not pick one explicitly. There is always at
|
||||
least one effective square — :func:`list_publish_squares` injects a built-in
|
||||
"默认广场" when the admin has configured none, so the publish flow always has
|
||||
a valid target.
|
||||
"""
|
||||
|
||||
squares: list[PublishSquare] = Field(default_factory=list, description="Ordered list of configured squares")
|
||||
default_square_id: str = Field(default=DEFAULT_SQUARE_ID, description="Square id used when a publisher picks none")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
def list_publish_squares(settings: PublishSquaresSettings) -> list[PublishSquare]:
|
||||
"""Return the effective square list, always non-empty.
|
||||
|
||||
Falls back to a single built-in default square when the admin has not
|
||||
configured any, so callers never have to special-case the empty state.
|
||||
"""
|
||||
if settings.squares:
|
||||
return list(settings.squares)
|
||||
return [PublishSquare(id=DEFAULT_SQUARE_ID, name="默认广场", description="")]
|
||||
|
||||
|
||||
def resolve_default_square_id(settings: PublishSquaresSettings) -> str:
|
||||
"""Resolve the effective default square id against the configured set."""
|
||||
ids = [s.id for s in settings.squares]
|
||||
if settings.default_square_id and settings.default_square_id in ids:
|
||||
return settings.default_square_id
|
||||
return ids[0] if ids else DEFAULT_SQUARE_ID
|
||||
|
||||
|
||||
def normalize_square_id(square_id: str | None, settings: PublishSquaresSettings) -> str:
|
||||
"""Coerce an arbitrary ``square_id`` to a valid configured square id.
|
||||
|
||||
Empty / unknown / orphaned (square later deleted) ids all collapse to the
|
||||
effective default square, so a published item always resolves to a real
|
||||
tab.
|
||||
"""
|
||||
default_id = resolve_default_square_id(settings)
|
||||
if not square_id:
|
||||
return default_id
|
||||
valid = {s.id for s in list_publish_squares(settings)}
|
||||
return square_id if square_id in valid else default_id
|
||||
|
||||
|
||||
class LeaderboardSettings(BaseModel):
|
||||
"""Admin-configurable defaults for the user leaderboard analytics."""
|
||||
|
||||
@@ -102,6 +171,7 @@ class SystemSettings(BaseModel):
|
||||
|
||||
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
|
||||
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
|
||||
publish_squares: PublishSquaresSettings = Field(default_factory=PublishSquaresSettings)
|
||||
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
|
||||
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
|
||||
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
|
||||
|
||||
@@ -25,10 +25,22 @@ def _vault_key(vault_path: str | None) -> str | None:
|
||||
return key
|
||||
|
||||
|
||||
def normalize_blacklist(labels: list[str] | set[str] | None) -> set[str]:
|
||||
"""Normalize blacklist labels for matching (trimmed, lower-cased, non-empty)."""
|
||||
out: set[str] = set()
|
||||
for raw in labels or []:
|
||||
label = (raw or "").strip().lower()
|
||||
if label:
|
||||
out.add(label)
|
||||
return out
|
||||
|
||||
|
||||
def build_graph(
|
||||
notes: list[dict[str, Any]],
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
*,
|
||||
blacklist: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return ``{nodes, edges}`` from notes (+ optional entities/relations).
|
||||
|
||||
@@ -36,7 +48,20 @@ def build_graph(
|
||||
when ``entities``/``relations`` are supplied (phase 4), entity nodes and
|
||||
note→entity / entity→entity edges are added on top of the wikilink and
|
||||
shared-tag edges.
|
||||
|
||||
``blacklist`` is a set of *normalized* labels (see :func:`normalize_blacklist`);
|
||||
notes whose title and entities whose name match are dropped, together with
|
||||
every edge touching them (edges only join surviving nodes).
|
||||
"""
|
||||
blocked = blacklist or set()
|
||||
|
||||
def _is_blocked(label: str | None) -> bool:
|
||||
return bool(blocked) and (label or "").strip().lower() in blocked
|
||||
|
||||
# Drop blacklisted notes up front so the wikilink / shared-tag edge loops
|
||||
# below never reference them either.
|
||||
notes = [n for n in notes if not _is_blocked(n.get("title"))]
|
||||
|
||||
nodes = []
|
||||
by_key: dict[str, str] = {}
|
||||
note_ids: set[str] = set()
|
||||
@@ -59,6 +84,8 @@ def build_graph(
|
||||
for ent in entities or []:
|
||||
if ent["id"] not in mentioned:
|
||||
continue
|
||||
if _is_blocked(ent.get("name")):
|
||||
continue
|
||||
nodes.append({"id": ent["id"], "type": "entity", "label": ent.get("name") or ent["id"], "category": ent.get("entity_type")})
|
||||
entity_ids.add(ent["id"])
|
||||
|
||||
@@ -135,7 +162,15 @@ def to_cypher(graph: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def to_html(graph: dict[str, Any]) -> str:
|
||||
"""Self-contained force-directed graph (canvas + vanilla JS, no CDN)."""
|
||||
"""Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN).
|
||||
|
||||
The page reads ``?theme=light|dark`` from its own URL so the embedding iframe
|
||||
can stay in sync with the app's day/night mode (an iframe cannot inherit the
|
||||
parent's CSS). The layout settles and then **freezes** so a dense graph stops
|
||||
drifting and stays clickable; node size scales with degree; and hovering or
|
||||
clicking a node focuses it + its direct neighbours while dimming everything
|
||||
else, which cuts through heavy node/edge clutter.
|
||||
"""
|
||||
data_json = json.dumps(graph, ensure_ascii=False)
|
||||
return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json)
|
||||
|
||||
@@ -148,11 +183,12 @@ def export_all(
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
write_html: bool = True,
|
||||
write_json: bool = True,
|
||||
blacklist: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate graph artifacts under ``<vault>/wiki-export/`` and return stats."""
|
||||
out_dir = Path(vault_root).resolve() / "wiki-export"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
graph = build_graph(notes, entities, relations)
|
||||
graph = build_graph(notes, entities, relations, blacklist=blacklist)
|
||||
written: list[str] = []
|
||||
if write_json:
|
||||
(out_dir / "graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
@@ -178,50 +214,250 @@ _HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Knowledge Graph</title>
|
||||
<style>
|
||||
html,body{margin:0;height:100%;background:#0b1020;color:#e7ecff;font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
#hud{position:fixed;top:10px;left:12px;font-size:12px;opacity:.8;z-index:2}
|
||||
#tip{position:fixed;padding:4px 8px;background:#111a36;border:1px solid #2a3a66;border-radius:4px;font-size:12px;pointer-events:none;display:none;z-index:3}
|
||||
canvas{display:block}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.6}
|
||||
:root{
|
||||
--bg:#0b1020; --fg:#e7ecff; --sub:rgba(231,236,255,.78);
|
||||
--panel:rgba(17,26,54,.92); --panel-border:#2a3a66;
|
||||
}
|
||||
html.light{
|
||||
--bg:#f6f8fc; --fg:#1f2433; --sub:rgba(31,36,51,.72);
|
||||
--panel:rgba(255,255,255,.94); --panel-border:#d7deec;
|
||||
}
|
||||
html,body{margin:0;height:100%;background:var(--bg);color:var(--fg);font-family:system-ui,Segoe UI,Arial,sans-serif;overflow:hidden}
|
||||
canvas{display:block;cursor:grab}
|
||||
canvas.grabbing{cursor:grabbing}
|
||||
.card{position:fixed;background:var(--panel);border:1px solid var(--panel-border);border-radius:8px;font-size:12px;z-index:3;backdrop-filter:blur(6px)}
|
||||
#hud{top:10px;left:12px;padding:6px 10px;color:var(--sub);z-index:2}
|
||||
#legend{top:10px;right:12px;padding:8px 10px;display:flex;flex-direction:column;gap:6px}
|
||||
#legend .row{display:flex;align-items:center;gap:6px;color:var(--sub)}
|
||||
#legend .dot{width:10px;height:10px;border-radius:50%}
|
||||
#tip{padding:4px 8px;pointer-events:none;display:none}
|
||||
#panel{left:12px;bottom:12px;max-width:300px;padding:12px 14px;display:none}
|
||||
#panel h3{margin:0 0 6px;font-size:14px;line-height:1.3;word-break:break-word}
|
||||
#panel .meta{color:var(--sub);margin-bottom:8px}
|
||||
#panel .nb{display:flex;flex-wrap:wrap;gap:5px;max-height:120px;overflow:auto}
|
||||
#panel .nb span{padding:2px 7px;border:1px solid var(--panel-border);border-radius:999px;line-height:1.4;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
#panel .close{position:absolute;top:8px;right:10px;cursor:pointer;color:var(--sub);font-size:14px}
|
||||
#reset{bottom:12px;right:12px;padding:7px 12px;cursor:pointer;color:var(--fg);user-select:none}
|
||||
#reset:hover{filter:brightness(1.1)}
|
||||
#empty{position:fixed;inset:0;display:flex;align-items:center;justify-content:center;opacity:.55}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="hud"></div>
|
||||
<div id="tip"></div>
|
||||
<div id="hud" class="card"></div>
|
||||
<div id="legend" class="card">
|
||||
<div class="row"><span class="dot" id="lg-note"></span>知识笔记</div>
|
||||
<div class="row"><span class="dot" id="lg-entity"></span>实体</div>
|
||||
</div>
|
||||
<div id="tip" class="card"></div>
|
||||
<div id="panel" class="card"><span class="close" id="panel-close">×</span><h3></h3><div class="meta"></div><div class="nb"></div></div>
|
||||
<div id="reset" class="card">重置视图</div>
|
||||
<canvas id="c"></canvas>
|
||||
<script>
|
||||
const GRAPH = __GRAPH_DATA__;
|
||||
// Theme is supplied by the embedding iframe (it can't read the parent's CSS).
|
||||
const THEME = (new URLSearchParams(location.search).get('theme') === 'light') ? 'light' : 'dark';
|
||||
const PAL = {
|
||||
dark: {note:'#5b8cff', entity:'#ffb454', label:'rgba(231,236,255,.92)', edge:[150,178,255], halo:'rgba(11,16,32,.55)'},
|
||||
light: {note:'#2f6df6', entity:'#d97706', label:'rgba(31,36,51,.92)', edge:[63,96,168], halo:'rgba(255,255,255,.7)'},
|
||||
}[THEME];
|
||||
document.documentElement.classList.toggle('light', THEME === 'light');
|
||||
document.getElementById('lg-note').style.background = PAL.note;
|
||||
document.getElementById('lg-entity').style.background = PAL.entity;
|
||||
|
||||
const canvas = document.getElementById('c'), ctx = canvas.getContext('2d');
|
||||
const hud = document.getElementById('hud'), tip = document.getElementById('tip');
|
||||
let W=0,H=0;
|
||||
function resize(){W=canvas.width=innerWidth;H=canvas.height=innerHeight;}
|
||||
addEventListener('resize',resize);resize();
|
||||
const panel = document.getElementById('panel');
|
||||
let W=0,H=0,DPR=Math.min(window.devicePixelRatio||1,2);
|
||||
function resize(){W=innerWidth;H=innerHeight;canvas.width=W*DPR;canvas.height=H*DPR;canvas.style.width=W+'px';canvas.style.height=H+'px';}
|
||||
addEventListener('resize',()=>{resize();bump();});resize();
|
||||
|
||||
hud.textContent = GRAPH.nodes.length + ' 个知识节点 · ' + GRAPH.edges.length + ' 条关系';
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id=\"empty\">暂无知识图谱数据</div>');}
|
||||
const idx={};GRAPH.nodes.forEach((n,i)=>{idx[n.id]=n;n.x=W/2+Math.cos(i)*200*Math.random()+(Math.random()-.5)*200;n.y=H/2+Math.sin(i)*200*Math.random()+(Math.random()-.5)*200;n.vx=0;n.vy=0;});
|
||||
const edges=GRAPH.edges.map(e=>({s:idx[e.source],t:idx[e.target],w:e.w||e.weight||1})).filter(e=>e.s&&e.t);
|
||||
let cam={x:0,y:0,k:1},drag=null,pan=null;
|
||||
function step(){
|
||||
for(const n of GRAPH.nodes){for(const m of GRAPH.nodes){if(n===m)continue;let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=1200/d2;n.vx+=dx*f*0.0006;n.vy+=dy*f*0.0006;}}
|
||||
for(const e of edges){let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.sqrt(dx*dx+dy*dy)||1,f=(d-110)*0.01;e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;}
|
||||
for(const n of GRAPH.nodes){n.vx+=(W/2-n.x)*0.0008;n.vy+=(H/2-n.y)*0.0008;n.x+=n.vx*=0.85;n.y+=n.vy*=0.85;}
|
||||
if(!GRAPH.nodes.length){document.body.insertAdjacentHTML('beforeend','<div id="empty">暂无知识图谱数据</div>');}
|
||||
|
||||
// --- index, adjacency, degree, node sizing -------------------------------
|
||||
const idx={}, adj={};
|
||||
GRAPH.nodes.forEach(n=>{idx[n.id]=n;adj[n.id]=new Set();});
|
||||
const edges=[];
|
||||
for(const e of GRAPH.edges){
|
||||
const s=idx[e.source], t=idx[e.target];
|
||||
if(!s||!t||s===t) continue;
|
||||
edges.push({s,t,w:e.weight||e.w||1,label:e.label||''});
|
||||
adj[e.source].add(e.target); adj[e.target].add(e.source);
|
||||
}
|
||||
function draw(){
|
||||
ctx.setTransform(cam.k,0,0,cam.k,cam.x,cam.y);ctx.clearRect(-cam.x/cam.k,-cam.y/cam.k,W/cam.k,H/cam.k);
|
||||
ctx.strokeStyle='rgba(120,150,255,.25)';
|
||||
for(const e of edges){ctx.lineWidth=e.w*0.6;ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();}
|
||||
for(const n of GRAPH.nodes){const r=6;ctx.fillStyle='#5b8cff';ctx.beginPath();ctx.arc(n.x,n.y,r,0,7);ctx.fill();
|
||||
ctx.fillStyle='rgba(231,236,255,.85)';ctx.font='11px system-ui';ctx.fillText((n.label||'').slice(0,18),n.x+9,n.y+3);}
|
||||
const N=GRAPH.nodes.length;
|
||||
GRAPH.nodes.forEach((n,i)=>{
|
||||
n.deg=adj[n.id].size;
|
||||
n.r=4.5+Math.min(11,Math.sqrt(n.deg)*2.4); // hubs are visibly bigger → easier to hit
|
||||
const a=i*2.399963, rad=18*Math.sqrt(i); // phyllotaxis spread = no initial overlap
|
||||
n.x=Math.cos(a)*rad; n.y=Math.sin(a)*rad; n.vx=0; n.vy=0;
|
||||
});
|
||||
// "Hub" nodes keep their label even when zoomed out; the rest declutter.
|
||||
const degs=GRAPH.nodes.map(n=>n.deg).sort((a,b)=>b-a);
|
||||
const HUB=Math.max(3,(degs[Math.floor(N*0.08)]||3));
|
||||
|
||||
// --- force layout that cools down and FREEZES (frozen graph = clickable) --
|
||||
const REP=9000, LINK=92, CENTER=0.0016;
|
||||
let alpha=1, drag=null;
|
||||
function physics(){
|
||||
for(let i=0;i<N;i++){
|
||||
const n=GRAPH.nodes[i];
|
||||
for(let j=i+1;j<N;j++){
|
||||
const m=GRAPH.nodes[j];
|
||||
let dx=n.x-m.x,dy=n.y-m.y,d2=dx*dx+dy*dy+0.01,f=REP/d2;
|
||||
let fx=dx*f,fy=dy*f; n.vx+=fx;n.vy+=fy;m.vx-=fx;m.vy-=fy;
|
||||
}
|
||||
}
|
||||
for(const e of edges){
|
||||
let dx=e.t.x-e.s.x,dy=e.t.y-e.s.y,d=Math.hypot(dx,dy)||1,f=(d-LINK)*0.02;
|
||||
e.s.vx+=dx/d*f;e.s.vy+=dy/d*f;e.t.vx-=dx/d*f;e.t.vy-=dy/d*f;
|
||||
}
|
||||
for(const n of GRAPH.nodes){
|
||||
if(n===drag){n.vx=n.vy=0;continue;}
|
||||
n.vx-=n.x*CENTER; n.vy-=n.y*CENTER;
|
||||
n.x+=(n.vx*=0.82)*alpha; n.y+=(n.vy*=0.82)*alpha;
|
||||
}
|
||||
alpha*=0.985;
|
||||
}
|
||||
function loop(){step();draw();requestAnimationFrame(loop);}loop();
|
||||
|
||||
// --- camera / fit-to-view -------------------------------------------------
|
||||
let cam={x:0,y:0,k:1};
|
||||
function toWorld(x,y){return {x:(x-cam.x)/cam.k,y:(y-cam.y)/cam.k};}
|
||||
canvas.addEventListener('mousedown',ev=>{const p=toWorld(ev.clientX,ev.clientY);let hit=null;
|
||||
for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hit=n;break;}}if(hit)drag=hit;else pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};});
|
||||
addEventListener('mousemove',ev=>{const p=toWorld(ev.clientX,ev.clientY);if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;}else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
let hov=null;for(const n of GRAPH.nodes){if((n.x-p.x)**2+(n.y-p.y)**2<100){hov=n;break;}}
|
||||
if(hov){tip.style.display='block';tip.style.left=(ev.clientX+12)+'px';tip.style.top=(ev.clientY+12)+'px';tip.textContent=hov.label;}else{tip.style.display='none';}});
|
||||
addEventListener('mouseup',()=>{drag=null;pan=null;});
|
||||
canvas.addEventListener('wheel',ev=>{ev.preventDefault();const s=ev.deltaY<0?1.1:0.9;cam.k*=s;cam.x=ev.clientX-(ev.clientX-cam.x)*s;cam.y=ev.clientY-(ev.clientY-cam.y)*s;},{passive:false});
|
||||
function fitView(){
|
||||
if(!N){return;}
|
||||
let a=1e9,b=1e9,c=-1e9,d=-1e9;
|
||||
for(const n of GRAPH.nodes){a=Math.min(a,n.x);b=Math.min(b,n.y);c=Math.max(c,n.x);d=Math.max(d,n.y);}
|
||||
const gw=Math.max(c-a,1),gh=Math.max(d-b,1),pad=90;
|
||||
cam.k=Math.max(0.12,Math.min(2.2,Math.min((W-pad)/gw,(H-pad)/gh)));
|
||||
cam.x=W/2-((a+c)/2)*cam.k; cam.y=H/2-((b+d)/2)*cam.k;
|
||||
}
|
||||
let fitted=false;
|
||||
function bump(){alpha=Math.max(alpha,0.35);} // re-warm the sim on interaction
|
||||
|
||||
// --- focus / hover state --------------------------------------------------
|
||||
let hoverNode=null, focusNode=null;
|
||||
function active(){return focusNode||hoverNode;}
|
||||
function neighbors(n){return n?adj[n.id]:null;}
|
||||
|
||||
function draw(){
|
||||
ctx.setTransform(DPR,0,0,DPR,0,0);
|
||||
ctx.clearRect(0,0,W,H);
|
||||
ctx.translate(cam.x,cam.y); ctx.scale(cam.k,cam.k);
|
||||
const act=active(), nb=neighbors(act), zoomed=cam.k>1.35;
|
||||
const [er,eg,eb]=PAL.edge;
|
||||
|
||||
// edges: clearly visible by default; when a node is focused, only its edges
|
||||
// stay bright and the rest fade back (but never to invisible).
|
||||
for(const e of edges){
|
||||
const lit = act && (e.s===act||e.t===act);
|
||||
const op = act ? (lit?0.98:0.12) : Math.min(0.85,0.55+e.w*0.06);
|
||||
// Keep a stable on-screen thickness regardless of zoom (>=1px), so lines
|
||||
// never thin out to a hairline when zoomed in or on dense graphs.
|
||||
const screenW = lit ? 2.6 : Math.min(2.4,1.4+e.w*0.45);
|
||||
ctx.strokeStyle='rgba('+er+','+eg+','+eb+','+op+')';
|
||||
ctx.lineWidth=Math.max(1,screenW)/cam.k;
|
||||
ctx.beginPath();ctx.moveTo(e.s.x,e.s.y);ctx.lineTo(e.t.x,e.t.y);ctx.stroke();
|
||||
}
|
||||
// nodes
|
||||
for(const n of GRAPH.nodes){
|
||||
const dim = act && n!==act && !(nb&&nb.has(n.id));
|
||||
ctx.globalAlpha = dim?0.22:1;
|
||||
ctx.fillStyle = n.type==='entity'?PAL.entity:PAL.note;
|
||||
ctx.beginPath();ctx.arc(n.x,n.y,n.r,0,6.2832);ctx.fill();
|
||||
if(n===act){ctx.lineWidth=2/cam.k;ctx.strokeStyle=PAL.label;ctx.stroke();}
|
||||
ctx.globalAlpha=1;
|
||||
}
|
||||
// labels: only for focus/neighbors, hubs, or when zoomed in — kills clutter.
|
||||
ctx.textBaseline='middle';
|
||||
for(const n of GRAPH.nodes){
|
||||
const show = act ? (n===act||(nb&&nb.has(n.id))) : (zoomed||n.deg>=HUB);
|
||||
if(!show) continue;
|
||||
const txt=(n.label||'').slice(0,20);
|
||||
if(!txt) continue;
|
||||
ctx.font=(n===act?'bold ':'')+(11/cam.k)+'px system-ui';
|
||||
const tx=n.x+n.r+4/cam.k, ty=n.y;
|
||||
ctx.lineWidth=3/cam.k; ctx.strokeStyle=PAL.halo;
|
||||
ctx.strokeText(txt,tx,ty);
|
||||
ctx.fillStyle=PAL.label; ctx.fillText(txt,tx,ty);
|
||||
}
|
||||
}
|
||||
|
||||
function loop(){
|
||||
if(alpha>0.02 || drag){physics();} else if(!fitted){fitView();fitted=true;}
|
||||
draw();
|
||||
requestAnimationFrame(loop);
|
||||
}
|
||||
// Pre-settle a few iterations so the first paint is already roughly laid out.
|
||||
for(let i=0;i<60;i++)physics();
|
||||
fitView();
|
||||
loop();
|
||||
|
||||
// --- picking & interaction ------------------------------------------------
|
||||
function pick(p){
|
||||
let best=null,bd=1e9;
|
||||
for(const n of GRAPH.nodes){
|
||||
const dx=n.x-p.x,dy=n.y-p.y,d=dx*dx+dy*dy, hit=(n.r+8/cam.k);
|
||||
if(d<hit*hit && d<bd){bd=d;best=n;}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
let pan=null, downXY=null, moved=false;
|
||||
canvas.addEventListener('mousedown',ev=>{
|
||||
const p=toWorld(ev.clientX,ev.clientY), hit=pick(p);
|
||||
downXY={x:ev.clientX,y:ev.clientY,hit}; moved=false;
|
||||
if(hit){drag=hit;} else {pan={x:ev.clientX,y:ev.clientY,cx:cam.x,cy:cam.y};}
|
||||
canvas.classList.add('grabbing');
|
||||
});
|
||||
addEventListener('mousemove',ev=>{
|
||||
const p=toWorld(ev.clientX,ev.clientY);
|
||||
if(downXY && (Math.abs(ev.clientX-downXY.x)>3||Math.abs(ev.clientY-downXY.y)>3)) moved=true;
|
||||
if(drag){drag.x=p.x;drag.y=p.y;drag.vx=drag.vy=0;bump();}
|
||||
else if(pan){cam.x=pan.cx+ev.clientX-pan.x;cam.y=pan.cy+ev.clientY-pan.y;}
|
||||
hoverNode=drag||pan?hoverNode:pick(p);
|
||||
if(hoverNode && !drag && !pan){
|
||||
tip.style.display='block';
|
||||
tip.style.left=Math.min(ev.clientX+14,W-220)+'px';
|
||||
tip.style.top=(ev.clientY+14)+'px';
|
||||
tip.textContent=hoverNode.label||'';
|
||||
} else { tip.style.display='none'; }
|
||||
});
|
||||
addEventListener('mouseup',ev=>{
|
||||
if(downXY && !moved){ // a click, not a drag → toggle focus / detail panel
|
||||
focusNode = (downXY.hit && downXY.hit!==focusNode) ? downXY.hit : null;
|
||||
renderPanel();
|
||||
}
|
||||
drag=null;pan=null;downXY=null;canvas.classList.remove('grabbing');
|
||||
});
|
||||
canvas.addEventListener('wheel',ev=>{
|
||||
ev.preventDefault();
|
||||
const s=ev.deltaY<0?1.12:0.89;
|
||||
const nk=Math.max(0.08,Math.min(6,cam.k*s)), r=nk/cam.k;
|
||||
cam.x=ev.clientX-(ev.clientX-cam.x)*r; cam.y=ev.clientY-(ev.clientY-cam.y)*r; cam.k=nk;
|
||||
},{passive:false});
|
||||
|
||||
// --- detail panel ---------------------------------------------------------
|
||||
const TYPE_LABEL={note:'知识笔记',entity:'实体'};
|
||||
function renderPanel(){
|
||||
if(!focusNode){panel.style.display='none';return;}
|
||||
const n=focusNode;
|
||||
panel.querySelector('h3').textContent=n.label||n.id;
|
||||
const cat=n.category?' · '+n.category:'';
|
||||
panel.querySelector('.meta').textContent=(TYPE_LABEL[n.type]||n.type||'节点')+cat+' · '+n.deg+' 条关系';
|
||||
const nb=panel.querySelector('.nb'); nb.innerHTML='';
|
||||
let count=0;
|
||||
for(const id of adj[n.id]){
|
||||
if(count++>=24)break;
|
||||
const m=idx[id]; if(!m)continue;
|
||||
const s=document.createElement('span'); s.textContent=m.label||id;
|
||||
s.title=m.label||id;
|
||||
s.style.cursor='pointer';
|
||||
s.onclick=()=>{focusNode=m;renderPanel();};
|
||||
nb.appendChild(s);
|
||||
}
|
||||
if(!adj[n.id].size){const s=document.createElement('span');s.textContent='(无关联)';s.style.opacity=.6;nb.appendChild(s);}
|
||||
panel.style.display='block';
|
||||
}
|
||||
document.getElementById('panel-close').onclick=()=>{focusNode=null;renderPanel();};
|
||||
document.getElementById('reset').onclick=()=>{focusNode=null;renderPanel();fitView();};
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
+146
-13
@@ -84,6 +84,46 @@ references(外部来源汇总)、decision(架构/设计决策)、journal
|
||||
}"""
|
||||
|
||||
|
||||
_DOC_SYSTEM_PROMPT = """\
|
||||
你是知识库整理专家,遵循 obsidian-wiki 的 wiki-capture 规范,把一篇**文档**提炼成\
|
||||
**陈述式知识(declarative knowledge)**,而不是照抄原文。
|
||||
|
||||
要求:
|
||||
- 只保留高价值内容:核心结论、关键事实、定义、流程、数据、决策与其依据。
|
||||
- 丢弃:目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。
|
||||
- 用自己的话重组为结构化知识,不要逐段复制原文。
|
||||
- 推断内容用 ^[inferred] 标记,不确定/冲突用 ^[ambiguous] 标记。
|
||||
- 分类 category 取值之一:synthesis(分析/方案/结论)、concepts(概念/框架)、\
|
||||
references(外部来源汇总)、decision(架构/设计决策)、journal(纪要)。
|
||||
- 用中文输出。
|
||||
|
||||
一篇文档可能只讲一个主题,也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识:
|
||||
- 默认只产出 **1 页**;只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。
|
||||
- 每一页聚焦单一主题,标题与 summary 必须彼此区分。
|
||||
- 文档没有可沉淀的知识时返回空的 notes 数组。
|
||||
|
||||
只输出一个 JSON 对象,不要任何额外文字或 Markdown 代码块,结构与对话提炼一致:
|
||||
{
|
||||
"should_ingest": true,
|
||||
"reason": "为什么值得/不值得沉淀",
|
||||
"notes": [
|
||||
{
|
||||
"title": "简洁的知识标题(不超过 40 字)",
|
||||
"summary": "1-2 句话概括这页知识",
|
||||
"category": "synthesis",
|
||||
"context": "文档背景或适用场景,1-3 句",
|
||||
"findings": ["结论/要点(陈述式,可含 ^[inferred])", "..."],
|
||||
"reasoning": "为什么成立、取舍、不确定性",
|
||||
"implications": ["后续建议/风险/下一步", "..."],
|
||||
"tags": ["标签1", "标签2"],
|
||||
"entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}],
|
||||
"relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
for i, t in enumerate(turns, 1):
|
||||
@@ -171,21 +211,27 @@ def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _build_note_draft(
|
||||
note_data: dict[str, Any],
|
||||
*,
|
||||
thread_id: str,
|
||||
thread_title: str | None,
|
||||
turns: list[dict[str, str]],
|
||||
source_type: str,
|
||||
source_id: str | None,
|
||||
source_key: str,
|
||||
fallback_title: str,
|
||||
title_override: str | None,
|
||||
tags: list[str] | None,
|
||||
project_name: str | None,
|
||||
sources: list[SourceDraft],
|
||||
source_key_suffix: str = "",
|
||||
) -> NoteDraft:
|
||||
"""Render one note dict into a :class:`NoteDraft`."""
|
||||
"""Render one note dict into a :class:`NoteDraft`.
|
||||
|
||||
Source-agnostic: the caller supplies ``source_type`` / ``source_id`` /
|
||||
``source_key`` and a ``fallback_title`` (used when neither the user override
|
||||
nor the LLM produced a usable title), so both thread and document capture
|
||||
share one renderer.
|
||||
"""
|
||||
category = str(note_data.get("category") or "synthesis").strip().lower()
|
||||
if category not in _VALID_CATEGORIES:
|
||||
category = "synthesis"
|
||||
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns)
|
||||
final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title
|
||||
summary = str(note_data.get("summary") or "").strip()
|
||||
related = extractor.related_links(project_name)
|
||||
body = _render_body(note_data, final_title, related, sources)
|
||||
@@ -210,15 +256,15 @@ def _build_note_draft(
|
||||
summary=summary or final_title,
|
||||
content_md=body,
|
||||
category=category,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
tags=base_tags,
|
||||
confidence=max(0.0, min(confidence, 1.0)),
|
||||
related=related,
|
||||
sources=sources,
|
||||
entities=[e for e in entities if isinstance(e, dict) and e.get("name")],
|
||||
relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")],
|
||||
source_key=f"thread:{thread_id}{source_key_suffix}",
|
||||
source_key=source_key,
|
||||
)
|
||||
|
||||
|
||||
@@ -293,6 +339,7 @@ async def extract_thread_llm_multi(
|
||||
else:
|
||||
note_dicts = note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = extractor._auto_title(thread_title, turns)
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
# The user-supplied title override only applies to a single-note result;
|
||||
@@ -302,14 +349,100 @@ async def extract_thread_llm_multi(
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
thread_id=thread_id,
|
||||
thread_title=thread_title,
|
||||
turns=turns,
|
||||
source_type="thread",
|
||||
source_id=thread_id,
|
||||
source_key=f"thread:{thread_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
|
||||
async def extract_document_llm_multi(
|
||||
text: str,
|
||||
*,
|
||||
doc_id: str,
|
||||
doc_title: str | None = None,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_name: str | None = "zncm",
|
||||
sources: list[SourceDraft] | None = None,
|
||||
model_name: str | None = None,
|
||||
max_input_chars: int = 12000,
|
||||
allow_multi: bool = True,
|
||||
max_notes: int = 4,
|
||||
) -> list[NoteDraft]:
|
||||
"""Distill an imported **document** into one *or several* knowledge notes.
|
||||
|
||||
Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already
|
||||
converted to Markdown/plain text) instead of a conversation. Returns ``[]``
|
||||
when the LLM path fails (callers fall back to a verbatim note); raises
|
||||
:class:`LLMRejectedIngest` when the model deliberately judges the document
|
||||
worthless. Resulting notes carry ``source_type="file"``.
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
sources = sources or []
|
||||
|
||||
try:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(name=model_name or None, thinking_enabled=False)
|
||||
body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…(已截断)"
|
||||
header = f"文档标题:{doc_title}\n\n" if doc_title else ""
|
||||
prompt = [
|
||||
SystemMessage(content=_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE),
|
||||
HumanMessage(content=f"以下是一篇文档,请按规范提炼为知识 JSON:\n\n{header}{body}"),
|
||||
]
|
||||
response = await model.ainvoke(prompt)
|
||||
raw = response.content if isinstance(response.content, str) else str(response.content)
|
||||
data = _parse_json(raw)
|
||||
except Exception:
|
||||
logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
if not data:
|
||||
logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id)
|
||||
return []
|
||||
|
||||
note_dicts = _notes_from_data(data)
|
||||
if not note_dicts:
|
||||
reason = str(data.get("reason") or "")[:200]
|
||||
if _is_falsey(data.get("should_ingest")):
|
||||
logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason)
|
||||
raise LLMRejectedIngest(reason or "no knowledge value")
|
||||
logger.warning(
|
||||
"[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback",
|
||||
doc_id,
|
||||
data.get("should_ingest"),
|
||||
reason,
|
||||
)
|
||||
return []
|
||||
note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)]
|
||||
|
||||
fallback_title = (doc_title or title or "文档知识").strip() or "文档知识"
|
||||
drafts: list[NoteDraft] = []
|
||||
for i, nd in enumerate(note_dicts):
|
||||
override = title if (len(note_dicts) == 1) else None
|
||||
suffix = "" if len(note_dicts) == 1 else f"#{i}"
|
||||
drafts.append(
|
||||
_build_note_draft(
|
||||
nd,
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
source_key=f"file:{doc_id}{suffix}",
|
||||
fallback_title=fallback_title,
|
||||
title_override=override,
|
||||
tags=tags,
|
||||
project_name=project_name,
|
||||
sources=sources,
|
||||
source_key_suffix=suffix,
|
||||
)
|
||||
)
|
||||
return drafts
|
||||
|
||||
@@ -126,6 +126,21 @@ class KnowledgeEntityRow(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeGraphBlacklistRow(Base):
|
||||
"""Graph blacklist: node labels (entity names / note titles) hidden from the graph.
|
||||
|
||||
Matching is case-insensitive on the trimmed label. Global/shared like the
|
||||
rest of the knowledge base; ``created_by`` is audit-only.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_graph_blacklist"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
label: Mapped[str] = mapped_column(String(512), unique=True, index=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class KnowledgeRelationRow(Base):
|
||||
"""Relation between notes/entities (phase 4, mirrors ``knowledge_relations``)."""
|
||||
|
||||
|
||||
+2
@@ -150,6 +150,7 @@ class ObsidianWikiAdapter:
|
||||
entities: list[dict[str, Any]] | None = None,
|
||||
relations: list[dict[str, Any]] | None = None,
|
||||
updated: str,
|
||||
blacklist: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.ensure_ready()
|
||||
result = exporter.export_all(
|
||||
@@ -159,6 +160,7 @@ class ObsidianWikiAdapter:
|
||||
relations=relations,
|
||||
write_html=self.export_html,
|
||||
write_json=self.export_json,
|
||||
blacklist=blacklist,
|
||||
)
|
||||
self._append_log("EXPORT", "graph", f"nodes={result['stats']['nodes']} edges={result['stats']['edges']}", updated)
|
||||
return result
|
||||
|
||||
@@ -18,6 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from deerflow.knowledge.models import (
|
||||
KnowledgeEmbeddingRow,
|
||||
KnowledgeEntityRow,
|
||||
KnowledgeGraphBlacklistRow,
|
||||
KnowledgeNoteRow,
|
||||
KnowledgeNoteTagRow,
|
||||
KnowledgeNoteVersionRow,
|
||||
@@ -485,6 +486,49 @@ class KnowledgeRepository:
|
||||
rows = (await session.execute(select(KnowledgeEntityRow))).scalars()
|
||||
return [{"id": r.id, "name": r.name, "entity_type": r.entity_type} for r in rows]
|
||||
|
||||
# -- graph blacklist ---------------------------------------------------
|
||||
async def list_graph_blacklist(self) -> list[dict[str, Any]]:
|
||||
"""Return every blacklisted graph node label, newest first."""
|
||||
async with self._sf() as session:
|
||||
stmt = select(KnowledgeGraphBlacklistRow).order_by(KnowledgeGraphBlacklistRow.created_at.desc())
|
||||
rows = (await session.execute(stmt)).scalars()
|
||||
return [{"id": r.id, "label": r.label, "created_by": r.created_by, "created_at": _iso(r.created_at)} for r in rows]
|
||||
|
||||
async def add_graph_blacklist(self, labels: list[str], *, created_by: str | None = None) -> int:
|
||||
"""Add labels to the graph blacklist (trimmed, case-insensitive dedup). Returns the count actually added."""
|
||||
cleaned: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in labels or []:
|
||||
label = (raw or "").strip()
|
||||
if not label or label.lower() in seen:
|
||||
continue
|
||||
seen.add(label.lower())
|
||||
cleaned.append(label)
|
||||
if not cleaned:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
existing = {
|
||||
value
|
||||
for (value,) in (
|
||||
await session.execute(select(func.lower(KnowledgeGraphBlacklistRow.label)).where(func.lower(KnowledgeGraphBlacklistRow.label).in_([label.lower() for label in cleaned])))
|
||||
).all()
|
||||
}
|
||||
added = 0
|
||||
for label in cleaned:
|
||||
if label.lower() in existing:
|
||||
continue
|
||||
session.add(KnowledgeGraphBlacklistRow(id=_new_id("kbblk"), label=label, created_by=created_by, created_at=datetime.now(UTC)))
|
||||
added += 1
|
||||
if added:
|
||||
await session.commit()
|
||||
return added
|
||||
|
||||
async def remove_graph_blacklist(self, entry_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(KnowledgeGraphBlacklistRow).where(KnowledgeGraphBlacklistRow.id == entry_id))
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def list_relations(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(select(KnowledgeRelationRow))).scalars()
|
||||
|
||||
@@ -394,6 +394,129 @@ class KnowledgeService:
|
||||
)
|
||||
return await self._persist_draft(draft, created_by=created_by)
|
||||
|
||||
def _verbatim_file_draft(self, body: str, filename: str, *, title: str | None, tags: list[str] | None, doc_id: str) -> NoteDraft:
|
||||
"""Build a verbatim fallback note from a file when LLM distillation fails.
|
||||
|
||||
Lands as a ``draft`` (like the thread fallback) since it is the raw
|
||||
document, not distilled knowledge.
|
||||
"""
|
||||
base_tags = list(tags or [])
|
||||
for default in (self.project_name, "knowledge-base"):
|
||||
if default and default not in base_tags:
|
||||
base_tags.append(default)
|
||||
name = (title or Path(filename).stem or "文档知识").strip() or "文档知识"
|
||||
summary = next((ln.strip() for ln in body.splitlines() if ln.strip()), name)[:280]
|
||||
return NoteDraft(
|
||||
title=name,
|
||||
summary=summary,
|
||||
content_md=body,
|
||||
category="references",
|
||||
source_type="file",
|
||||
source_id=doc_id,
|
||||
status="draft",
|
||||
tags=base_tags,
|
||||
confidence=0.4,
|
||||
related=extractor.related_links(self.project_name),
|
||||
source_key=f"file:{doc_id}",
|
||||
)
|
||||
|
||||
async def capture_file(
|
||||
self,
|
||||
*,
|
||||
content_md: str,
|
||||
filename: str,
|
||||
title: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str = "approved",
|
||||
created_by: str | None = None,
|
||||
model_name: str | None = None,
|
||||
skip_existing: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Import a file (already converted to Markdown/text) into knowledge notes.
|
||||
|
||||
Runs the same wiki-capture LLM distillation as thread capture, so an
|
||||
uploaded document becomes one or more declarative wiki notes in the vault.
|
||||
Falls back to a verbatim ``draft`` note when the LLM is disabled or returns
|
||||
nothing usable. Returns ``{"note", "created", "notes", "llm_fallback"}``;
|
||||
raises :class:`EmptyThreadError` when there is nothing to keep.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
text = (content_md or "").strip()
|
||||
if not text:
|
||||
raise EmptyThreadError(f"File {filename} is empty")
|
||||
# Strip any frontmatter the document carries; the vault writer adds its own.
|
||||
_, parsed = MarkdownVault.parse(text)
|
||||
body = (parsed or text).strip()
|
||||
if not body:
|
||||
raise EmptyThreadError(f"File {filename} has no body content")
|
||||
# Stable id from name + size so re-importing the same file can be skipped.
|
||||
doc_id = hashlib.sha1(f"{filename}:{len(body)}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
logger.info(
|
||||
"[knowledge] importing file '%s' (chars=%d, doc_id=%s, llm_extract=%s, skip_existing=%s)",
|
||||
filename,
|
||||
len(body),
|
||||
doc_id,
|
||||
self.config.llm_extract_enabled,
|
||||
skip_existing,
|
||||
)
|
||||
|
||||
if skip_existing:
|
||||
existing = await self.repo.find_by_source("file", doc_id)
|
||||
if existing:
|
||||
logger.info("[knowledge] file '%s' already imported (note %s); skipping", filename, existing.get("id"))
|
||||
return {"note": existing, "created": False, "notes": [existing], "llm_fallback": False}
|
||||
|
||||
drafts: list[NoteDraft] = []
|
||||
llm_fallback = False
|
||||
if self.config.llm_extract_enabled:
|
||||
try:
|
||||
drafts = await llm_extractor.extract_document_llm_multi(
|
||||
body,
|
||||
doc_id=doc_id,
|
||||
doc_title=title or Path(filename).stem,
|
||||
title=title,
|
||||
tags=tags,
|
||||
project_name=self.project_name,
|
||||
model_name=model_name or self.config.extract_model_name,
|
||||
max_input_chars=self.config.extract_max_input_chars,
|
||||
allow_multi=self.config.multi_note_enabled,
|
||||
)
|
||||
except llm_extractor.LLMRejectedIngest as exc:
|
||||
raise EmptyThreadError(f"File {filename} judged not worth ingesting: {exc}") from exc
|
||||
if not drafts:
|
||||
llm_fallback = True
|
||||
if not drafts:
|
||||
drafts = [self._verbatim_file_draft(body, filename, title=title, tags=tags, doc_id=doc_id)]
|
||||
|
||||
effective_status = status
|
||||
if llm_fallback and normalize_status(status) == "approved":
|
||||
effective_status = "draft"
|
||||
normalized = normalize_status(effective_status)
|
||||
for draft in drafts:
|
||||
draft.status = normalized
|
||||
|
||||
# Cross-link sibling notes split from the same document (resolved by title
|
||||
# once all are persisted), mirroring multi-note thread capture.
|
||||
if len(drafts) > 1:
|
||||
for i, draft in enumerate(drafts):
|
||||
siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title]
|
||||
if siblings:
|
||||
draft.content_md = _inject_related_notes(draft.content_md, siblings)
|
||||
draft.related = list(dict.fromkeys([*siblings, *draft.related]))
|
||||
|
||||
notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts]
|
||||
logger.info(
|
||||
"[knowledge] file '%s' imported → %d note(s) [%s], status=%s, llm_fallback=%s",
|
||||
filename,
|
||||
len(notes),
|
||||
", ".join(n.get("title", "") for n in notes),
|
||||
normalized,
|
||||
llm_fallback,
|
||||
)
|
||||
return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback}
|
||||
|
||||
# -- CRUD -------------------------------------------------------------
|
||||
async def list_notes(
|
||||
self,
|
||||
@@ -673,7 +796,8 @@ class KnowledgeService:
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso())
|
||||
blacklist = await self._graph_blacklist_set()
|
||||
result = self.adapter.export_graph(notes, entities=entities, relations=relations, updated=now_iso(), blacklist=blacklist)
|
||||
return {"files": result["files"], "stats": result["stats"]}
|
||||
|
||||
async def get_graph(self) -> dict[str, Any]:
|
||||
@@ -683,7 +807,26 @@ class KnowledgeService:
|
||||
notes = await self.repo.list_all()
|
||||
entities = await self.repo.list_entities() if self.config.entity_extraction_enabled else []
|
||||
relations = await self.repo.list_relations() if self.config.entity_extraction_enabled else []
|
||||
return exporter.build_graph(notes, entities, relations)
|
||||
blacklist = await self._graph_blacklist_set()
|
||||
return exporter.build_graph(notes, entities, relations, blacklist=blacklist)
|
||||
|
||||
# -- graph blacklist ---------------------------------------------------
|
||||
async def _graph_blacklist_set(self) -> set[str]:
|
||||
from deerflow.knowledge import exporter
|
||||
|
||||
items = await self.repo.list_graph_blacklist()
|
||||
return exporter.normalize_blacklist([item["label"] for item in items])
|
||||
|
||||
async def list_graph_blacklist(self) -> list[dict[str, Any]]:
|
||||
"""Return blacklist entries (node labels hidden from the graph)."""
|
||||
return await self.repo.list_graph_blacklist()
|
||||
|
||||
async def add_graph_blacklist(self, labels: list[str], *, created_by: str | None = None) -> int:
|
||||
"""Blacklist node labels (entity names / note titles). Returns the count added."""
|
||||
return await self.repo.add_graph_blacklist(labels, created_by=created_by)
|
||||
|
||||
async def remove_graph_blacklist(self, entry_id: str) -> bool:
|
||||
return await self.repo.remove_graph_blacklist(entry_id)
|
||||
|
||||
def export_file_path(self, filename: str) -> Path | None:
|
||||
return self.adapter.export_file_path(filename)
|
||||
@@ -131,6 +131,8 @@ class AgentStore(ABC):
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return one page of agents plus the total matching count.
|
||||
|
||||
|
||||
+16
-3
@@ -61,6 +61,8 @@ class MemoryAgentStore(AgentStore):
|
||||
"description": data.get("description") or "",
|
||||
"user_id": data.get("user_id"),
|
||||
"published": bool(data.get("published", False)),
|
||||
"is_private": bool(data.get("is_private", False)),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"featured_order": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
@@ -93,7 +95,7 @@ class MemoryAgentStore(AgentStore):
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
@@ -103,7 +105,7 @@ class MemoryAgentStore(AgentStore):
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None or row.get("user_id") is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
@@ -159,7 +161,7 @@ class MemoryAgentStore(AgentStore):
|
||||
row = self._agents.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
row[key] = data[key]
|
||||
row["updated_at"] = _now()
|
||||
@@ -198,16 +200,27 @@ class MemoryAgentStore(AgentStore):
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
|
||||
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
|
||||
owner_key = owner.strip() if owner and owner.strip() else None
|
||||
square_key = square_id.strip() if square_id and square_id.strip() else None
|
||||
|
||||
def _square_ok(a: dict[str, Any]) -> bool:
|
||||
if square_key is None:
|
||||
return True
|
||||
sid = str(a.get("square_id") or "")
|
||||
return sid == square_key or (square_is_default and sid == "")
|
||||
|
||||
matched = [
|
||||
a
|
||||
for a in self._agents.values()
|
||||
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
|
||||
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
|
||||
and (owner_key is None or a.get("user_id") == owner_key)
|
||||
and _square_ok(a)
|
||||
]
|
||||
if sort == "time_asc":
|
||||
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
|
||||
|
||||
+23
-17
@@ -27,6 +27,12 @@ class AgentRow(Base):
|
||||
# visible to its owner in the "我的" tab, and visible to anyone in the
|
||||
# private-square listing once the password has been verified.
|
||||
is_private: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
# Publish square (发布广场) this agent is filed under. A pure organizational
|
||||
# category with no access control — independent of ``published`` /
|
||||
# ``is_private``. Empty string = "not assigned" → the listing layer treats
|
||||
# it as the admin-configured default square. Each agent belongs to at most
|
||||
# one square (single column), so it can never be published to two squares.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
|
||||
# NULL = not featured on the homepage selector. Non-NULL = admin-curated
|
||||
# featured rank (lower sorts first). Only ``published=True`` agents are
|
||||
# eligible; the router enforces that invariant on writes. BIGINT so the
|
||||
@@ -37,7 +43,7 @@ class AgentRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentFavoriteRow(Base):
|
||||
class AgentFavoriteRow(Base):
|
||||
"""A user's bookmark of a (typically public-square) agent.
|
||||
|
||||
Lets a user collect agents from the 广场 (square) into their "我的" tab
|
||||
@@ -51,22 +57,22 @@ class AgentFavoriteRow(Base):
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentPinRow(Base):
|
||||
"""A per-user pin that keeps an agent near the top of that user's lists."""
|
||||
|
||||
__tablename__ = "agent_pins"
|
||||
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentSkillRow(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentPinRow(Base):
|
||||
"""A per-user pin that keeps an agent near the top of that user's lists."""
|
||||
|
||||
__tablename__ = "agent_pins"
|
||||
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class AgentSkillRow(Base):
|
||||
"""Denormalized cache of a custom agent's selected skills.
|
||||
|
||||
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
|
||||
|
||||
+14
-3
@@ -93,6 +93,7 @@ class AgentRepository(AgentStore):
|
||||
user_id=data.get("user_id"),
|
||||
published=bool(data.get("published", False)),
|
||||
is_private=bool(data.get("is_private", False)),
|
||||
square_id=str(data.get("square_id") or ""),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -131,7 +132,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -144,7 +145,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None or row.user_id is not None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -224,7 +225,7 @@ class AgentRepository(AgentStore):
|
||||
row = await session.get(AgentRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in ("name", "description", "published", "is_private"):
|
||||
for key in ("name", "description", "published", "is_private", "square_id"):
|
||||
if key in data:
|
||||
setattr(row, key, data[key])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
@@ -290,6 +291,8 @@ class AgentRepository(AgentStore):
|
||||
admin: bool = False,
|
||||
sort: str = "time_desc",
|
||||
owner: str | None = None,
|
||||
square_id: str | None = None,
|
||||
square_is_default: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
filters = []
|
||||
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
|
||||
@@ -302,6 +305,14 @@ class AgentRepository(AgentStore):
|
||||
if owner and owner.strip():
|
||||
# Filter to one publisher. Built-in agents (user_id IS NULL) never match.
|
||||
filters.append(AgentRow.user_id == owner.strip())
|
||||
if square_id and square_id.strip():
|
||||
sid = square_id.strip()
|
||||
if square_is_default:
|
||||
# Legacy / unassigned rows (square_id == "") belong to the
|
||||
# default square, so include them alongside explicit matches.
|
||||
filters.append(or_(AgentRow.square_id == sid, AgentRow.square_id == ""))
|
||||
else:
|
||||
filters.append(AgentRow.square_id == sid)
|
||||
|
||||
count_stmt = select(func.count()).select_from(AgentRow)
|
||||
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Add ``agents.square_id`` and ``skills.square_id`` for the publish-square (发布广场) feature.
|
||||
|
||||
A nullable-default (server_default '') string column tagging which admin-
|
||||
configured publish square an agent / skill is filed under. Pure organizational
|
||||
category — independent of the ``published`` / ``is_private`` flags. Empty string
|
||||
means "unassigned" and is resolved to the default square at read time.
|
||||
|
||||
Revision ID: 20260611_01
|
||||
Revises: 20260609_02
|
||||
Create Date: 2026-06-11
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260611_01"
|
||||
down_revision = "20260609_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def _add_square_id(table_name: str, index_name: str) -> None:
|
||||
if _has_table(table_name) and not _has_column(table_name, "square_id"):
|
||||
op.add_column(
|
||||
table_name,
|
||||
sa.Column(
|
||||
"square_id",
|
||||
sa.String(length=64),
|
||||
nullable=False,
|
||||
server_default="",
|
||||
),
|
||||
)
|
||||
if _has_table(table_name) and not _has_index(table_name, index_name):
|
||||
op.create_index(index_name, table_name, ["square_id"])
|
||||
|
||||
|
||||
def _drop_square_id(table_name: str, index_name: str) -> None:
|
||||
if _has_index(table_name, index_name):
|
||||
op.drop_index(index_name, table_name=table_name)
|
||||
if _has_column(table_name, "square_id"):
|
||||
op.drop_column(table_name, "square_id")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_add_square_id("agents", "ix_agents_square_id")
|
||||
_add_square_id("skills", "ix_skills_square_id")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
_drop_square_id("skills", "ix_skills_square_id")
|
||||
_drop_square_id("agents", "ix_agents_square_id")
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Add task_id column to roundtable_drafts.
|
||||
|
||||
External task id (passed by the wujie-embedded roundtable drawer). When set, the
|
||||
draft is keyed/shared by ``task_id`` and is no longer partitioned by ``user_id``
|
||||
(read/write/delete ownership checks are relaxed for task-bound drafts — see
|
||||
``sql.py``); when NULL the original per-user isolation is preserved.
|
||||
``create_all`` adds the column for brand-new DBs; this revision patches DBs that
|
||||
already have the table.
|
||||
|
||||
Revision ID: 20260611_01
|
||||
Revises: 20260609_02
|
||||
Create Date: 2026-06-11
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260611_01"
|
||||
down_revision = "20260609_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("roundtable_drafts") and not _has_column("roundtable_drafts", "task_id"):
|
||||
op.add_column("roundtable_drafts", sa.Column("task_id", sa.String(128), nullable=True))
|
||||
op.create_index("ix_roundtable_drafts_task_id", "roundtable_drafts", ["task_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_column("roundtable_drafts", "task_id"):
|
||||
op.drop_index("ix_roundtable_drafts_task_id", table_name="roundtable_drafts")
|
||||
op.drop_column("roundtable_drafts", "task_id")
|
||||
+2
@@ -28,6 +28,7 @@ from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.knowledge.models import (
|
||||
KnowledgeEmbeddingRow,
|
||||
KnowledgeEntityRow,
|
||||
KnowledgeGraphBlacklistRow,
|
||||
KnowledgeNoteRow,
|
||||
KnowledgeNoteTagRow,
|
||||
KnowledgeNoteVersionRow,
|
||||
@@ -75,6 +76,7 @@ __all__ = [
|
||||
"BusinessMappingRow",
|
||||
"KnowledgeEmbeddingRow",
|
||||
"KnowledgeEntityRow",
|
||||
"KnowledgeGraphBlacklistRow",
|
||||
"KnowledgeNoteRow",
|
||||
"KnowledgeNoteTagRow",
|
||||
"KnowledgeNoteVersionRow",
|
||||
|
||||
+3
@@ -26,6 +26,9 @@ class RoundtableDraftRow(Base):
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# 外部任务 id(无界嵌入抽屉传入)。非空时该草稿按 task_id 共享、不再按 user_id 分权
|
||||
# (读/写/删放开归属校验,见 sql.py);为空时保持原「按登录用户隔离」语义。
|
||||
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
# Auto-derived task summary (objective → first user message → timestamp).
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# Furthest step reached when this snapshot was taken (1 | 2 | 3).
|
||||
|
||||
+24
-3
@@ -53,6 +53,7 @@ class RoundtableDraftRepository:
|
||||
"""Lightweight projection for the history dropdown (no big blobs)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"task_id": row.task_id,
|
||||
"title": row.title,
|
||||
"furthest_step": row.furthest_step,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
@@ -81,15 +82,30 @@ class RoundtableDraftRepository:
|
||||
async def get_draft(self, draft_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableDraftRow, draft_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
# task 绑定的草稿共享:放开 user 归属校验(不分权);非 task 草稿仍按 user 隔离。
|
||||
if row is None or (row.user_id != user_id and not row.task_id):
|
||||
return None
|
||||
return self._draft_to_dict(row)
|
||||
|
||||
async def get_latest_draft_by_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""按外部任务 id 取最新一条草稿(**忽略 user_id**,不分权)。无则 None。"""
|
||||
stmt = (
|
||||
select(RoundtableDraftRow)
|
||||
.where(RoundtableDraftRow.task_id == task_id)
|
||||
.order_by(RoundtableDraftRow.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalars().first()
|
||||
return self._draft_to_dict(row) if row is not None else None
|
||||
|
||||
async def create_draft(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = RoundtableDraftRow(
|
||||
id=data["id"],
|
||||
user_id=user_id,
|
||||
task_id=(data.get("task_id") or None),
|
||||
title=(data.get("title") or "").strip(),
|
||||
furthest_step=int(data.get("furthest_step") or 1),
|
||||
step1=_dumps(data.get("step1")),
|
||||
@@ -114,11 +130,15 @@ class RoundtableDraftRepository:
|
||||
step1=_UNSET,
|
||||
step2=_UNSET,
|
||||
step3=_UNSET,
|
||||
task_id=_UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableDraftRow, draft_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
# task 绑定的草稿共享:放开 user 归属校验(不分权)。
|
||||
if row is None or (row.user_id != user_id and not row.task_id):
|
||||
return None
|
||||
if task_id is not _UNSET and task_id is not None:
|
||||
row.task_id = str(task_id)
|
||||
if title is not _UNSET and title is not None:
|
||||
row.title = str(title).strip()
|
||||
if furthest_step is not _UNSET and furthest_step is not None:
|
||||
@@ -137,7 +157,8 @@ class RoundtableDraftRepository:
|
||||
async def delete_draft(self, draft_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RoundtableDraftRow, draft_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
# task 绑定的草稿共享:放开 user 归属校验(不分权)。
|
||||
if row is None or (row.user_id != user_id and not row.task_id):
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
|
||||
+5
@@ -71,6 +71,7 @@ class MemorySkillStore(SkillStore):
|
||||
"published": bool(data.get("published", False)),
|
||||
"detail": data.get("detail"),
|
||||
"name_zh": data.get("name_zh"),
|
||||
"square_id": str(data.get("square_id") or ""),
|
||||
"created_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
@@ -106,6 +107,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["detail"] = data["detail"]
|
||||
if "name_zh" in data:
|
||||
record["name_zh"] = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
record["square_id"] = str(data["square_id"] or "")
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
@@ -121,6 +124,8 @@ class MemorySkillStore(SkillStore):
|
||||
record["detail"] = data["detail"]
|
||||
if "name_zh" in data:
|
||||
record["name_zh"] = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
record["square_id"] = str(data["square_id"] or "")
|
||||
record["updated_at"] = _now()
|
||||
return _with_derived(_copy(record)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@@ -33,6 +33,10 @@ class SkillRow(Base):
|
||||
# the on-disk SKILL.md) so it can be set for built-in skills too, and so the
|
||||
# shared source file is never rewritten.
|
||||
name_zh: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
|
||||
# Publish square (发布广场) this skill is filed under once published. Pure
|
||||
# organizational category, no access control. Empty string = unassigned →
|
||||
# the listing layer resolves it to the default square. One square per skill.
|
||||
square_id: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ class SkillRepository(SkillStore):
|
||||
published=bool(data.get("published", False)),
|
||||
detail=data.get("detail"),
|
||||
name_zh=data.get("name_zh"),
|
||||
square_id=str(data.get("square_id") or ""),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -109,6 +110,8 @@ class SkillRepository(SkillStore):
|
||||
row.detail = data["detail"]
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
row.square_id = str(data["square_id"] or "")
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
@@ -127,6 +130,8 @@ class SkillRepository(SkillStore):
|
||||
row.detail = data["detail"]
|
||||
if "name_zh" in data:
|
||||
row.name_zh = data["name_zh"]
|
||||
if "square_id" in data:
|
||||
row.square_id = str(data["square_id"] or "")
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
+27
@@ -45,7 +45,34 @@ class ThreadMetaStore(abc.ABC):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Search threads ordered by ``updated_at`` desc.
|
||||
|
||||
``exclude_system=True`` drops threads whose ``metadata.system`` is
|
||||
truthy (scheduler runs, roundtable workers, …) so user-facing
|
||||
conversation lists are not crowded out by machine-created threads.
|
||||
``query`` is a case-insensitive substring match on the thread title
|
||||
(``display_name``). ``limit``/``offset`` apply to the *filtered*
|
||||
result set.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def count(
|
||||
self,
|
||||
*,
|
||||
metadata: dict | None = None,
|
||||
status: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> int:
|
||||
"""Count threads matching the same filters as :meth:`search`.
|
||||
|
||||
Backs pagination UIs that need a total alongside per-page results.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
|
||||
+85
-7
@@ -74,6 +74,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.search")
|
||||
filter_dict: dict[str, Any] = {}
|
||||
@@ -84,13 +86,89 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
if resolved_user_id is not None:
|
||||
filter_dict["user_id"] = resolved_user_id
|
||||
|
||||
items = await self._store.asearch(
|
||||
THREADS_NS,
|
||||
filter=filter_dict or None,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return [self._item_to_dict(item) for item in items]
|
||||
normalized_query = (query or "").strip().lower()
|
||||
if not exclude_system and not normalized_query:
|
||||
items = await self._store.asearch(
|
||||
THREADS_NS,
|
||||
filter=filter_dict or None,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return [self._item_to_dict(item) for item in items]
|
||||
|
||||
# The Store filter only supports equality, so the system-thread
|
||||
# exclusion and title search are applied in Python: scan in batches
|
||||
# until the requested window is full so ``limit``/``offset`` refer to
|
||||
# filtered rows.
|
||||
def _matches(record: dict) -> bool:
|
||||
if exclude_system and (record.get("metadata") or {}).get("system"):
|
||||
return False
|
||||
if normalized_query and normalized_query not in (record.get("display_name") or "").lower():
|
||||
return False
|
||||
return True
|
||||
|
||||
needed = offset + limit
|
||||
batch_size = max(500, needed)
|
||||
matched: list[dict] = []
|
||||
scan_offset = 0
|
||||
while len(matched) < needed:
|
||||
items = await self._store.asearch(
|
||||
THREADS_NS,
|
||||
filter=filter_dict or None,
|
||||
limit=batch_size,
|
||||
offset=scan_offset,
|
||||
)
|
||||
if not items:
|
||||
break
|
||||
matched.extend(record for record in (self._item_to_dict(item) for item in items) if _matches(record))
|
||||
if len(items) < batch_size:
|
||||
break
|
||||
scan_offset += len(items)
|
||||
return matched[offset:needed]
|
||||
|
||||
async def count(
|
||||
self,
|
||||
*,
|
||||
metadata: dict | None = None,
|
||||
status: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> int:
|
||||
"""Count threads matching the same filters as :meth:`search`."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.count")
|
||||
filter_dict: dict[str, Any] = {}
|
||||
if metadata:
|
||||
filter_dict.update(metadata)
|
||||
if status:
|
||||
filter_dict["status"] = status
|
||||
if resolved_user_id is not None:
|
||||
filter_dict["user_id"] = resolved_user_id
|
||||
|
||||
normalized_query = (query or "").strip().lower()
|
||||
total = 0
|
||||
batch_size = 500
|
||||
scan_offset = 0
|
||||
while True:
|
||||
items = await self._store.asearch(
|
||||
THREADS_NS,
|
||||
filter=filter_dict or None,
|
||||
limit=batch_size,
|
||||
offset=scan_offset,
|
||||
)
|
||||
if not items:
|
||||
break
|
||||
for item in items:
|
||||
record = self._item_to_dict(item)
|
||||
if exclude_system and (record.get("metadata") or {}).get("system"):
|
||||
continue
|
||||
if normalized_query and normalized_query not in (record.get("display_name") or "").lower():
|
||||
continue
|
||||
total += 1
|
||||
if len(items) < batch_size:
|
||||
break
|
||||
scan_offset += len(items)
|
||||
return total
|
||||
|
||||
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False) -> bool:
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
|
||||
+94
-13
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore
|
||||
@@ -109,6 +109,8 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Search threads with optional metadata and status filters.
|
||||
|
||||
@@ -116,28 +118,107 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
context. Pass ``user_id=None`` to bypass (migration/CLI).
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
|
||||
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc())
|
||||
# thread_id tie-breaker keeps OFFSET pagination stable when many rows
|
||||
# share the same updated_at (bulk imports, scheduler bursts).
|
||||
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc(), ThreadMetaRow.thread_id.desc())
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
stmt = stmt.where(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
# Case-insensitive title search. display_name is a real column, so
|
||||
# this filter runs in SQL even when the JSON filters below fall
|
||||
# back to Python-side scanning.
|
||||
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
if metadata:
|
||||
# When metadata filter is active, fetch a larger window and filter
|
||||
# in Python. TODO(Phase 2): use JSON DB operators (Postgres @>,
|
||||
# SQLite json_extract) for server-side filtering.
|
||||
stmt = stmt.limit(limit * 5 + offset)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
rows = [self._row_to_dict(r) for r in result.scalars()]
|
||||
rows = [r for r in rows if all(r.get("metadata", {}).get(k) == v for k, v in metadata.items())]
|
||||
return rows[offset : offset + limit]
|
||||
else:
|
||||
if not metadata and not exclude_system:
|
||||
stmt = stmt.limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
# Metadata filters live inside the JSON blob, so rows are scanned in
|
||||
# batches and filtered in Python until the requested window is full.
|
||||
# A fixed over-fetch window would under-fill pages when most rows are
|
||||
# filtered out (e.g. heavy scheduled-task usage), making callers that
|
||||
# paginate believe the list is exhausted. TODO(Phase 2): use JSON DB
|
||||
# operators (Postgres @>, SQLite json_extract) for server-side filtering.
|
||||
def _matches(row: dict) -> bool:
|
||||
md = row.get("metadata") or {}
|
||||
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
|
||||
return False
|
||||
if exclude_system and md.get("system"):
|
||||
return False
|
||||
return True
|
||||
|
||||
needed = offset + limit
|
||||
batch_size = max(500, needed)
|
||||
matched: list[dict] = []
|
||||
scan_offset = 0
|
||||
async with self._sf() as session:
|
||||
while len(matched) < needed:
|
||||
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
|
||||
rows = result.scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
matched.extend(d for d in (self._row_to_dict(r) for r in rows) if _matches(d))
|
||||
if len(rows) < batch_size:
|
||||
break
|
||||
scan_offset += len(rows)
|
||||
return matched[offset:needed]
|
||||
|
||||
async def count(
|
||||
self,
|
||||
*,
|
||||
metadata: dict | None = None,
|
||||
status: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
exclude_system: bool = False,
|
||||
query: str | None = None,
|
||||
) -> int:
|
||||
"""Count threads matching the same filters as :meth:`search`."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.count")
|
||||
conditions = []
|
||||
if resolved_user_id is not None:
|
||||
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
conditions.append(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
if not metadata and not exclude_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
async with self._sf() as session:
|
||||
return int((await session.execute(stmt)).scalar_one())
|
||||
|
||||
# JSON filters can't run in portable SQL — scan just the metadata
|
||||
# column in batches and count in Python (mirrors search()).
|
||||
stmt = select(ThreadMetaRow.metadata_json).order_by(ThreadMetaRow.thread_id)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
total = 0
|
||||
batch_size = 1000
|
||||
scan_offset = 0
|
||||
async with self._sf() as session:
|
||||
while True:
|
||||
result = await session.execute(stmt.limit(batch_size).offset(scan_offset))
|
||||
rows = result.scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
for metadata_json in rows:
|
||||
md = metadata_json or {}
|
||||
if metadata and not all(md.get(k) == v for k, v in metadata.items()):
|
||||
continue
|
||||
if exclude_system and md.get("system"):
|
||||
continue
|
||||
total += 1
|
||||
if len(rows) < batch_size:
|
||||
break
|
||||
scan_offset += len(rows)
|
||||
return total
|
||||
|
||||
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
|
||||
"""Return True if the row exists and is owned (or filter bypassed)."""
|
||||
if resolved_user_id is None:
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Post-turn background skill review worker.
|
||||
|
||||
After a conversation turn completes, SkillReviewMiddleware calls
|
||||
``maybe_start_review()``. This module runs a lightweight LLM React agent
|
||||
(with skill_manage + skill_list tools) that reads the conversation history
|
||||
and decides whether to create or patch skills.
|
||||
|
||||
Concurrency model: per-thread asyncio.Lock + a single _pending slot.
|
||||
If a review is already running for the thread, the newer messages overwrite
|
||||
the pending slot; once the current review finishes, it drains that slot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import textwrap
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from langchain_core.messages import BaseMessage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_locks: dict[str, asyncio.Lock] = {}
|
||||
_pending: dict[str, list[BaseMessage]] = {}
|
||||
|
||||
|
||||
def _get_lock(thread_id: str) -> asyncio.Lock:
|
||||
if thread_id not in _locks:
|
||||
_locks[thread_id] = asyncio.Lock()
|
||||
return _locks[thread_id]
|
||||
|
||||
|
||||
def extract_tool_call_count(messages: list[BaseMessage]) -> int:
|
||||
"""Count total tool calls across all AI messages in the conversation."""
|
||||
count = 0
|
||||
for msg in messages:
|
||||
tool_calls: Any = None
|
||||
if hasattr(msg, "tool_calls"):
|
||||
tool_calls = msg.tool_calls
|
||||
elif isinstance(msg, dict):
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if tool_calls:
|
||||
count += len(tool_calls)
|
||||
return count
|
||||
|
||||
|
||||
def _format_messages_for_review(messages: list[BaseMessage], max_messages: int) -> str:
|
||||
"""Serialize recent messages to a readable text block for the LLM reviewer."""
|
||||
recent = messages[-max_messages:]
|
||||
lines: list[str] = []
|
||||
for msg in recent:
|
||||
role = type(msg).__name__.replace("Message", "")
|
||||
content = msg.content if isinstance(msg.content, str) else str(msg.content)
|
||||
tool_calls = getattr(msg, "tool_calls", None)
|
||||
if tool_calls:
|
||||
calls_str = ", ".join(
|
||||
f"{tc.name if hasattr(tc, 'name') else tc.get('name', '?')}()"
|
||||
for tc in tool_calls
|
||||
)
|
||||
lines.append(f"[{role}] <tool_calls: {calls_str}>")
|
||||
else:
|
||||
preview = content[:400].replace("\n", " ")
|
||||
lines.append(f"[{role}] {preview}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
_REVIEW_SYSTEM_PROMPT = textwrap.dedent("""\
|
||||
# Role: 技能评审代理(保守模式)
|
||||
|
||||
## Profile
|
||||
- language: zh-CN
|
||||
- description: 审阅已完成对话,仅在出现强证据的高价值可复用工作流时才沉淀为技能。**默认决策是 do-nothing** —— 宁可漏掉一个普通技能,也绝不创建垃圾技能。
|
||||
- background: 历史问题:之前版本触发过于宽松,沉淀了大量低价值技能,污染了技能库。本版本针对性收紧。
|
||||
- personality: 极度保守、苛刻挑剔、宁缺勿滥;面对模糊场景默认 do-nothing。
|
||||
- expertise: 高价值工作流识别、严格去重、SKILL.md 规范、do-nothing 判定。
|
||||
- target_audience: 技能库维护者、平台工程师。
|
||||
|
||||
## Skills
|
||||
|
||||
1. 工作流抽象与评审
|
||||
- 价值评估: 区分一次性数据请求与具有沉淀价值的工作流。
|
||||
- 去数据化归纳: 抽象出与具体数据无关的通用步骤、参数与注意事项。
|
||||
- 决策映射: 在 do-nothing / patch / create 三者中做出可解释的唯一决定。
|
||||
|
||||
2. 文档与规范控制
|
||||
- SKILL.md 编写: 简洁步骤 + 命令 + 坑点提示(非对话稿)。
|
||||
- 命名规范: kebab-case(小写字母/数字/连字符,≤64 字符)。
|
||||
- 触发描述: 以 "When" 开头的单句(≤1024 字符、无尖括号)。
|
||||
- 结构校验: frontmatter 允许键、name 一致性、首尾分隔线。
|
||||
|
||||
3. 工具使用与错误处理
|
||||
- skill_list 优先调用,识别功能重叠与现有覆盖范围。
|
||||
- skill_manage 正确选 action=create / patch,并传递合规内容。
|
||||
- 报错诊断与修正后才能重试,禁止内容不变重提。
|
||||
|
||||
4. 质量控制与安全
|
||||
- 去重策略: 少而精,避免噪声与重复。
|
||||
- 隐私合规: 严禁嵌入用户特定数据、文件路径、凭据。
|
||||
- 一致性: name / description / content 三者语义对齐。
|
||||
|
||||
## Rules
|
||||
|
||||
### 0. 默认决策
|
||||
|
||||
**默认就是 do-nothing。** 仅当下面"必要条件"全部满足时才考虑创建;任一不满足 → do-nothing。垃圾技能比少一个技能更糟糕。
|
||||
|
||||
### 1. 创建必要条件(必须全部满足,AND 关系)
|
||||
|
||||
要创建一个新技能,以下条件**必须同时**满足:
|
||||
|
||||
1. **多步且有依赖**: 对话中至少 5 次工具调用,且步骤间存在依赖关系(不是并行查询,也不是同一工具的简单循环)。
|
||||
2. **包含纠错经验**: 包含至少一次失败 / 报错 / 意外结果,agent 通过修正后达到目标。这种"踩坑—纠错"才是有沉淀价值的知识。
|
||||
3. **跨场景可复用**: 抽离掉对话中的具体数据后,步骤本身在不同输入 / 不同数据集下仍然成立。
|
||||
4. **不与现有技能重叠**: skill_list 中没有功能覆盖率 ≥70% 的现有技能(若有,应当 patch 而非新建)。
|
||||
5. **能写出有料的 SKILL.md**: 抽象后能产出 ≥300 字、包含具体步骤 / 命令 / 坑点的工作流指南。写不出说明本来就没料。
|
||||
|
||||
### 2. 明确不要创建(即使工具调用很多也不要建)
|
||||
|
||||
- **单工具循环**: 连续多次 web_search 都只是搜索,没有形成工作流。
|
||||
- **一次性数据查询**: 仅是"搜索 + 总结"模式,没有可复用结构。
|
||||
- **LLM 常识可答**: 如"如何配置 Python 虚拟环境"这类公开文档 / 通识问题。
|
||||
- **流程过度通用**: 如"先搜索再总结再回答"这种描述等于没说。
|
||||
- **场景过窄**: 只在某一特定数据集 / 系统 / 项目下有效,换环境就失效。
|
||||
- **内容字数不足**: 写不出 ≥300 字含真干货的 SKILL.md。
|
||||
- **仅为"做点什么"而建**: 找不到清晰理由就 do-nothing,不要为了"完成 review"而强行创建。
|
||||
|
||||
### 3. 创建前自检(4 题,全部"是"才能 create)
|
||||
|
||||
提交 create 前在心中依次回答以下问题,任一为"否" → 立即放弃创建、改 do-nothing:
|
||||
|
||||
1. 如果我不创建这个技能,下一个遇到类似任务的用户会损失什么?(答得上来才证明有沉淀价值)
|
||||
2. 这个技能是否能被一句话简短复述?若"能",说明信息量不足 → do-nothing。
|
||||
3. SKILL.md 写完后是否包含至少 3 个具体步骤或命令?
|
||||
4. 步骤中是否含有非显而易见的坑点提示(不是 LLM 凭常识就能猜到的)?
|
||||
|
||||
### 4. 单次 review 上限
|
||||
|
||||
**每次 review 最多创建 1 条新技能。** 即便发现多个候选,也只挑最有价值的那一个,其余 do-nothing。这是硬约束,不接受例外。
|
||||
|
||||
### 5. Patch 优先于 Create
|
||||
|
||||
若 skill_list 中已存在功能相近(即便不完全相同)的技能,优先选择 patch 现有技能补充改进点,而非新建。新技能名额只留给**全新类别**的工作流。
|
||||
|
||||
### 6. 格式与命名硬约束
|
||||
|
||||
- 命名:kebab-case,仅含小写字母 / 数字 / 连字符,≤64 字符(如 git-bisect-bug-hunt、api-load-test)。
|
||||
- description:以 "When " 开头,单句,≤1024 字符,禁止尖括号 < >。
|
||||
- content:合法 SKILL.md,包含 frontmatter 与正文。
|
||||
- frontmatter 仅允许键:name、description、license、allowed-tools、metadata、compatibility、version、author。
|
||||
- frontmatter 首行与末行必须为 "---" 独占一行。
|
||||
- frontmatter 的 name 字段必须与 skill_manage 调用的 name 参数**完全一致**;不一致直接调用失败。
|
||||
- 禁止添加 tags、category、type 等额外键。
|
||||
- content 必须是简洁工作流指南,禁止对话转录。
|
||||
- 严禁包含用户特定数据、文件路径、凭据。
|
||||
|
||||
### 7. 错误处理
|
||||
|
||||
- 调用 skill_manage 报错 → 阅读错误信息,定位根因,**修正内容后**再重试。
|
||||
- 同名已存在 → 改用 action=patch(直接 create 同名会失败)。
|
||||
- 禁止在未修改内容的情况下重复提交。
|
||||
|
||||
## Workflows
|
||||
|
||||
### 目标
|
||||
仅在发现**高价值、可复用、有踩坑沉淀**的工作流时,创建或修补一条技能;其余一律 do-nothing。
|
||||
|
||||
### 步骤 1: 获取基线
|
||||
调用 skill_list,记录现有技能名称与功能覆盖范围。这一步在每次 review 中必须执行。
|
||||
|
||||
### 步骤 2: 价值初筛(三关,任一不过即 do-nothing 结束)
|
||||
通读对话并依次确认:
|
||||
- 工具调用总数是否 ≥5?否则 do-nothing 结束。
|
||||
- 是否包含失败 / 报错 / 修正过程?否则 do-nothing 结束。
|
||||
- 步骤间是否存在依赖关系?否则 do-nothing 结束。
|
||||
|
||||
只有三关全过才进入步骤 3。
|
||||
|
||||
### 步骤 3: 候选抽象
|
||||
将对话抽象为候选技能要素:
|
||||
- 候选名称(kebab-case)
|
||||
- 触发描述(When ... )
|
||||
- 关键步骤、命令、坑点提示
|
||||
- 剔除所有数据特定内容与隐私信息
|
||||
|
||||
### 步骤 4: 重叠检查
|
||||
与 skill_list 比对:
|
||||
- 完全无重叠 → 候选 action=create,进入步骤 5。
|
||||
- 部分重叠(功能覆盖 ≥70%)→ 候选 action=patch,跳到步骤 6。
|
||||
- 完全覆盖 → do-nothing 结束。
|
||||
|
||||
### 步骤 5: 创建前自检
|
||||
若候选为 create,依次回答 Rules 第 3 节的 4 个自检问题。任一为"否" → 立即放弃创建,do-nothing 结束。全部为"是"才进入步骤 6。
|
||||
|
||||
### 步骤 6: 提交
|
||||
- create:调用 skill_manage(action="create", name, content)。
|
||||
- patch:调用 skill_manage(action="patch", name, content),仅针对差异 / 改进点。
|
||||
- 一次 review 最多一次写操作。
|
||||
|
||||
### 步骤 7: 错误处理
|
||||
若报错,按 Rules 第 7 节处理:阅读错误信息 → 修正根因 → 再试。禁止内容不变重提。
|
||||
|
||||
### 预期结果
|
||||
- **多数情况下**: do-nothing。这是好的结果,不是失败。
|
||||
- **少数情况下**: 一条高价值新技能,或对现有技能的一次有针对性 patch。
|
||||
- **永远不要**: 为了"做点什么"而创建低价值技能。
|
||||
|
||||
## Initialization
|
||||
作为技能评审代理(保守模式),你必须严格遵守 Rules,按 Workflows 执行。**默认决策是 do-nothing。** 只有强证据完全满足所有创建条件时,才考虑创建。创建一条垃圾技能比放弃一条普通技能更糟糕。
|
||||
""")
|
||||
|
||||
|
||||
async def _run_review(thread_id: str, messages: list[BaseMessage], app_config: "AppConfig", user_id: str = "default") -> None:
|
||||
"""Core review logic: create_agent with full middleware chain, no 'silent creation' constraint.
|
||||
|
||||
Uses create_agent (the deerflow agent factory) directly so we can:
|
||||
- Reuse the full middleware chain (ToolErrorHandlingMiddleware, etc.)
|
||||
- Reuse the full tool set
|
||||
- But pass app_config with skill_evolution.enabled=False to strip the
|
||||
"Never create skills silently without user confirmation" constraint
|
||||
from the system prompt.
|
||||
Review instructions are injected via PromptPrefixMiddleware through
|
||||
additional_kwargs["prompt_prefix"] on the HumanMessage.
|
||||
"""
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template
|
||||
from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMiddleware
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.tools.tools import get_available_tools
|
||||
|
||||
evolution_cfg = app_config.skill_evolution
|
||||
timeout = evolution_cfg.review_worker.timeout_seconds
|
||||
max_messages = evolution_cfg.review_worker.max_messages
|
||||
model_name = evolution_cfg.review_worker.model_name
|
||||
|
||||
conversation_text = _format_messages_for_review(messages, max_messages)
|
||||
|
||||
# Disable skill_evolution so the system prompt does NOT include
|
||||
# "Never create skills silently without user confirmation".
|
||||
# Also prevents SkillReviewMiddleware from registering (no recursive reviews).
|
||||
review_app_config = app_config.model_copy(deep=True)
|
||||
review_app_config.skill_evolution.enabled = False
|
||||
|
||||
try:
|
||||
model = create_chat_model(name=model_name, app_config=review_app_config)
|
||||
except Exception:
|
||||
logger.warning("skill_review_worker: could not load model %r, aborting review", model_name)
|
||||
return
|
||||
|
||||
tools = get_available_tools(app_config=review_app_config, subagent_enabled=False)
|
||||
middlewares = build_lead_runtime_middlewares(app_config=review_app_config)
|
||||
middlewares.append(PromptPrefixMiddleware())
|
||||
|
||||
system_prompt = apply_prompt_template(
|
||||
subagent_enabled=False,
|
||||
max_concurrent_subagents=0,
|
||||
app_config=review_app_config,
|
||||
memory_injection_enabled=False,
|
||||
)
|
||||
|
||||
agent = create_agent(
|
||||
model=model,
|
||||
tools=tools,
|
||||
middleware=middlewares,
|
||||
system_prompt=system_prompt,
|
||||
state_schema=ThreadState,
|
||||
)
|
||||
|
||||
# RunnableConfig supplies thread_id so ThreadDataMiddleware is satisfied.
|
||||
import uuid
|
||||
run_config = RunnableConfig(configurable={"thread_id": f"__skill-review__{uuid.uuid4()}"})
|
||||
|
||||
# Review instructions arrive as prompt_prefix: PromptPrefixMiddleware prepends
|
||||
# them before every model call without persisting them to state.
|
||||
first_message = HumanMessage(
|
||||
content=(
|
||||
"Review this conversation and apply any skill changes you deem appropriate.\n\n"
|
||||
f"<conversation>\n{conversation_text}\n</conversation>"
|
||||
),
|
||||
additional_kwargs={"prompt_prefix": _REVIEW_SYSTEM_PROMPT},
|
||||
)
|
||||
|
||||
def _invoke_with_user_context() -> dict:
|
||||
from deerflow.runtime.user_context import set_current_user, reset_current_user
|
||||
|
||||
class _SimpleUser:
|
||||
def __init__(self, uid: str) -> None:
|
||||
self.id = uid
|
||||
|
||||
token = set_current_user(_SimpleUser(user_id))
|
||||
try:
|
||||
return agent.invoke({"messages": [first_message]}, config=run_config)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.get_event_loop().run_in_executor(None, _invoke_with_user_context),
|
||||
timeout=timeout,
|
||||
)
|
||||
# Log the final AI message so we can see what the review agent decided.
|
||||
final_msgs = (result or {}).get("messages", [])
|
||||
final_text = ""
|
||||
for msg in reversed(final_msgs):
|
||||
content = getattr(msg, "content", None)
|
||||
if content and isinstance(content, str) and content.strip():
|
||||
final_text = content.strip()[:1000]
|
||||
break
|
||||
logger.info(
|
||||
"skill_review_worker: review completed for thread %s — agent said: %s",
|
||||
thread_id,
|
||||
final_text or "(no text response)",
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning("skill_review_worker: review timed out after %ds (thread %s)", timeout, thread_id)
|
||||
except Exception:
|
||||
logger.exception("skill_review_worker: review failed for thread %s", thread_id)
|
||||
|
||||
|
||||
async def _run_review_with_pending(
|
||||
thread_id: str,
|
||||
messages: list[BaseMessage],
|
||||
app_config: "AppConfig",
|
||||
user_id: str = "default",
|
||||
) -> None:
|
||||
"""Run one review; if a newer snapshot arrived while we ran, drain it."""
|
||||
lock = _get_lock(thread_id)
|
||||
async with lock:
|
||||
await _run_review(thread_id, messages, app_config, user_id)
|
||||
|
||||
pending = _pending.pop(thread_id, None)
|
||||
if pending is not None:
|
||||
asyncio.create_task(_run_review_with_pending(thread_id, pending, app_config, user_id))
|
||||
|
||||
|
||||
def maybe_start_review(
|
||||
thread_id: str,
|
||||
messages: list[BaseMessage],
|
||||
app_config: "AppConfig",
|
||||
user_id: str = "default",
|
||||
) -> bool:
|
||||
"""Schedule a background review for ``thread_id``.
|
||||
|
||||
Returns True if a new task was started, False if queued or skipped.
|
||||
"""
|
||||
evolution_cfg = app_config.skill_evolution
|
||||
if not evolution_cfg.enabled or not evolution_cfg.review_worker.enabled:
|
||||
return False
|
||||
|
||||
tool_count = extract_tool_call_count(messages)
|
||||
if tool_count < evolution_cfg.review_worker.min_tool_calls:
|
||||
logger.debug(
|
||||
"skill_review_worker: skipping review for thread %s (%d tool calls < min %d)",
|
||||
thread_id,
|
||||
tool_count,
|
||||
evolution_cfg.review_worker.min_tool_calls,
|
||||
)
|
||||
return False
|
||||
|
||||
lock = _get_lock(thread_id)
|
||||
if lock.locked():
|
||||
_pending[thread_id] = messages
|
||||
logger.debug("skill_review_worker: queued review for thread %s (review in progress)", thread_id)
|
||||
return False
|
||||
|
||||
asyncio.create_task(_run_review_with_pending(thread_id, messages, app_config, user_id))
|
||||
logger.info("skill_review_worker: started review for thread %s (%d tool calls) user=%s", thread_id, tool_count, user_id)
|
||||
return True
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""笔记本 RAG 检索工具。
|
||||
"""空间 RAG 检索工具。
|
||||
|
||||
从 LangGraph configurable 中读取 notebook_search_space_id / notebook_source_ids /
|
||||
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在笔记本模式下引用回答。
|
||||
notebook_token,调用外部检索接口返回相关文本块,供 LLM 在空间模式下引用回答。
|
||||
|
||||
API base URL 通过环境变量 DEER_FLOW_NOTEBOOK_API_BASE 配置。
|
||||
"""
|
||||
|
||||
+65
-20
@@ -134,27 +134,58 @@ def _default_agent_store() -> Any | None:
|
||||
return None
|
||||
|
||||
|
||||
async def _assert_agent_may_delete(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
|
||||
"""Pre-flight ownership check before agent-delete touches the on-disk file.
|
||||
async def _assert_skill_is_agent_authored(name: str) -> None:
|
||||
"""Refuse to modify skills authored by a human (UI upload / adopt / legacy).
|
||||
|
||||
LocalSkillStorage doesn't know about users — it deletes anything that
|
||||
matches by name. Without this guard, a malicious or hallucinating agent
|
||||
could destroy another user's skill just by knowing the name. We refuse
|
||||
unless the DB confirms ``user_id`` owns the skill outright.
|
||||
Even when ownership lines up, we draw a hard line at content the *human*
|
||||
curated through the Web UI: uploads, adopted orphan skills, and legacy
|
||||
skills with no usage entry (which default to ``source='upload'``).
|
||||
Those represent reviewed content the user has explicitly endorsed —
|
||||
letting the agent rewrite them risks silent drift away from what was
|
||||
approved. Only skills marked ``source='agent'`` (set by
|
||||
``mark_agent_created`` in the ``create`` flow) remain editable so the
|
||||
evolution loop can keep refining them.
|
||||
|
||||
Returns the DB record for use in the subsequent cascade step. Raises
|
||||
``ValueError`` (mapped to a tool error visible to the LLM) on refusal.
|
||||
Raises ``ValueError`` (mapped to a tool error visible to the LLM) on
|
||||
refusal.
|
||||
"""
|
||||
from deerflow.skills.usage import get_entry
|
||||
|
||||
entry = await _to_thread(get_entry, name)
|
||||
if entry.get("source") != "agent":
|
||||
raise ValueError(
|
||||
f"Refused: skill '{name}' was authored by a human via the Web UI "
|
||||
f"and cannot be modified by an agent. Ask the owner to edit it "
|
||||
f"through the Web UI, or have the user duplicate it to a new "
|
||||
f"agent-owned skill first."
|
||||
)
|
||||
|
||||
|
||||
async def _assert_agent_may_modify(store: SkillStore, name: str, user_id: str) -> dict[str, Any]:
|
||||
"""Pre-flight ownership check before an agent-driven write touches a custom skill.
|
||||
|
||||
LocalSkillStorage operates by name only — patch/edit/write_file/remove_file
|
||||
and delete all resolve a name to a directory without knowing who owns it.
|
||||
Without this guard a malicious or hallucinating agent could overwrite or
|
||||
destroy another user's skill (including UI-uploaded skills) just by
|
||||
knowing the name. We refuse unless the DB confirms ``user_id`` owns the
|
||||
skill outright.
|
||||
|
||||
Returns the DB record so callers that need owner info (e.g. delete
|
||||
cascade) can reuse it. Raises ``ValueError`` (mapped to a tool error
|
||||
visible to the LLM) on refusal.
|
||||
"""
|
||||
record = await store.get_any(name)
|
||||
if record is None:
|
||||
# No DB row — could be a legacy/orphan skill. Refuse rather than
|
||||
# delete blindly; admin can clean up via the Web UI.
|
||||
# touch blindly; users should adopt it via /api/skills/reconcile-custom
|
||||
# or an admin can clean it up via the Web UI.
|
||||
raise ValueError(
|
||||
f"Refused: skill '{name}' has no ownership record. Ask a human to delete it via the Web UI."
|
||||
f"Refused: skill '{name}' has no ownership record. Ask a human to adopt or manage it via the Web UI."
|
||||
)
|
||||
owner = record.get("owner_user_id")
|
||||
if owner is None:
|
||||
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be deleted by an agent.")
|
||||
raise ValueError(f"Refused: skill '{name}' is built-in and cannot be modified by an agent.")
|
||||
if owner != user_id:
|
||||
raise ValueError(f"Refused: skill '{name}' is owned by another user.")
|
||||
return record
|
||||
@@ -200,6 +231,10 @@ async def _skill_manage_impl(
|
||||
lock = _get_lock(name)
|
||||
thread_id = _get_thread_id(runtime)
|
||||
skill_storage = get_or_new_skill_storage()
|
||||
# Resolved up-front so ownership checks in the write-action branches stay
|
||||
# one-liners. Cheap when the DB is unconfigured (returns None).
|
||||
user_id = get_effective_user_id()
|
||||
skill_store = _default_skill_store()
|
||||
|
||||
async with lock:
|
||||
if action == "create":
|
||||
@@ -224,14 +259,16 @@ async def _skill_manage_impl(
|
||||
# surfaces it (custom skills without a DB row are filtered out by
|
||||
# ``GET /api/skills``). Resolve the user_id *here* on the main
|
||||
# async stack — contextvars don't propagate into ``to_thread``.
|
||||
store = _default_skill_store()
|
||||
if store is not None:
|
||||
await _ensure_skill_db_registered(store, name, get_effective_user_id())
|
||||
if skill_store is not None:
|
||||
await _ensure_skill_db_registered(skill_store, name, user_id)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
return f"Created custom skill '{name}'."
|
||||
|
||||
if action == "edit":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if content is None:
|
||||
raise ValueError("content is required for edit.")
|
||||
await _to_thread(skill_storage.validate_skill_markdown_content, name, content)
|
||||
@@ -250,6 +287,9 @@ async def _skill_manage_impl(
|
||||
|
||||
if action == "patch":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if find is None or replace is None:
|
||||
raise ValueError("find and replace are required for patch.")
|
||||
skill_file = skill_storage.get_custom_skill_file(name)
|
||||
@@ -277,11 +317,10 @@ async def _skill_manage_impl(
|
||||
# Ownership pre-check: LocalSkillStorage deletes purely by name,
|
||||
# so without this an agent could blow away another user's skill
|
||||
# just by knowing the name. Raise BEFORE touching the filesystem.
|
||||
user_id = get_effective_user_id()
|
||||
store = _default_skill_store()
|
||||
record: dict[str, Any] | None = None
|
||||
if store is not None:
|
||||
record = await _assert_agent_may_delete(store, name, user_id)
|
||||
if skill_store is not None:
|
||||
record = await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
|
||||
await _to_thread(
|
||||
skill_storage.delete_custom_skill,
|
||||
@@ -306,9 +345,9 @@ async def _skill_manage_impl(
|
||||
await _to_thread(forget, name)
|
||||
except Exception:
|
||||
logger.debug("Failed to forget usage entry for %r", name, exc_info=True)
|
||||
if store is not None:
|
||||
if skill_store is not None:
|
||||
try:
|
||||
await store.delete(name, user_id)
|
||||
await skill_store.delete(name, user_id)
|
||||
except Exception:
|
||||
logger.debug("Failed to delete DB row for %r", name, exc_info=True)
|
||||
agent_store = _default_agent_store()
|
||||
@@ -330,6 +369,9 @@ async def _skill_manage_impl(
|
||||
|
||||
if action == "write_file":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if path is None or content is None:
|
||||
raise ValueError("path and content are required for write_file.")
|
||||
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)
|
||||
@@ -347,6 +389,9 @@ async def _skill_manage_impl(
|
||||
|
||||
if action == "remove_file":
|
||||
await _to_thread(skill_storage.ensure_custom_skill_is_editable, name)
|
||||
if skill_store is not None:
|
||||
await _assert_agent_may_modify(skill_store, name, user_id)
|
||||
await _assert_skill_is_agent_authored(name)
|
||||
if path is None:
|
||||
raise ValueError("path is required for remove_file.")
|
||||
target = await _to_thread(skill_storage.ensure_safe_support_path, name, path)
|
||||
|
||||
@@ -62,13 +62,43 @@ CALL _add_column_if_missing('skills', 'name_zh', 'VARCHAR(128) NULL DEFAULT NULL
|
||||
-- detail: 技能的长文介绍,对应 migration 20260518_01。
|
||||
-- ORM: SkillRow.detail = Text, nullable=True。
|
||||
CALL _add_column_if_missing('skills', 'detail', 'TEXT NULL DEFAULT NULL');
|
||||
-- square_id: 技能所属的发布广场(发布广场功能),对应 migration 20260611_01。
|
||||
-- ORM: SkillRow.square_id = String(64), nullable=False, default=''。
|
||||
CALL _add_column_if_missing('skills', 'square_id', "VARCHAR(64) NOT NULL DEFAULT ''");
|
||||
|
||||
-- ---- agents ------------------------------------------------------------------
|
||||
-- square_id: 智能体所属的发布广场(发布广场功能),对应 migration 20260611_01。
|
||||
-- ORM: AgentRow.square_id = String(64), nullable=False, default=''。
|
||||
CALL _add_column_if_missing('agents', 'square_id', "VARCHAR(64) NOT NULL DEFAULT ''");
|
||||
|
||||
-- ---- roundtable_drafts -------------------------------------------------------
|
||||
-- task_id: 外部任务 id(无界嵌入抽屉传入),对应 migration 20260611_01。非空时草稿
|
||||
-- 按 task_id 共享、不再按 user_id 分权。ORM: RoundtableDraftRow.task_id =
|
||||
-- String(128), nullable=True, index=True。
|
||||
CALL _add_column_if_missing('roundtable_drafts', 'task_id', 'VARCHAR(128) NULL DEFAULT NULL');
|
||||
|
||||
-- ---- 清理临时存储过程 --------------------------------------------------------
|
||||
DROP PROCEDURE IF EXISTS _add_column_if_missing;
|
||||
|
||||
-- ---- roundtable_drafts.task_id 索引(幂等) -----------------------------------
|
||||
SET @idx_exists = (
|
||||
SELECT COUNT(*) FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = 'roundtable_drafts'
|
||||
AND index_name = 'ix_roundtable_drafts_task_id'
|
||||
);
|
||||
SET @idx_ddl = IF(@idx_exists = 0,
|
||||
'CREATE INDEX `ix_roundtable_drafts_task_id` ON `roundtable_drafts` (`task_id`)',
|
||||
'SELECT ''SKIPPED ix_roundtable_drafts_task_id (already exists)'' AS result');
|
||||
PREPARE stmt FROM @idx_ddl;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- ---- 执行后自检:确认本批次涉及的列都已存在 ---------------------------------
|
||||
SELECT table_name, column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE()
|
||||
AND (table_name = 'skills' AND column_name IN ('name_zh', 'detail'))
|
||||
AND ((table_name = 'skills' AND column_name IN ('name_zh', 'detail', 'square_id'))
|
||||
OR (table_name = 'agents' AND column_name IN ('square_id'))
|
||||
OR (table_name = 'roundtable_drafts' AND column_name = 'task_id'))
|
||||
ORDER BY table_name, column_name;
|
||||
@@ -177,12 +177,60 @@ def test_load_config_extras_returns_values_written_to_yaml(isolated_paths) -> No
|
||||
)
|
||||
|
||||
extras = _load_config_extras(agent_id)
|
||||
assert extras == {"skills": ["s1"], "model": None, "tool_groups": None}
|
||||
assert extras == {"skills": ["s1"], "model": None, "tool_groups": None, "common_questions": None}
|
||||
|
||||
|
||||
def test_load_config_extras_missing_directory_returns_none_triple(isolated_paths) -> None:
|
||||
extras = _load_config_extras("does-not-exist")
|
||||
assert extras == {"skills": None, "model": None, "tool_groups": None}
|
||||
assert extras == {"skills": None, "model": None, "tool_groups": None, "common_questions": None}
|
||||
|
||||
|
||||
def test_write_agent_config_persists_common_questions(isolated_paths) -> None:
|
||||
agent_id = "agent-cq-1"
|
||||
# Structured {title, prompt} questions round-trip as clean dict mappings.
|
||||
questions = [
|
||||
{"title": "能做什么", "prompt": "这个智能体能做什么?"},
|
||||
{"title": "上传文件", "prompt": "如何上传文件?"},
|
||||
]
|
||||
_write_agent_config(agent_id, name="助手", description="", common_questions=questions)
|
||||
|
||||
config_file = paths_module.get_paths().agent_dir(agent_id) / "config.yaml"
|
||||
raw = yaml.safe_load(config_file.read_text(encoding="utf-8"))
|
||||
assert raw["common_questions"] == questions
|
||||
|
||||
extras = _load_config_extras(agent_id)
|
||||
assert [q.model_dump() for q in extras["common_questions"]] == questions
|
||||
|
||||
|
||||
def test_write_agent_config_coerces_legacy_string_questions(isolated_paths) -> None:
|
||||
"""Legacy ``list[str]`` configs coerce to {title, prompt} with title=content."""
|
||||
agent_id = "agent-cq-legacy"
|
||||
_write_agent_config(
|
||||
agent_id, name="助手", description="", common_questions=["如何上传文件?"]
|
||||
)
|
||||
|
||||
config_file = paths_module.get_paths().agent_dir(agent_id) / "config.yaml"
|
||||
raw = yaml.safe_load(config_file.read_text(encoding="utf-8"))
|
||||
assert raw["common_questions"] == [
|
||||
{"title": "如何上传文件?", "prompt": "如何上传文件?"}
|
||||
]
|
||||
|
||||
|
||||
def test_write_agent_config_omits_common_questions_when_none(isolated_paths) -> None:
|
||||
agent_id = "agent-cq-2"
|
||||
_write_agent_config(agent_id, name="x", description="")
|
||||
|
||||
config_file = paths_module.get_paths().agent_dir(agent_id) / "config.yaml"
|
||||
raw = yaml.safe_load(config_file.read_text(encoding="utf-8"))
|
||||
assert "common_questions" not in raw
|
||||
assert _load_config_extras(agent_id)["common_questions"] is None
|
||||
|
||||
|
||||
def test_write_agent_config_explicit_empty_common_questions_is_persisted(isolated_paths) -> None:
|
||||
agent_id = "agent-cq-3"
|
||||
_write_agent_config(agent_id, name="x", description="", common_questions=[])
|
||||
|
||||
assert _load_config_extras(agent_id)["common_questions"] == []
|
||||
|
||||
|
||||
def test_write_agent_config_explicit_empty_skills_is_persisted(isolated_paths) -> None:
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tests that the curator scheduler does not block application startup.
|
||||
|
||||
Verifies:
|
||||
1. _curator_scheduler_loop is launched as a non-blocking asyncio task.
|
||||
2. The scheduler sleeps for the startup delay before doing any work.
|
||||
3. maybe_run_curator is NOT called during the startup delay window.
|
||||
4. When the lock is unavailable the loop exits immediately without calling the curator.
|
||||
|
||||
NOTE: ``import app.gateway.app as m`` resolves to the FastAPI *instance* because
|
||||
``app/gateway/__init__.py`` re-exports ``app = FastAPI(...)``. The actual module
|
||||
object lives in ``sys.modules["app.gateway.app"]`` and is used here via a helper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
def _gateway_module():
|
||||
"""Return the real app.gateway.app *module* (not the FastAPI instance)."""
|
||||
import importlib
|
||||
importlib.import_module("app.gateway.app") # side-effect: populates sys.modules
|
||||
return sys.modules["app.gateway.app"]
|
||||
|
||||
|
||||
_STARTUP_DELAY = 300 # matches _CURATOR_STARTUP_DELAY_SECONDS in app.gateway.app
|
||||
_CHECK_INTERVAL = 3600 # matches _CURATOR_CHECK_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def test_scheduler_does_not_block_startup():
|
||||
"""create_task must return immediately — startup must not await the loop body."""
|
||||
mod = _gateway_module()
|
||||
curator_called = False
|
||||
|
||||
async def fake_sleep(_delay):
|
||||
pass
|
||||
|
||||
async def fake_curator():
|
||||
nonlocal curator_called
|
||||
curator_called = True
|
||||
|
||||
async def run():
|
||||
with (
|
||||
patch.object(mod, "_acquire_singleton_lock", return_value=MagicMock()),
|
||||
patch.object(mod, "_release_singleton_lock"),
|
||||
patch.object(mod, "_CURATOR_STARTUP_DELAY_SECONDS", _STARTUP_DELAY),
|
||||
patch.object(mod, "_CURATOR_CHECK_INTERVAL_SECONDS", _CHECK_INTERVAL),
|
||||
patch("deerflow.skills.curator.maybe_run_curator", fake_curator),
|
||||
patch("asyncio.sleep", fake_sleep),
|
||||
):
|
||||
task = asyncio.create_task(mod._curator_scheduler_loop())
|
||||
# create_task must return immediately; the coroutine is still pending
|
||||
assert not task.done(), "task should be running in the background, not finished"
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
asyncio.run(run())
|
||||
assert not curator_called, "curator must not fire before the startup delay completes"
|
||||
|
||||
|
||||
def test_scheduler_sleeps_startup_delay_before_first_curator_call():
|
||||
"""The very first asyncio.sleep must use the startup-delay value (300 s)."""
|
||||
mod = _gateway_module()
|
||||
sleep_calls: list[float] = []
|
||||
curator_called = False
|
||||
|
||||
async def fake_sleep(delay):
|
||||
sleep_calls.append(delay)
|
||||
raise asyncio.CancelledError # stop after recording first sleep
|
||||
|
||||
async def fake_curator():
|
||||
nonlocal curator_called
|
||||
curator_called = True
|
||||
|
||||
async def run():
|
||||
with (
|
||||
patch.object(mod, "_acquire_singleton_lock", return_value=MagicMock()),
|
||||
patch.object(mod, "_release_singleton_lock"),
|
||||
patch.object(mod, "_CURATOR_STARTUP_DELAY_SECONDS", _STARTUP_DELAY),
|
||||
patch.object(mod, "_CURATOR_CHECK_INTERVAL_SECONDS", _CHECK_INTERVAL),
|
||||
patch("deerflow.skills.curator.maybe_run_curator", fake_curator),
|
||||
patch("asyncio.sleep", fake_sleep),
|
||||
):
|
||||
task = asyncio.create_task(mod._curator_scheduler_loop())
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
assert sleep_calls, "asyncio.sleep was never called"
|
||||
assert sleep_calls[0] == _STARTUP_DELAY, (
|
||||
f"first sleep must be the startup delay ({_STARTUP_DELAY}s), got {sleep_calls[0]}"
|
||||
)
|
||||
assert not curator_called, "curator must not be called before startup delay elapses"
|
||||
|
||||
|
||||
def test_scheduler_actually_calls_curator_after_startup_delay():
|
||||
"""After the startup delay elapses, maybe_run_curator must be called."""
|
||||
mod = _gateway_module()
|
||||
curator_call_count = 0
|
||||
|
||||
async def fake_sleep(*_):
|
||||
pass # instant — simulate time passing
|
||||
|
||||
async def fake_curator():
|
||||
nonlocal curator_call_count
|
||||
curator_call_count += 1
|
||||
# After first call cancel the loop so the test terminates
|
||||
raise asyncio.CancelledError
|
||||
|
||||
async def run():
|
||||
with (
|
||||
patch.object(mod, "_acquire_singleton_lock", return_value=MagicMock()),
|
||||
patch.object(mod, "_release_singleton_lock"),
|
||||
patch.object(mod, "_CURATOR_STARTUP_DELAY_SECONDS", _STARTUP_DELAY),
|
||||
patch.object(mod, "_CURATOR_CHECK_INTERVAL_SECONDS", _CHECK_INTERVAL),
|
||||
patch("deerflow.skills.curator.maybe_run_curator", fake_curator),
|
||||
patch("asyncio.sleep", fake_sleep),
|
||||
):
|
||||
task = asyncio.create_task(mod._curator_scheduler_loop())
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
asyncio.run(run())
|
||||
assert curator_call_count >= 1, (
|
||||
f"maybe_run_curator should have been called at least once, got {curator_call_count}"
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_skips_when_lock_unavailable():
|
||||
"""When another worker holds the lock the loop must exit without doing any work."""
|
||||
mod = _gateway_module()
|
||||
curator_called = False
|
||||
|
||||
async def fake_curator():
|
||||
nonlocal curator_called
|
||||
curator_called = True
|
||||
|
||||
async def run():
|
||||
with (
|
||||
patch.object(mod, "_acquire_singleton_lock", return_value=None),
|
||||
patch("deerflow.skills.curator.maybe_run_curator", fake_curator),
|
||||
):
|
||||
await mod._curator_scheduler_loop()
|
||||
|
||||
asyncio.run(run())
|
||||
assert not curator_called, "curator must not run when lock is not acquired"
|
||||
@@ -84,6 +84,14 @@ class TestRegistration:
|
||||
assert manager.add_provider(_FakeProvider("mem0")) is False
|
||||
assert len(manager.providers) == 2
|
||||
|
||||
def test_knowledge_does_not_consume_external_slot(self):
|
||||
manager = MemoryManager()
|
||||
manager.add_provider(_FakeProvider("builtin"))
|
||||
manager.add_provider(_FakeProvider("knowledge"))
|
||||
assert manager.add_provider(_FakeProvider("hindsight")) is True
|
||||
assert manager.add_provider(_FakeProvider("mem0")) is False
|
||||
assert [p.name for p in manager.providers] == ["builtin", "knowledge", "hindsight"]
|
||||
|
||||
def test_get_provider(self):
|
||||
manager = MemoryManager()
|
||||
builtin = _FakeProvider("builtin")
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Tests for SkillReviewMiddleware."""
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
|
||||
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
|
||||
|
||||
|
||||
def _make_state(messages):
|
||||
state = MagicMock()
|
||||
state.get.side_effect = lambda key, default=None: messages if key == "messages" else default
|
||||
return state
|
||||
|
||||
|
||||
def _make_runtime(thread_id="thread-abc"):
|
||||
runtime = MagicMock()
|
||||
runtime.context = {"thread_id": thread_id} if thread_id else {}
|
||||
return runtime
|
||||
|
||||
|
||||
def test_after_agent_calls_maybe_start_review():
|
||||
"""When enabled and tool calls exist, maybe_start_review is called with correct thread_id."""
|
||||
middleware = SkillReviewMiddleware()
|
||||
|
||||
ai_msg = MagicMock(spec=AIMessage)
|
||||
tc = MagicMock()
|
||||
tc.name = "bash"
|
||||
ai_msg.tool_calls = [tc]
|
||||
|
||||
state = _make_state([HumanMessage(content="hi"), ai_msg])
|
||||
runtime = _make_runtime("thread-xyz")
|
||||
|
||||
app_cfg = MagicMock()
|
||||
with patch("deerflow.agents.middlewares.skill_review_middleware.get_app_config", return_value=app_cfg):
|
||||
with patch("deerflow.agents.middlewares.skill_review_middleware.maybe_start_review") as mock_review:
|
||||
middleware.after_agent(state, runtime)
|
||||
|
||||
mock_review.assert_called_once()
|
||||
call_args = mock_review.call_args
|
||||
assert call_args[0][0] == "thread-xyz" # thread_id
|
||||
assert call_args[0][2] is app_cfg # app_config
|
||||
|
||||
|
||||
def test_after_agent_no_thread_id_skips_review():
|
||||
"""When thread_id is missing, maybe_start_review is NOT called."""
|
||||
middleware = SkillReviewMiddleware()
|
||||
state = _make_state([HumanMessage(content="hi")])
|
||||
runtime = _make_runtime("")
|
||||
|
||||
app_cfg = MagicMock()
|
||||
with patch("deerflow.agents.middlewares.skill_review_middleware.get_app_config", return_value=app_cfg):
|
||||
with patch("deerflow.agents.middlewares.skill_review_middleware.maybe_start_review") as mock_review:
|
||||
middleware.after_agent(state, runtime)
|
||||
|
||||
mock_review.assert_not_called()
|
||||
|
||||
|
||||
def test_after_agent_exception_is_swallowed():
|
||||
"""Exceptions in after_agent must not propagate."""
|
||||
middleware = SkillReviewMiddleware()
|
||||
state = _make_state([HumanMessage(content="hi")])
|
||||
runtime = _make_runtime("thread-err")
|
||||
|
||||
with patch("deerflow.agents.middlewares.skill_review_middleware.get_app_config", side_effect=RuntimeError("boom")):
|
||||
# Must not raise
|
||||
middleware.after_agent(state, runtime)
|
||||
|
||||
|
||||
def test_skill_review_middleware_in_chain_when_enabled():
|
||||
"""SkillReviewMiddleware appears in middleware chain before LoopDetectionMiddleware when enabled."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
from deerflow.agents.lead_agent.agent import _build_middlewares
|
||||
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
|
||||
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "test",
|
||||
"is_plan_mode": False,
|
||||
"is_bootstrap": False,
|
||||
"is_scheduled_run": False,
|
||||
}
|
||||
}
|
||||
|
||||
app_cfg = MagicMock()
|
||||
app_cfg.skill_evolution.enabled = True
|
||||
app_cfg.skill_evolution.review_worker.enabled = True
|
||||
app_cfg.token_usage.enabled = False
|
||||
app_cfg.tool_search.enabled = False
|
||||
app_cfg.skill_stop.enabled = False
|
||||
app_cfg.summarization.enabled = False
|
||||
app_cfg.memory = MagicMock()
|
||||
app_cfg.guardrails.enabled = False
|
||||
app_cfg.title = MagicMock()
|
||||
app_cfg.title.model_name = None
|
||||
|
||||
with patch("deerflow.agents.lead_agent.agent.get_app_config", return_value=app_cfg):
|
||||
with patch("deerflow.agents.lead_agent.agent.build_lead_runtime_middlewares", return_value=[]):
|
||||
with patch("deerflow.agents.lead_agent.agent._create_summarization_middleware", return_value=None):
|
||||
with patch("deerflow.agents.lead_agent.agent._create_todo_list_middleware", return_value=None):
|
||||
with patch("deerflow.models.vision.vision_enabled", return_value=False):
|
||||
middlewares = _build_middlewares(config, model_name=None, app_config=app_cfg)
|
||||
|
||||
type_names = [type(m).__name__ for m in middlewares]
|
||||
assert "SkillReviewMiddleware" in type_names, f"SkillReviewMiddleware not in chain: {type_names}"
|
||||
|
||||
review_idx = type_names.index("SkillReviewMiddleware")
|
||||
loop_idx = type_names.index("LoopDetectionMiddleware")
|
||||
assert review_idx < loop_idx, f"SkillReviewMiddleware ({review_idx}) must be before LoopDetectionMiddleware ({loop_idx})"
|
||||
@@ -0,0 +1,102 @@
|
||||
from deerflow.config.skill_evolution_config import SkillEvolutionConfig
|
||||
|
||||
|
||||
def test_review_worker_config_defaults():
|
||||
cfg = SkillEvolutionConfig()
|
||||
assert cfg.review_worker.enabled is False
|
||||
assert cfg.review_worker.min_tool_calls == 5
|
||||
assert cfg.review_worker.timeout_seconds == 120
|
||||
assert cfg.review_worker.model_name is None
|
||||
assert cfg.review_worker.max_messages == 60
|
||||
|
||||
|
||||
def test_review_worker_config_from_dict():
|
||||
cfg = SkillEvolutionConfig.model_validate({
|
||||
"enabled": True,
|
||||
"review_worker": {"enabled": True, "min_tool_calls": 5, "model_name": "my-model"},
|
||||
})
|
||||
assert cfg.review_worker.enabled is True
|
||||
assert cfg.review_worker.min_tool_calls == 5
|
||||
assert cfg.review_worker.model_name == "my-model"
|
||||
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import asyncio
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
|
||||
def _make_ai_message(tool_calls_data: list[dict]):
|
||||
msg = MagicMock()
|
||||
msg.__class__.__name__ = "AIMessage"
|
||||
tcs = []
|
||||
for tc in tool_calls_data:
|
||||
m = MagicMock()
|
||||
m.name = tc["name"]
|
||||
m.args = tc.get("args", {})
|
||||
tcs.append(m)
|
||||
msg.tool_calls = tcs
|
||||
return msg
|
||||
|
||||
|
||||
def test_extract_tool_call_count_counts_all():
|
||||
from deerflow.skills.review_worker import extract_tool_call_count
|
||||
msgs = [
|
||||
HumanMessage(content="do something"),
|
||||
_make_ai_message([{"name": "bash"}, {"name": "read_file"}]),
|
||||
_make_ai_message([{"name": "write_file"}]),
|
||||
]
|
||||
assert extract_tool_call_count(msgs) == 3
|
||||
|
||||
|
||||
def test_extract_tool_call_count_empty():
|
||||
from deerflow.skills.review_worker import extract_tool_call_count
|
||||
assert extract_tool_call_count([]) == 0
|
||||
|
||||
|
||||
def test_maybe_start_review_disabled_returns_false():
|
||||
from deerflow.skills.review_worker import maybe_start_review
|
||||
cfg = MagicMock()
|
||||
cfg.skill_evolution.enabled = False
|
||||
cfg.skill_evolution.review_worker.enabled = True
|
||||
cfg.skill_evolution.review_worker.min_tool_calls = 2
|
||||
msgs = [_make_ai_message([{"name": "bash"}, {"name": "read_file"}, {"name": "write_file"}])]
|
||||
assert maybe_start_review("thread-1", msgs, cfg) is False
|
||||
|
||||
|
||||
def test_maybe_start_review_too_few_tool_calls_returns_false():
|
||||
from deerflow.skills.review_worker import maybe_start_review
|
||||
cfg = MagicMock()
|
||||
cfg.skill_evolution.enabled = True
|
||||
cfg.skill_evolution.review_worker.enabled = True
|
||||
cfg.skill_evolution.review_worker.min_tool_calls = 5
|
||||
msgs = [_make_ai_message([{"name": "bash"}])]
|
||||
assert maybe_start_review("thread-4", msgs, cfg) is False
|
||||
|
||||
|
||||
def test_maybe_start_review_starts_task():
|
||||
from deerflow.skills import review_worker as rw
|
||||
# Reset module state so thread-3 has no lock
|
||||
rw._locks.pop("thread-3", None)
|
||||
rw._pending.pop("thread-3", None)
|
||||
|
||||
cfg = MagicMock()
|
||||
cfg.skill_evolution.enabled = True
|
||||
cfg.skill_evolution.review_worker.enabled = True
|
||||
cfg.skill_evolution.review_worker.min_tool_calls = 2
|
||||
cfg.skill_evolution.review_worker.timeout_seconds = 30
|
||||
cfg.skill_evolution.review_worker.max_messages = 20
|
||||
cfg.skill_evolution.review_worker.model_name = None
|
||||
|
||||
msgs = [_make_ai_message([{"name": "bash"}, {"name": "read_file"}, {"name": "write_file"}])]
|
||||
|
||||
with patch("deerflow.skills.review_worker.asyncio") as mock_asyncio:
|
||||
fake_lock = MagicMock()
|
||||
fake_lock.locked.return_value = False
|
||||
mock_asyncio.Lock.return_value = fake_lock
|
||||
mock_asyncio.create_task = MagicMock()
|
||||
|
||||
result = rw.maybe_start_review("thread-3", msgs, cfg)
|
||||
|
||||
assert result is True
|
||||
mock_asyncio.create_task.assert_called_once()
|
||||
@@ -71,7 +71,27 @@ models:
|
||||
display_name: deepseek-ai/DeepSeek-V4-Flash
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: deepseek-ai/DeepSeek-V4-Flash
|
||||
api_key: sk-qwwwvjompnzxqhndctlwmmomvqdgaqqqgxvibfxchsikvgod
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: Pro/MiniMaxAI/MiniMax-M2.5
|
||||
display_name: Pro/MiniMaxAI/MiniMax-M2.5
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: Pro/MiniMaxAI/MiniMax-M2.5
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: inclusionAI/Ling-flash-2.0
|
||||
display_name: inclusionAI/Ling-flash-2.0
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: inclusionAI/Ling-flash-2.0
|
||||
api_key: sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
@@ -200,7 +220,7 @@ checkpointer:
|
||||
type: postgres
|
||||
# true: 新 checkpoint 写入 Postgres;false: 保留 PG 配置但继续使用 SQLite。
|
||||
# 也可临时用环境变量 DEER_FLOW_USE_PG_CHECKPOINTER=0 覆盖。
|
||||
use_postgres: true
|
||||
use_postgres: false
|
||||
connection_string: postgresql://postgres:postgres123@47.88.25.99:8432/deerflow
|
||||
sqlite_connection_string: backend/.deer-flow/data/deerflow.db
|
||||
dual_read_enabled: true
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
{
|
||||
"enabled": true,
|
||||
"moderation_model_name": null,
|
||||
"curator": {
|
||||
"enabled": false,
|
||||
"interval_hours": 168,
|
||||
"enabled": true,
|
||||
"interval_hours": 24,
|
||||
"stale_after_days": 30,
|
||||
"archive_after_days": 90
|
||||
},
|
||||
"review_worker": {
|
||||
"enabled": true,
|
||||
"min_tool_calls": 3,
|
||||
"max_messages": 60,
|
||||
"timeout_seconds": 120,
|
||||
"model_name": null
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user