diff --git a/frontend-web/src/components/workspace/input-box.tsx b/frontend-web/src/components/workspace/input-box.tsx index d84d5060..f4a30d63 100644 --- a/frontend-web/src/components/workspace/input-box.tsx +++ b/frontend-web/src/components/workspace/input-box.tsx @@ -10,6 +10,7 @@ import { LightbulbIcon, PaperclipIcon, PlusIcon, + ShrinkIcon, SparklesIcon, RocketIcon, XIcon, @@ -329,6 +330,14 @@ export function InputBox({ setLocalSettings("display", { ragModeEnabled: !ragModeEnabled }); }, [ragModeEnabled, ragModeLocked, setLocalSettings]); + const contextCompressionEnabled = + localSettings.display.contextCompressionEnabled ?? false; + const handleContextCompressionToggle = useCallback(() => { + setLocalSettings("display", { + contextCompressionEnabled: !contextCompressionEnabled, + }); + }, [contextCompressionEnabled, setLocalSettings]); + const handleSubmit = useCallback( async (message: PromptInputMessage) => { if (status === "streaming") { @@ -735,6 +744,35 @@ export function InputBox({ 参考文献 + + + {contextCompressionEnabled && ( + + )} + + 上下文压缩 + + { writing_artifact_path?: string; memory_injection_enabled?: boolean; rag_mode_enabled?: boolean; + /** Per-request switch for context compression (上下文压缩). Default off. */ + summarization_enabled?: boolean; /** Per-conversation switch for knowledge-base RAG injection (phase 2). */ knowledge_rag_enabled?: boolean; agent_id?: string; diff --git a/frontend-web/src/pages/KnowledgeBasePage.tsx b/frontend-web/src/pages/KnowledgeBasePage.tsx index 4326ac2b..5380610c 100644 --- a/frontend-web/src/pages/KnowledgeBasePage.tsx +++ b/frontend-web/src/pages/KnowledgeBasePage.tsx @@ -176,10 +176,21 @@ function BatchImportDialog({ skip_existing: skipExisting, status: "approved", include_sources: true, + // Run in the background so the request returns immediately — a large + // batch otherwise blocks past the gateway timeout and shows "request + // failed". Threads stream into the list as they finish. + background: true, model_name: model, template_id: template, folder, }); + if (result.queued) { + toast.success( + `已转入后台导入 ${result.processed} 个线程，完成后会陆续出现在左侧列表（可稍后刷新查看）`, + ); + onOpenChange(false); + return; + } const base = `批量导入完成：新增 ${result.created}，跳过 ${result.skipped}，失败 ${result.failed}`; if (result.llm_fallback && result.llm_fallback > 0) { // The model failed on some threads — they were saved via rule-based @@ -549,7 +560,7 @@ export default function KnowledgeBasePage() { keyword: debouncedKeyword || undefined, source_type: sourceType === "all" ? undefined : sourceType, status: status === "all" ? undefined : status, - limit: 500, + limit: 1000, }), [debouncedKeyword, sourceType, status], ); diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 3530e9c8..d75ca744 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -252,7 +252,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA | **HTML Page Favorites** (`/api/html-page-favorites`) | Bookmarked generated HTML pages, owner-scoped. `GET /` - list; `POST /` `{page_id, title?, description?, tags?}` - bookmark a `scheduled_html_pages` row (extracts a reusable style/layout summary via GLM5 at favorite time); a page can only be favorited once per user (`DuplicateFavoriteError` ? HTTP 409). `GET/DELETE /{id}` - read/delete (the create-task reference picker deletes favorites here). A favorite can be selected as a style reference when creating a new `html_page` task. | | **Public Scheduled Tasks** (`/api/public/scheduled-tasks`) | **Unauthenticated** mirror for embedding. Adds `GET /runs/{rid}/html` and `GET /{task_id}/latest-html` for the `html_page` preview; the run result keeps `result.html` (only file paths are redacted). | | **Light Apps** (`/api/light-apps`) | Light-app registry (????? / ????), **global/shared** (no user scoping; `created_by`/`updated_by` audit-only). Two kinds: `window_open` (launched in a new tab from the ???? card grid) and `iframe` (embedded under a parent sidebar menu via `mount_location` and dynamically injected into the sidebar). Both carry `route_params` ? a list of `{key, paramType: login\|theme, source, customValue, themeValue}` descriptors the **frontend** resolves at launch time (login params read from `localStorage.userInfo`; theme params use a literal value) and appends to `url` as query string. `GET /` - list (filters `app_type`/`mount_location`/`enabled_only`/`search`); `GET /{app_id}` - read one (both open to any authenticated user); `POST /`, `PUT /{app_id}`, `DELETE /{app_id}` - **admin only**; iframe apps must carry a `mount_location` (400 otherwise); the unique `name` collides as HTTP 409. Backed by `deerflow.persistence.light_apps` (table `light_apps`, auto-created by `create_all`); store wired in `deps.py` as `app.state.light_app_store`. Tests: `tests/test_light_apps.py`. | -| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` ? `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "???????". The "??????" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/file` - **import an uploaded file** (multipart: `file` + optional `title`/`tags`/`status`/`model_name`) into wiki note(s): the file is converted to Markdown (md/txt read directly, pdf/word/ppt/excel via `convert_file_to_markdown`), distilled by the same wiki-capture LLM pipeline as thread capture (`source_type="file"`), and falls back to a verbatim **draft** note when the model returns nothing usable (`llm_fallback:true`); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector?keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities, graph-blacklist filtered); `GET/POST /graph/blacklist` + `DELETE /graph/blacklist/{id}` - graph node blacklist (labels = entity names / note titles, case-insensitive match; blacklisted nodes and their edges are dropped from `/graph` and the wiki-export artifacts; table `knowledge_graph_blacklist`, auto-created); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[?]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). `GET/POST /folders` + `PUT/DELETE /folders/{id}` - user-managed directories (notes carry a `folder` path; `GET /notes?folder=` filters a dir + sub-dirs); `GET/POST /extract-templates` + `GET/PUT/DELETE /extract-templates/{id}` - editable extraction-prompt templates (thread/document ingest accept `template_id`; capture/PUT-note accept `folder`). Backed by the `deerflow.knowledge` module (see below). | +| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Defaults to `background=true`**: the slow per-thread extraction loop is deferred to a FastAPI background task and the endpoint returns `{queued:true, processed:N}` immediately (each thread's transcript is truncated to `max_input_chars` before the LLM call). Running the loop inline for a large batch outlives the nginx gateway timeout (504), which the frontend's `parseResponse` surfaces as a generic "Request failed"; `background=false` keeps the blocking path (full per-thread `created/skipped/failed` tally) for small batches/scripts. Shared loop: `_run_batch_ingest`. **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` ? `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "???????". The "??????" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/file` - **import an uploaded file** (multipart: `file` + optional `title`/`tags`/`status`/`model_name`) into wiki note(s): the file is converted to Markdown (md/txt read directly, pdf/word/ppt/excel via `convert_file_to_markdown`), distilled by the same wiki-capture LLM pipeline as thread capture (`source_type="file"`), and falls back to a verbatim **draft** note when the model returns nothing usable (`llm_fallback:true`); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector?keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities, graph-blacklist filtered); `GET/POST /graph/blacklist` + `DELETE /graph/blacklist/{id}` - graph node blacklist (labels = entity names / note titles, case-insensitive match; blacklisted nodes and their edges are dropped from `/graph` and the wiki-export artifacts; table `knowledge_graph_blacklist`, auto-created); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[?]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). `GET/POST /folders` + `PUT/DELETE /folders/{id}` - user-managed directories (notes carry a `folder` path; `GET /notes?folder=` filters a dir + sub-dirs); `GET/POST /extract-templates` + `GET/PUT/DELETE /extract-templates/{id}` - editable extraction-prompt templates (thread/document ingest accept `template_id`; capture/PUT-note accept `folder`). Backed by the `deerflow.knowledge` module (see below). | Proxied through nginx: `/api/langgraph/*` ? LangGraph, all other `/api/*` ? Gateway. @@ -846,10 +846,20 @@ See [docs/plan_mode_usage.md](docs/plan_mode_usage.md) for details. ### Context Summarization Automatic conversation summarization when approaching token limits: -- Configured in `config.yaml` under `summarization` key -- Trigger types: tokens, messages, or fraction of max input -- Keeps recent messages while summarizing older ones +- **On/off switch is per-request, not the config file.** The chat input box has a + 「上下文压缩」 toggle (below 参考文献, **default off**) → frontend sends + `context.summarization_enabled` → whitelisted in `_CONTEXT_CONFIGURABLE_KEYS` + (`app/gateway/services.py`) → `_build_middlewares` reads it from the run config + and calls `_create_summarization_middleware(force_enabled=summarization_enabled)`. + `force_enabled=True` mounts the middleware even when `config.summarization.enabled` + is false; when the flag is absent (channels / scheduled runs) it falls back to + the config flag. `config.summarization.enabled` now defaults to **false**. +- `config.yaml` `summarization` still supplies the **tuning**: trigger types + (tokens, messages, or fraction of max input) and the keep policy. +- Keeps recent messages while summarizing older ones. +Tests: `tests/test_summarization_toggle.py` (gating + whitelist), +`tests/test_summarization_middleware.py` (cache behavior). See [docs/summarization.md](docs/summarization.md) for details. ### Vision Support / Image Recognition (??) diff --git a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py index 0c036816..b3af7de8 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py +++ b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py @@ -65,6 +65,10 @@ class BatchIngestRequest(BaseModel): include_sources: bool = True skip_existing: bool = True mode: str = "summary" + background: bool = Field( + default=True, + description="When true (default), the per-thread extraction loop runs in a background task and the endpoint returns immediately ({queued:true}). Batch capture of many threads is far too slow to finish inside one HTTP request — running it inline makes nginx time out (504), which the frontend surfaces as a generic 'request failed'.", + ) model_name: str | None = Field(default=None, description="可选:指定用于知识提炼的大模型名称;留空使用默认配置") template_id: str | None = Field(default=None, description="可选:抽取提示词模板 id;留空使用对话默认模板") folder: str | None = Field(default=None, description="可选:沉淀结果归入的目录(斜杠分隔路径)") @@ -285,15 +289,14 @@ async def ingest_thread(thread_id: str, body: IngestThreadRequest, request: Requ } -@router.post("/ingest/threads/batch") -@require_auth -async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> dict[str, Any]: - """Batch-sediment the caller's recent threads into knowledge notes.""" - service = get_knowledge_service(request) - checkpointer = get_checkpointer(request) - thread_store = get_thread_store(request) +async def _run_batch_ingest(service, checkpointer, threads: list[dict[str, Any]], body: BatchIngestRequest, created_by: str | None) -> dict[str, Any]: + """Run the per-thread sediment loop and tally the outcome. - threads = await thread_store.search(limit=body.limit, offset=body.offset) + Shared by the inline and background paths; ``created_by`` is passed in + explicitly because the user contextvar is gone once a background task runs. + Each thread's transcript is truncated by the extractor (``max_input_chars``) + before the LLM call, so a single huge conversation can't blow the context. + """ created = skipped = failed = 0 # Notes that landed via the rule-based fallback because the chosen model # produced nothing usable (timeout / API error / non-JSON). Surfaced so the @@ -319,7 +322,7 @@ async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> di status=body.status, include_sources=body.include_sources, reference_batches=batches, - created_by=get_effective_user_id(), + created_by=created_by, skip_existing=body.skip_existing, model_name=body.model_name, template_id=body.template_id, @@ -347,6 +350,49 @@ async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> di return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "llm_fallback": llm_fallback, "items": items} +async def _ingest_threads_batch_background(service, checkpointer, threads: list[dict[str, Any]], body: BatchIngestRequest, created_by: str | None) -> None: + """Background variant of the batch loop (failures swallowed/logged).""" + try: + result = await _run_batch_ingest(service, checkpointer, threads, body, created_by) + logger.info( + "Background batch ingest finished: processed=%d created=%d skipped=%d failed=%d llm_fallback=%d", + result["processed"], result["created"], result["skipped"], result["failed"], result["llm_fallback"], + ) + except Exception: + logger.exception("Background batch ingest failed") + + +@router.post("/ingest/threads/batch") +@require_auth +async def ingest_threads_batch(body: BatchIngestRequest, request: Request, background_tasks: BackgroundTasks) -> dict[str, Any]: + """Batch-sediment the caller's recent threads into knowledge notes. + + Defaults to ``background=true``: the (slow) per-thread extraction runs in a + background task and the endpoint returns ``{"queued": true, "processed": N}`` + immediately, so the request never outlives the nginx gateway timeout. Pass + ``background=false`` to block until every thread is processed (small batches / + scripts that want the full per-thread tally). + """ + service = get_knowledge_service(request) + checkpointer = get_checkpointer(request) + thread_store = get_thread_store(request) + + threads = await thread_store.search(limit=body.limit, offset=body.offset) + if body.background: + # Capture the user id now — the contextvar is gone once the task runs. + background_tasks.add_task( + _ingest_threads_batch_background, + service, + checkpointer, + threads, + body, + get_effective_user_id(), + ) + return {"ok": True, "queued": True, "processed": len(threads), "created": 0, "skipped": 0, "failed": 0, "llm_fallback": 0, "items": []} + + return await _run_batch_ingest(service, checkpointer, threads, body, get_effective_user_id()) + + async def _extract_upload_text(file: UploadFile) -> tuple[str, str]: """Read an uploaded file and return ``(filename, markdown_text)``. @@ -476,7 +522,7 @@ async def list_notes( offset: int = 0, ) -> dict[str, Any]: service = get_knowledge_service(request) - limit = max(1, min(limit, 100)) + limit = max(1, min(limit, 1000)) offset = max(0, offset) return await service.list_notes(keyword=keyword, tag=tag, entity=entity, source_type=source_type, status=status, folder=folder, limit=limit, offset=offset) diff --git a/offline-backend-20260512/backend/app/gateway/services.py b/offline-backend-20260512/backend/app/gateway/services.py index 12c8781b..362c5b9c 100644 --- a/offline-backend-20260512/backend/app/gateway/services.py +++ b/offline-backend-20260512/backend/app/gateway/services.py @@ -243,6 +243,9 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset( # 关掉个人记忆召回 / builtin 记忆注入(见 multi_agent._run_payload)。 "memory_recall_disabled", "memory_injection_enabled", + # 上下文压缩开关:由聊天框下方的按钮控制(默认关闭),透传到 run config 后 + # 在 _build_middlewares 里决定是否挂载 SummarizationMiddleware。 + "summarization_enabled", } ) diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index 44794ead..0ca799fe 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -200,7 +200,10 @@ title: # grep「上下文压缩」/「context compaction」确认生效;确认后再调回生产值 # (trigger tokens 60000 / messages 80,keep tokens 16000)。 summarization: - enabled: true + # 开关已移到聊天框下方(参考文献下面)的「上下文压缩」按钮,默认关闭。 + # 这里 enabled 仅作为非 Web 路径(渠道/定时任务)的后备开关;下面的 trigger/keep + # 等仅是调参,Web 端是否压缩由前端按钮(summarization_enabled)决定。 + enabled: false # model_name: deepseek-chat # 留空则用主对话模型 trigger: - type: tokens diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py index 88824e72..00a2d1cb 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py @@ -54,12 +54,19 @@ def _resolve_model_name(requested_model_name: str | None = None, *, app_config: return default_model_name -def _create_summarization_middleware(*, app_config: AppConfig | None = None) -> DeerFlowSummarizationMiddleware | None: - """Create and configure the summarization middleware from config.""" +def _create_summarization_middleware(*, app_config: AppConfig | None = None, force_enabled: bool = False) -> DeerFlowSummarizationMiddleware | None: + """Create and configure the summarization middleware from config. + + Args: + force_enabled: When True, build the middleware regardless of the config + ``summarization.enabled`` flag. The per-request 上下文压缩 toggle + (chat input box, default off) drives this — the config file no longer + owns the on/off switch, only the tuning (trigger/keep/...). + """ resolved_app_config = app_config or get_app_config() config = resolved_app_config.summarization - if not config.enabled: + if not (config.enabled or force_enabled): return None # Prepare trigger parameter @@ -268,8 +275,13 @@ def _build_middlewares( if cfg.get("rag_mode_enabled", False): middlewares.append(RagCitationMiddleware()) - # Add summarization middleware if enabled - summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config) + # Add summarization middleware. The on/off switch now lives in the chat UI + # (上下文压缩 toggle, default off) and arrives as ``summarization_enabled`` in + # the runtime config; the config file only supplies tuning. force_enabled + # lets the per-request toggle turn it on even when config.enabled is false, + # while non-UI paths (channels/scheduled) still fall back to config.enabled. + summarization_requested = cfg.get("summarization_enabled", False) + summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config, force_enabled=summarization_requested) if summarization_middleware is not None: middlewares.append(summarization_middleware) diff --git a/offline-backend-20260512/config.yaml b/offline-backend-20260512/config.yaml index 6070507e..be0d5b21 100644 --- a/offline-backend-20260512/config.yaml +++ b/offline-backend-20260512/config.yaml @@ -303,7 +303,10 @@ log_system: label: "日志系统" summarization: - enabled: true + # 开关已移到聊天框下方(参考文献下面)的「上下文压缩」按钮,默认关闭。 + # 这里 enabled 仅作为非 Web 路径(渠道/定时任务)的后备开关;下面的 trigger/keep + # 等仅是调参,Web 端是否压缩由前端按钮(summarization_enabled)决定。 + enabled: false # model_name: deepseek-chat trigger: - type: tokens