解决 artifact 预览组件冲突:mermaid 渲染(dhk-1) 与 markdown 锚点跳转(master) 取并集;package-lock.json 按 master 删除(项目用 pnpm)。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
董宏凯andClaude Opus 4.8 committed 2026-06-05 19:43:02 +08:00
commit b2132dd668
104 files changed
+8775 -12038

No files matched your search

@@ -1,8 +1,10 @@
"""Middleware to inject uploaded files information into agent context."""
import logging
from pathlib import Path
from typing import NotRequired, override
import logging
import shutil
from pathlib import Path
from typing import NotRequired, override
from urllib.parse import unquote
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
@@ -13,7 +15,7 @@ from deerflow.config.paths import Paths, get_paths
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.file_conversion import extract_outline
logger = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
_OUTLINE_PREVIEW_LINES = 5
@@ -22,7 +24,8 @@ _OUTLINE_PREVIEW_LINES = 5
# configured vision model), never by reading their bytes. Listing them with the
# generic read_file/grep guidance made the agent try to parse image binaries
# with bash/python, which fails and wastes turns.
_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"}
_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"}
_ARTIFACT_URL_MARKER = "/artifacts/"
def _extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]:
@@ -69,7 +72,7 @@ class UploadsMiddlewareState(AgentState):
uploaded_files: NotRequired[list[dict] | None]
class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
"""Middleware to inject uploaded files information into the agent context.
Reads file metadata from the current message's additional_kwargs.files
@@ -122,7 +125,7 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
lines.append(" Use `grep` to search for keywords (e.g. `grep(pattern='keyword', path='/mnt/user-data/uploads/')`).")
lines.append("")
def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str:
def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str:
"""Create a formatted message listing uploaded files.
Args:
@@ -160,9 +163,61 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
lines.append("- Only fall back to web search if the file content is clearly insufficient to answer the question.")
lines.append("</uploaded_files>")
return "\n".join(lines)
def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None:
return "\n".join(lines)
def _import_file_from_artifact_url(
self,
file_info: dict,
uploads_dir: Path | None,
) -> bool:
"""Copy a previously uploaded file into the current thread when possible.
iframe-style callers may pass file metadata that points at an upload
from another DeerFlow thread. The current run thread still needs a real
file under its own ``/mnt/user-data/uploads`` directory, otherwise this
middleware will filter the entry out during existence checks.
"""
filename = file_info.get("filename") or ""
artifact_url = str(file_info.get("artifact_url") or "").strip()
if not filename or uploads_dir is None or not artifact_url:
return False
if not artifact_url.startswith("/api/threads/") or _ARTIFACT_URL_MARKER not in artifact_url:
return False
prefix, encoded_virtual_path = artifact_url.split(_ARTIFACT_URL_MARKER, 1)
source_thread_id = prefix.removeprefix("/api/threads/").strip("/")
if not source_thread_id or not encoded_virtual_path:
return False
virtual_path = "/" + unquote(encoded_virtual_path.lstrip("/"))
if Path(virtual_path).name != filename:
return False
try:
source_path = self._paths.resolve_virtual_path(
source_thread_id,
virtual_path,
user_id=get_effective_user_id(),
)
except ValueError:
logger.debug("Failed to resolve uploaded artifact path: %s", artifact_url, exc_info=True)
return False
if not source_path.is_file():
return False
uploads_dir.mkdir(parents=True, exist_ok=True)
target_path = uploads_dir / filename
shutil.copy2(source_path, target_path)
source_markdown = source_path.with_suffix(".md")
if source_markdown.is_file() and source_markdown != source_path:
shutil.copy2(source_markdown, target_path.with_suffix(".md"))
return True
def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None:
"""Extract file info from message additional_kwargs.files.
The frontend sends uploaded file metadata in additional_kwargs.files
@@ -183,16 +238,17 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
files = []
for f in kwargs_files:
if not isinstance(f, dict):
continue
filename = f.get("filename") or ""
if not filename or Path(filename).name != filename:
continue
if uploads_dir is not None and not (uploads_dir / filename).is_file():
continue
files.append(
{
"filename": filename,
if not isinstance(f, dict):
continue
filename = f.get("filename") or ""
if not filename or Path(filename).name != filename:
continue
if uploads_dir is not None and not (uploads_dir / filename).is_file():
if not self._import_file_from_artifact_url(f, uploads_dir):
continue
files.append(
{
"filename": filename,
"size": int(f.get("size") or 0),
"path": f"/mnt/user-data/uploads/{filename}",
"extension": Path(filename).suffix,
@@ -243,6 +299,17 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
# Get newly uploaded files from the current message's additional_kwargs.files
new_files = self._files_from_kwargs(last_message, uploads_dir) or []
# Only inject when the *current* message actually carries new uploads.
# Turns without a fresh upload must not re-inject previously uploaded
# files: the original upload turn already injected (and persisted) their
# <uploaded_files> block, and the files remain physically available for
# read_file/grep/view_image. Re-injecting on every later message made the
# agent re-reference old images/docs, and — since the frontend parses
# files out of the <uploaded_files> content when additional_kwargs.files
# is empty — re-rendered the old attachment on each new question.
if not new_files:
return None
# Collect historical files from the uploads directory (all except the new ones)
new_filenames = {f["filename"] for f in new_files}
historical_files: list[dict] = []
@@ -270,9 +337,6 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
file["outline"] = outline
file["outline_preview"] = preview
if not new_files and not historical_files:
return None
logger.debug(f"New files: {[f['filename'] for f in new_files]}, historical: {[f['filename'] for f in historical_files]}")
# Create files message and prepend to the last human message content
@@ -0,0 +1,92 @@
"""Server-side mirror of the frontend agent-name desensitization map.
The web UI displays agent names through a static term→abbreviation map
(``frontend-web/src/lib/desensitize.ts``) — e.g. the stored name "态势分析" is
shown as "TS分析". The gallery search box therefore matches against the
*desensitized* display form, not the raw stored name.
When search moves server-side (paginated listing), the backend must match the
same way, so this module mirrors that static map verbatim. The frontend map is
never extended at runtime (``addDesensitizeEntries`` is defined but never
called), so a static copy stays in sync.
Used by the SQL listing query to match "raw name OR desensitized name".
"""
from __future__ import annotations
# Mirror of DESENSITIZE_MAP in frontend-web/src/lib/desensitize.ts.
# Keep entries identical to the frontend so server-side and client-side search
# behave the same. Order does not affect membership tests.
DESENSITIZE_MAP: dict[str, str] = {
"态势": "TS",
"你好": "nh",
"滨海作战体系": "第一张网",
"滨海作战部队": "第一张网",
"Taiwan Air Force Flight Units": "第二张网",
"台海空军飞行部队": "第二张网",
"空军": "KJ",
"海军": "HJ",
"陆军": "LJ",
"陆战队": "LZD",
"海巡": "HX",
"憲兵": "XB",
"資通電軍": "ZTDJ",
"情报": "QB",
"指挥": "ZH",
"信息准备": "XX准备",
"行为体": "XWT",
"驱动因素": "QDYS",
"叙事": "XS",
"前三定": "QSD",
"受众目标": "SZMB",
"作战": "ZZ",
"目的": "MD",
"关键传播节点": "GJCBJD",
"战法": "ZF",
"行动": "XD",
"打击": "DJ",
"目标": "MB",
"战力": "ZL",
"筹划": "COH",
"策划": "CEH",
"模拟": "MN",
"推演": "TY",
"认知": "RZ",
"触及": "CJ",
"军": "J",
"美军": "MJ",
"台军": "TJ",
}
def desensitize_name(name: str) -> str:
"""Return ``name`` with every mapped term replaced by its abbreviation.
Mirrors the frontend ``desensitize()`` — a sequence of literal replacements.
"""
out = name
for term, abbr in DESENSITIZE_MAP.items():
if term in out:
out = out.replace(term, abbr)
return out
def name_matches(name: str, needle: str) -> bool:
"""True when ``needle`` (case-insensitive) is in the raw OR desensitized name."""
n = needle.strip().lower()
if not n:
return True
return n in name.lower() or n in desensitize_name(name).lower()
def source_terms_for_needle(needle: str) -> list[str]:
"""Return the raw source terms whose abbreviation contains ``needle``.
Lets the SQL search add ``name LIKE %term%`` clauses so a user typing an
abbreviation (e.g. "TS") still matches the stored raw term (e.g. "态势").
"""
n = needle.strip().lower()
if not n:
return []
return [term for term, abbr in DESENSITIZE_MAP.items() if n in abbr.lower()]
@@ -15,6 +15,7 @@ from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
from deerflow.config.log_system_config import LogSystemConfig
from deerflow.config.login_config import AuthLoginConfig
from deerflow.config.memory_config import MemoryConfig, load_memory_config_from_dict
from deerflow.config.model_config import ModelConfig
@@ -108,6 +109,7 @@ class AppConfig(BaseModel):
models: list[ModelConfig] = Field(default_factory=list, description="Available models")
vision: VisionConfig = Field(default_factory=VisionConfig, description="Image recognition (识图) configuration")
auth_login: AuthLoginConfig = Field(default_factory=AuthLoginConfig, description="Custom login modes (username password gate, token exchange)")
log_system: LogSystemConfig = Field(default_factory=LogSystemConfig, description="Admin-only external log-system entry shown in the settings menu")
sandbox: SandboxConfig = Field(description="Sandbox configuration")
tools: list[ToolConfig] = Field(default_factory=list, description="Available tools")
tool_groups: list[ToolGroupConfig] = Field(default_factory=list, description="Available tool groups")
@@ -0,0 +1,18 @@
"""Configuration for the admin-only external log-system entry.
A small read-only section in ``config.yaml`` that lets a deployment point the
"日志系统" entry (shown in the bottom-right settings menu for admins) at an
external log/observability system. The URL is opened in a new browser tab.
"""
from __future__ import annotations
from pydantic import BaseModel, Field
class LogSystemConfig(BaseModel):
"""External log-system link surfaced in the admin settings menu."""
enabled: bool = Field(default=False, description="Whether the log-system entry is shown in the settings menu")
url: str = Field(default="", description="External log-system URL opened in a new tab when the entry is clicked")
label: str = Field(default="日志系统", description="Menu label shown for the entry")
@@ -78,6 +78,17 @@ class PrivateSquareSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class LeaderboardSettings(BaseModel):
"""Admin-configurable defaults for the user leaderboard analytics."""
exclude_user_ids: list[str] = Field(default_factory=list, description="User ids excluded from leaderboard aggregation.")
include_scheduled: bool = Field(default=True, description="Whether scheduled task runs participate in leaderboard statistics.")
include_admins: bool = Field(default=True, description="Whether admin accounts participate in leaderboard statistics.")
include_failed: bool = Field(default=True, description="Whether failed runs and provider calls participate in leaderboard statistics.")
cleanup_system_questions: bool = Field(default=True, description="Hide attachment/system marker questions such as <uploaded_files>.")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -85,6 +96,7 @@ class SystemSettings(BaseModel):
private_square: PrivateSquareSettings = Field(default_factory=PrivateSquareSettings)
appearance: AppearanceSettings = Field(default_factory=AppearanceSettings)
citation_display: CitationDisplaySettings = Field(default_factory=CitationDisplaySettings)
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
model_config = ConfigDict(extra="ignore")
@@ -4,13 +4,20 @@ from __future__ import annotations
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.memory import MemoryAgentStore
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.agents.sql import AgentRepository
__all__ = [
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRepository",
"AgentRow",
"AgentSkillRow",
"AgentStore",
"MemoryAgentStore",
"make_agent_store",
@@ -118,6 +118,69 @@ class AgentStore(ABC):
"""
raise NotImplementedError
@abstractmethod
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
"""Return one page of agents plus the total matching count.
``scope``:
- ``"mine"`` — owned by ``user_id`` OR favorited by ``user_id``.
- ``"square"`` — built-in or published, and not private-square.
- ``"private"``— flagged ``is_private=True`` (caller gates access).
- ``None`` — the default listing visibility (``list_visible``).
``admin=True`` bypasses the visibility/scope filter entirely (every
agent), but still honors ``search`` / ``tag_ids``. Caller authorizes.
``search`` matches the raw name OR its desensitized display form.
``tag_ids`` keeps agents carrying ANY of the tags (OR). Both filters,
the COUNT, and LIMIT/OFFSET all execute in SQL.
Returns ``(rows, total)`` where ``rows`` is at most ``page_size`` items
for the 1-based ``page``.
"""
raise NotImplementedError
# --- Listing cache (skills / tool_groups / model) ---
@abstractmethod
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
"""Replace the cached skill list for ``agent_id``.
``None`` and ``[]`` both clear the cache to "no skills" (mirrors the
tri-state semantics of ``AgentConfig.skills``).
"""
raise NotImplementedError
@abstractmethod
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
"""Upsert the cached ``tool_groups`` / ``model`` for ``agent_id``."""
raise NotImplementedError
@abstractmethod
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
"""Batch-fetch cached listing extras for ``agent_ids``.
Returns ``{agent_id: {"skills": list[str] | None, "tool_groups":
list[str] | None, "model": str | None}}``. An agent that was never
mirrored is absent from the mapping (caller treats it as unknown).
"""
raise NotImplementedError
@abstractmethod
async def delete_extras_for_agent(self, agent_id: str) -> None:
"""Drop the cached skills + extras rows for ``agent_id`` (on delete)."""
raise NotImplementedError
# --- Favorites ("收藏") ---
@abstractmethod
@@ -136,6 +199,28 @@ class AgentStore(ABC):
raise NotImplementedError
@abstractmethod
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
async def delete_favorites_for_agent(self, agent_id: str) -> None:
"""Drop every user's favorite of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
# --- Per-user pins ("置顶") ---
@abstractmethod
async def add_pin(self, user_id: str, agent_id: str) -> None:
"""Pin ``agent_id`` for ``user_id``. Idempotent."""
raise NotImplementedError
@abstractmethod
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
"""Remove a per-user pin. Returns ``False`` when none existed."""
raise NotImplementedError
@abstractmethod
async def list_pin_ids(self, user_id: str) -> set[str]:
"""Return the set of agent ids pinned by ``user_id``."""
raise NotImplementedError
@abstractmethod
async def delete_pins_for_agent(self, agent_id: str) -> None:
"""Drop every user's pin of ``agent_id`` (called on agent delete)."""
raise NotImplementedError
@@ -5,6 +5,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.config.agent_name_desensitize import name_matches
from deerflow.persistence.agents.base import AgentStore
@@ -20,8 +21,12 @@ def _copy(data: dict[str, Any]) -> dict[str, Any]:
class MemoryAgentStore(AgentStore):
def __init__(self) -> None:
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._agents: dict[str, dict[str, Any]] = {}
self._favorites: set[tuple[str, str]] = set()
self._pins: set[tuple[str, str]] = set()
# Listing cache mirrors (see AgentExtrasRow / AgentSkillRow):
self._skills: dict[str, list[str]] = {}
self._extras: dict[str, dict[str, Any]] = {}
@staticmethod
def _is_visible(agent: dict[str, Any], user_id: str) -> bool:
@@ -166,6 +171,96 @@ class MemoryAgentStore(AgentStore):
self._agents.pop(agent_id, None)
return True
# --- Paginated listing ---
def _matches_scope(self, agent: dict[str, Any], *, user_id: str, scope: str | None, admin: bool) -> bool:
if admin:
return True
if scope == "square":
return (agent.get("user_id") is None or bool(agent.get("published"))) and not bool(agent.get("is_private"))
if scope == "mine":
owned = agent.get("user_id") is not None and agent.get("user_id") == user_id
favorited = (user_id, agent["id"]) in self._favorites
return owned or favorited
if scope == "private":
return bool(agent.get("is_private"))
return self._is_visible(agent, user_id)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
# NOTE: the in-memory store has no tag-assignment data, so ``tag_ids``
# is ignored here. Tag-filter behavior is covered by the SQL repo tests.
matched = [
a
for a in self._agents.values()
if self._matches_scope(a, user_id=user_id, scope=scope, admin=admin)
and (not search or not search.strip() or name_matches(a.get("name") or "", search))
]
if sort == "time_asc":
matched.sort(key=lambda a: (a.get("created_at") or "", a.get("id") or ""))
elif sort == "updated_desc":
matched.sort(
key=lambda a: (a.get("updated_at") or "", a.get("id") or ""),
reverse=True,
)
elif sort == "updated_asc":
matched.sort(key=lambda a: (a.get("updated_at") or "", a.get("id") or ""))
elif sort == "name_asc":
matched.sort(key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""))
elif sort == "name_desc":
matched.sort(
key=lambda a: ((a.get("name") or "").lower(), a.get("id") or ""),
reverse=True,
)
else:
matched.sort(
key=lambda a: (a.get("created_at") or "", a.get("id") or ""),
reverse=True,
)
pin_ids = {agent_id for uid, agent_id in self._pins if uid == user_id}
matched.sort(key=lambda a: (0 if a.get("id") in pin_ids else 1,))
total = len(matched)
start = max(page - 1, 0) * page_size
page_rows = matched[start : start + page_size]
return [_copy(a) for a in page_rows], total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
self._skills[agent_id] = [s for s in (skills or []) if isinstance(s, str)]
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
self._extras[agent_id] = {"tool_groups": tool_groups, "model": model}
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
out: dict[str, dict[str, Any]] = {}
for agent_id in agent_ids:
has_extras = agent_id in self._extras
has_skills = agent_id in self._skills
if not has_extras and not has_skills:
continue
extras = self._extras.get(agent_id, {})
out[agent_id] = {
"skills": list(self._skills.get(agent_id, [])),
"tool_groups": extras.get("tool_groups"),
"model": extras.get("model"),
}
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
self._skills.pop(agent_id, None)
self._extras.pop(agent_id, None)
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
@@ -180,5 +275,23 @@ class MemoryAgentStore(AgentStore):
async def list_favorite_ids(self, user_id: str) -> set[str]:
return {agent_id for (uid, agent_id) in self._favorites if uid == user_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
self._favorites = {(uid, aid) for (uid, aid) in self._favorites if aid != agent_id}
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
self._pins.add((user_id, agent_id))
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
key = (user_id, agent_id)
if key not in self._pins:
return False
self._pins.remove(key)
return True
async def list_pin_ids(self, user_id: str) -> set[str]:
return {agent_id for uid, agent_id in self._pins if uid == user_id}
async def delete_pins_for_agent(self, agent_id: str) -> None:
self._pins = {(uid, aid) for (uid, aid) in self._pins if aid != agent_id}
@@ -4,11 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, String, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class AgentRow(Base):
@@ -37,7 +37,7 @@ class AgentRow(Base):
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
class AgentFavoriteRow(Base):
class AgentFavoriteRow(Base):
"""A user's bookmark of a (typically public-square) agent.
Lets a user collect agents from the 广场 (square) into their "我的" tab
@@ -51,4 +51,60 @@ class AgentFavoriteRow(Base):
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentPinRow(Base):
"""A per-user pin that keeps an agent near the top of that user's lists."""
__tablename__ = "agent_pins"
__table_args__ = (UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class AgentSkillRow(Base):
"""Denormalized cache of a custom agent's selected skills.
``config.yaml`` under ``.deer-flow/agents/{id}/`` stays the source of truth
for the agent runtime (the lead-agent prompt reads it directly). This table
is a read cache so the gallery list endpoint can render skill badges with a
single batched ``IN`` query instead of stat+YAML-parsing every agent's
``config.yaml`` on each request. Written through on create/update and
backfilled from disk by ``_sync_legacy_agents``.
One row per (agent, skill); ``position`` preserves the configured order.
"""
__tablename__ = "agent_skills"
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
skill_name: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
position: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
class AgentExtrasRow(Base):
"""Denormalized cache of a custom agent's ``tool_groups`` / ``model``.
Companion to :class:`AgentSkillRow` — same rationale (avoid per-agent
``config.yaml`` reads in the list path). One row per agent. The presence of
a row also marks "this agent has been mirrored": the list path treats a
missing row as "unknown / not yet backfilled" (``skills=None``) so it can
be told apart from an explicitly empty skill list.
``skills_synced_at`` records when the cache was last written from disk so
the backfill can skip agents whose ``config.yaml`` has not changed.
"""
__tablename__ = "agent_extras"
agent_id: Mapped[str] = mapped_column(String(64), primary_key=True)
# list[str] | None — the agent's tool_groups whitelist from config.yaml.
tool_groups: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True, default=None)
model: Mapped[str | None] = mapped_column(String(191), nullable=True, default=None)
skills_synced_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -6,12 +6,20 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, delete, func, or_, select
from sqlalchemy import and_, case, delete, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.agent_name_desensitize import source_terms_for_needle
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.tags.model import TagAssignmentRow
def _row_to_dict(row: AgentRow) -> dict[str, Any]:
@@ -233,6 +241,167 @@ class AgentRepository(AgentStore):
await session.commit()
return True
# --- Paginated listing ---
def _scope_filter(self, *, user_id: str, scope: str | None, admin: bool):
"""Build the visibility predicate for a paginated listing."""
if admin:
return None # no visibility filter — every agent
if scope == "square":
return and_(
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
)
if scope == "mine":
# owned OR favorited. A correlated IN-subquery (not a JOIN) keeps a
# single row per agent even when it is both owned and favorited.
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
return or_(AgentRow.user_id == user_id, AgentRow.id.in_(fav_subq))
if scope == "private":
return AgentRow.is_private.is_(True)
return self._listing_filter(user_id)
@staticmethod
def _search_clause(search: str):
"""Match the raw name OR (via the desensitize mirror) its display form."""
needle = search.strip().lower()
clauses = [func.lower(AgentRow.name).like(f"%{needle}%")]
for term in source_terms_for_needle(needle):
clauses.append(AgentRow.name.like(f"%{term}%"))
return or_(*clauses)
@staticmethod
def _tag_clause(tag_ids: list[str]):
subq = select(TagAssignmentRow.target_id).where(
TagAssignmentRow.target_type == "agent",
TagAssignmentRow.tag_id.in_(tag_ids),
)
return AgentRow.id.in_(subq)
async def list_paginated(
self,
*,
user_id: str,
scope: str | None,
search: str | None,
tag_ids: list[str] | None,
page: int,
page_size: int,
admin: bool = False,
sort: str = "time_desc",
) -> tuple[list[dict[str, Any]], int]:
filters = []
scope_filter = self._scope_filter(user_id=user_id, scope=scope, admin=admin)
if scope_filter is not None:
filters.append(scope_filter)
if search and search.strip():
filters.append(self._search_clause(search))
if tag_ids:
filters.append(self._tag_clause(tag_ids))
count_stmt = select(func.count()).select_from(AgentRow)
pin_subq = select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
pin_rank = case((AgentRow.id.in_(pin_subq), 0), else_=1)
if sort == "time_asc":
order_by = (AgentRow.created_at.asc(), AgentRow.id.asc())
elif sort == "updated_desc":
order_by = (AgentRow.updated_at.desc(), AgentRow.id.asc())
elif sort == "updated_asc":
order_by = (AgentRow.updated_at.asc(), AgentRow.id.asc())
elif sort == "name_asc":
order_by = (func.lower(AgentRow.name).asc(), AgentRow.id.asc())
elif sort == "name_desc":
order_by = (func.lower(AgentRow.name).desc(), AgentRow.id.asc())
else:
order_by = (AgentRow.created_at.desc(), AgentRow.id.asc())
page_stmt = (
select(AgentRow)
.order_by(pin_rank.asc(), *order_by)
.limit(page_size)
.offset(max(page - 1, 0) * page_size)
)
if filters:
where = and_(*filters)
count_stmt = count_stmt.where(where)
page_stmt = page_stmt.where(where)
async with self._sf() as session:
total = await session.scalar(count_stmt) or 0
result = await session.execute(page_stmt)
rows = [_row_to_dict(row) for row in result.scalars()]
return rows, total
# --- Listing cache (skills / tool_groups / model) ---
async def replace_skills(self, agent_id: str, skills: list[str] | None) -> None:
async with self._sf() as session:
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
for position, name in enumerate(skills or []):
if not isinstance(name, str):
continue
session.add(
AgentSkillRow(
id=uuid4().hex,
agent_id=agent_id,
skill_name=name,
position=position,
)
)
await session.commit()
async def set_extras(self, agent_id: str, *, tool_groups: list[str] | None, model: str | None) -> None:
async with self._sf() as session:
row = await session.get(AgentExtrasRow, agent_id)
now = datetime.now(UTC)
if row is None:
session.add(
AgentExtrasRow(
agent_id=agent_id,
tool_groups=tool_groups,
model=model,
skills_synced_at=now,
)
)
else:
row.tool_groups = tool_groups
row.model = model
row.skills_synced_at = now
await session.commit()
async def get_extras_for(self, agent_ids: list[str]) -> dict[str, dict[str, Any]]:
if not agent_ids:
return {}
out: dict[str, dict[str, Any]] = {}
async with self._sf() as session:
extras = await session.execute(
select(AgentExtrasRow).where(AgentExtrasRow.agent_id.in_(agent_ids))
)
for row in extras.scalars():
out[row.agent_id] = {
"skills": [], # default; filled below if skill rows exist
"tool_groups": row.tool_groups,
"model": row.model,
}
skill_rows = await session.execute(
select(AgentSkillRow.agent_id, AgentSkillRow.skill_name)
.where(AgentSkillRow.agent_id.in_(agent_ids))
.order_by(AgentSkillRow.agent_id.asc(), AgentSkillRow.position.asc())
)
for agent_id, skill_name in skill_rows.all():
entry = out.setdefault(
agent_id, {"skills": [], "tool_groups": None, "model": None}
)
entry["skills"].append(skill_name)
return out
async def delete_extras_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentSkillRow).where(AgentSkillRow.agent_id == agent_id))
await session.execute(delete(AgentExtrasRow).where(AgentExtrasRow.agent_id == agent_id))
await session.commit()
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
@@ -272,7 +441,49 @@ class AgentRepository(AgentStore):
)
return {row[0] for row in result.all()}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentFavoriteRow).where(AgentFavoriteRow.agent_id == agent_id))
await session.commit()
# --- Pins ("置顶") ---
async def add_pin(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentPinRow.id)
.where(AgentPinRow.user_id == user_id, AgentPinRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentPinRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
await session.rollback()
async def remove_pin(self, user_id: str, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(AgentPinRow).where(
AgentPinRow.user_id == user_id,
AgentPinRow.agent_id == agent_id,
)
)
await session.commit()
return (result.rowcount or 0) > 0
async def list_pin_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentPinRow.agent_id).where(AgentPinRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def delete_pins_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
await session.execute(delete(AgentPinRow).where(AgentPinRow.agent_id == agent_id))
await session.commit()
@@ -34,6 +34,9 @@ class LlmMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
# Multiple user ids — set when a fuzzy username search resolves to several
# accounts. Takes precedence over ``user_id`` when non-empty.
user_ids: list[str] | None = None
model_name: str | None = None
status: str | None = None
since: datetime | None = None
@@ -58,7 +58,9 @@ class SqlLlmMetricsStore(LlmMetricsStore):
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: LlmMetricsQuery):
if query.user_id:
if query.user_ids:
stmt = stmt.where(LlmCallMetricRow.user_id.in_(query.user_ids))
elif query.user_id:
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
if query.model_name:
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
@@ -92,6 +94,7 @@ class SqlLlmMetricsStore(LlmMetricsStore):
while True:
page_query = LlmMetricsQuery(
user_id=query.user_id,
user_ids=query.user_ids,
model_name=query.model_name,
status=query.status,
since=query.since,
@@ -0,0 +1,45 @@
"""Add ``agent_id`` to ``user_custom_prompts`` for per-agent prompt scoping.
NULL = global (shown for every agent); otherwise the prompt only surfaces when
that agent is selected.
Revision ID: 20260603_02
Revises: 20260603_01
Create Date: 2026-06-03
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260603_02"
down_revision = "20260603_01"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not _has_table("user_custom_prompts"):
return
if not _has_column("user_custom_prompts", "agent_id"):
op.add_column("user_custom_prompts", sa.Column("agent_id", sa.String(64), nullable=True))
op.create_index("ix_user_custom_prompts_agent_id", "user_custom_prompts", ["agent_id"])
def downgrade() -> None:
if _has_table("user_custom_prompts") and _has_column("user_custom_prompts", "agent_id"):
op.drop_index("ix_user_custom_prompts_agent_id", table_name="user_custom_prompts")
op.drop_column("user_custom_prompts", "agent_id")
@@ -0,0 +1,42 @@
"""Add ``user_preferences`` for per-user UI toggles (account-bound).
Holds a single JSON blob per user — e.g. ``hide_admin_prompt_prefix`` so a user
can choose to use only their own prompts and hide the admin-configured prefix.
Revision ID: 20260603_03
Revises: 20260603_02
Create Date: 2026-06-03
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260603_03"
down_revision = "20260603_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if not _has_table("user_preferences"):
op.create_table(
"user_preferences",
sa.Column("user_id", sa.String(64), primary_key=True),
sa.Column("preferences", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if _has_table("user_preferences"):
op.drop_table("user_preferences")
@@ -0,0 +1,56 @@
"""Per-user pinned agents.
Revision ID: 20260604_01
Revises: 20260603_03
Create Date: 2026-06-04
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260604_01"
down_revision = "20260603_03"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
if not _has_table("agent_pins"):
op.create_table(
"agent_pins",
sa.Column("id", sa.String(64), nullable=False),
sa.Column("user_id", sa.String(64), nullable=False),
sa.Column("agent_id", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("user_id", "agent_id", name="uq_agent_pins_user_agent"),
)
if not _has_index("agent_pins", "ix_agent_pins_user_id"):
op.create_index("ix_agent_pins_user_id", "agent_pins", ["user_id"])
if not _has_index("agent_pins", "ix_agent_pins_agent_id"):
op.create_index("ix_agent_pins_agent_id", "agent_pins", ["agent_id"])
def downgrade() -> None:
if _has_index("agent_pins", "ix_agent_pins_agent_id"):
op.drop_index("ix_agent_pins_agent_id", table_name="agent_pins")
if _has_index("agent_pins", "ix_agent_pins_user_id"):
op.drop_index("ix_agent_pins_user_id", table_name="agent_pins")
if _has_table("agent_pins"):
op.drop_table("agent_pins")
@@ -15,7 +15,12 @@ its storage implementation lives in ``deerflow.runtime.events.store.db`` and
there is no matching entity directory.
"""
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
@@ -44,12 +49,15 @@ from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
__all__ = [
"FeedbackRow",
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
"AgentSkillRow",
"LlmCallMetricRow",
"NotificationRow",
"RecommendedQuestionRow",
@@ -75,5 +83,6 @@ __all__ = [
"ToolCallMetricRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
"UserPreferencesRow",
"UserRow",
]
@@ -205,6 +205,21 @@ class RunRepository(RunStore):
await session.execute(update(RunRow).where(RunRow.run_id == run_id).values(**values))
await session.commit()
async def count_by_user(self) -> dict[str, int]:
"""Return ``{user_id: run_count}`` via a single GROUP BY query.
Each run is one user question, so this is the source for the admin
user center's 问答次数 column. Runs without a ``user_id`` are skipped.
"""
stmt = (
select(RunRow.user_id, func.count().label("cnt"))
.where(RunRow.user_id.is_not(None))
.group_by(RunRow.user_id)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return {r.user_id: r.cnt for r in rows}
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
"""Aggregate token usage via a single SQL GROUP BY query."""
_completed = RunRow.status.in_(("success", "error"))
@@ -0,0 +1,18 @@
"""Per-user UI preferences persistence."""
from __future__ import annotations
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_preferences.sql import UserPreferencesRepository
__all__ = [
"UserPreferencesRepository",
"UserPreferencesRow",
"make_user_preferences_store",
]
def make_user_preferences_store(session_factory):
if session_factory is None:
return None
return UserPreferencesRepository(session_factory)
@@ -0,0 +1,26 @@
"""ORM model for per-user UI preferences.
A single key-value (JSON) row per user, holding small UI-level toggles that
should follow the account across devices — e.g. whether to hide the
admin-configured prompt prefix in the chat input. Kept generic (one JSON blob)
so new toggles do not each require an ALTER TABLE.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class UserPreferencesRow(Base):
__tablename__ = "user_preferences"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
preferences: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -0,0 +1,41 @@
"""SQLAlchemy-backed per-user preferences repository."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.user_preferences.model import UserPreferencesRow
class UserPreferencesRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get_for_user(self, user_id: str) -> dict[str, Any]:
"""Return the user's stored preferences (empty dict when none)."""
async with self._sf() as session:
row = await session.get(UserPreferencesRow, user_id)
if row is None or not isinstance(row.preferences, dict):
return {}
return dict(row.preferences)
async def merge_for_user(self, user_id: str, patch: dict[str, Any]) -> dict[str, Any]:
"""Shallow-merge ``patch`` into the user's preferences and return the result."""
now = datetime.now(UTC)
async with self._sf() as session:
row = await session.get(UserPreferencesRow, user_id)
if row is None:
merged = dict(patch)
row = UserPreferencesRow(user_id=user_id, preferences=merged, created_at=now, updated_at=now)
session.add(row)
else:
merged = dict(row.preferences or {})
merged.update(patch)
# Reassign so SQLAlchemy detects the change on the JSON column.
row.preferences = merged
row.updated_at = now
await session.commit()
return dict(merged)
@@ -19,5 +19,8 @@ class UserCustomPromptRow(Base):
title: Mapped[str] = mapped_column(String(255), nullable=False, default="")
content: Mapped[str] = mapped_column(Text, nullable=False)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# Optional association to a single agent. NULL = global (shown everywhere);
# otherwise the prompt only surfaces when that agent is selected.
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -42,6 +42,7 @@ class UserCustomPromptRepository:
title=data.get("title") or "",
content=data["content"],
sort_order=data.get("sort_order", 0),
agent_id=(data.get("agent_id") or None),
created_at=now,
updated_at=now,
)
@@ -59,6 +60,10 @@ class UserCustomPromptRepository:
for key in ("title", "content", "sort_order"):
if key in data:
setattr(row, key, data[key])
# ``agent_id`` is explicitly nullable: an empty/None value clears the
# association (back to global), so accept None as a real update.
if "agent_id" in data:
row.agent_id = data["agent_id"] or None
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
@@ -93,3 +93,12 @@ class RunStore(abc.ABC):
by_caller ({lead_agent, subagent, middleware}).
"""
pass
async def count_by_user(self) -> dict[str, int]:
"""Return ``{user_id: run_count}`` across all runs.
One run corresponds to one user submission, so this backs the admin
user center's 问答次数 (Q&A count) column — each question counts once.
Non-overriding stores return an empty mapping (treated as 0 per user).
"""
return {}
@@ -38,6 +38,11 @@ _DANGEROUS_URL_RE = re.compile(r"^\s*(javascript|vbscript|data:text/html)", re.I
# Attributes carrying a URL we must scheme-check.
_URL_ATTRS = frozenset({"href", "src", "action", "formaction", "xlink:href", "background", "poster"})
_LOCAL_DOC_LINK_RE = re.compile(
r"^(?![a-z][a-z0-9+.-]*:)(?!//)(?P<path>[^#?]*\.(?:html?|md|markdown))(?P<query>\?[^#]*)?(?P<fragment>#.+)?$",
re.IGNORECASE,
)
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
"""Split a ``---`` YAML frontmatter block from the body of a SKILL.md.
@@ -225,6 +230,38 @@ def sanitize_html(html: str) -> tuple[str, list[str]]:
return str(soup), removed
def constrain_same_page_anchor_links(html: str) -> tuple[str, list[str]]:
"""Keep generated HTML document jumps inside the current HTML file."""
if not html or not isinstance(html, str):
return "", []
try:
from bs4 import BeautifulSoup
except Exception: # pragma: no cover - bs4 is a declared dependency
logger.warning("BeautifulSoup unavailable; returning HTML links unconstrained")
return html, []
changed: list[str] = []
soup = BeautifulSoup(html, "html.parser")
for tag in soup.find_all("a"):
href = tag.get("href")
if not isinstance(href, str):
continue
value = href.strip()
if not value or value.startswith("#"):
continue
match = _LOCAL_DOC_LINK_RE.match(value)
if not match:
continue
fragment = match.group("fragment")
if fragment:
tag["href"] = fragment
changed.append(f"{value}->{fragment}")
else:
del tag.attrs["href"]
changed.append(f"{value}->removed")
return str(soup), changed
def validate_html(html: str) -> list[str]:
"""Return a list of structural problems with ``html`` (empty == valid).
@@ -299,6 +336,7 @@ _GENERATION_TEMPLATE = """你是一个内网页面生成助手。请根据输入
6. 不要包含 <script>、外链脚本、iframe、表单提交、on* 事件属性。
7. 参考页面只用于风格和布局,页面内容必须来自“本次数据”,不要复用参考页旧数据。
8. 页面使用中文,排版适合在浏览器中直接预览。
9. HTML 内的目录/锚点跳转必须只跳转到当前这个 HTML 文件内部:只使用 href="#section-id",不要使用 href="index.html#section-id"、".md#..."、其它文件路径或任何跨文件锚点;目标区块必须有匹配的 id。
"""
_REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问题,请修复后重新输出完整页面。
@@ -317,6 +355,7 @@ _REPAIR_TEMPLATE = """你是一个 HTML 修复助手。下面的 HTML 存在问
2. fixed_html 必须是完整 HTML 文档,包含 <!doctype html>、html、head、body。
3. 移除所有 <script>、iframe、外链脚本、on* 事件属性、表单提交。
4. 保留原有内容与数据,只修复结构与安全问题。
5. HTML 内的目录/锚点跳转必须只跳转到当前这个 HTML 文件内部:只使用 href="#section-id",不要链接到 .html/.md 文件作为锚点。
"""
_STYLE_EXTRACT_TEMPLATE = """你是一个页面风格分析助手。请分析下面的 HTML 页面,提取可复用的风格与布局特征。
@@ -53,8 +53,18 @@ _MARKDOWN_MIN_CHARS = 100
_MARKDOWN_INSTRUCTION = (
"\n\n【输出要求】请将本次任务的最终结果生成为一个 Markdown(.md) 格式的文件,"
"保存到 /mnt/user-data/outputs 目录,并使用 present_files 展示该文件。"
"Markdown 内的目录/锚点跳转必须只跳转到当前这个 Markdown 文件内部:"
"只使用 [标题](#section-id),不要使用 [标题](xxx.md#section-id)、"
"[标题](xxx.html#section-id) 或任何跨文件锚点。"
)
_LOCAL_DOC_MARKDOWN_LINK_RE = re.compile(
r"^(?![a-z][a-z0-9+.-]*:)(?!//)(?P<path>[^#?]*\.(?:html?|md|markdown))(?P<query>\?[^#]*)?(?P<fragment>#.+)?$",
re.IGNORECASE,
)
_MARKDOWN_INLINE_LINK_RE = re.compile(r"(?<!!)\[([^\]]+)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
_MARKDOWN_REFERENCE_LINK_RE = re.compile(r"^(\s*\[[^\]]+\]:\s*)(\S+)(.*)$", re.MULTILINE)
class MarkdownRequirementError(Exception):
"""Raised when a ``require_markdown`` task did not produce a valid .md file.
@@ -219,6 +229,23 @@ def _task_requires_markdown(task: dict[str, Any]) -> bool:
return bool(isinstance(ctx, dict) and ctx.get("require_markdown"))
def _task_requires_html(task: dict[str, Any]) -> bool:
"""Whether a task must produce an HTML page (the ``必须生成 HTML`` switch).
The flag lives inside ``execution_context.require_html``. For backward
compatibility, tasks created under the old two-tab UI (which stored
``task_kind == "html_page"``) are still treated as HTML tasks, so no data
migration is needed. ``require_html`` and ``require_markdown`` are
independent — both may be on at once.
"""
ctx = task.get("execution_context")
if not isinstance(ctx, dict):
return False
if ctx.get("require_html"):
return True
return ctx.get("task_kind") == "html_page"
def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
"""True when the run produced a ``.md`` file whose text exceeds the minimum.
@@ -245,6 +272,43 @@ def _has_valid_markdown_file(result_files: list[dict[str, Any]] | None) -> bool:
return False
def _normalize_same_page_markdown_target(target: str) -> str | None:
value = (target or "").strip()
if not value or value.startswith("#"):
return value
match = _LOCAL_DOC_MARKDOWN_LINK_RE.match(value)
if not match:
return value
fragment = match.group("fragment")
return fragment or None
def _constrain_markdown_same_page_anchor_links(content: str) -> str:
"""Keep generated Markdown document jumps inside the current Markdown file."""
if not content or not isinstance(content, str):
return content or ""
def replace_inline(match: re.Match[str]) -> str:
label = match.group(1)
target = match.group(2)
normalized = _normalize_same_page_markdown_target(target)
if normalized is None:
return label
return f"[{label}]({normalized})"
def replace_reference(match: re.Match[str]) -> str:
prefix = match.group(1)
target = match.group(2)
suffix = match.group(3)
normalized = _normalize_same_page_markdown_target(target)
if normalized is None:
return f"{prefix}#"
return f"{prefix}{normalized}{suffix}"
content = _MARKDOWN_INLINE_LINK_RE.sub(replace_inline, content)
return _MARKDOWN_REFERENCE_LINK_RE.sub(replace_reference, content)
def _build_result(
task: dict[str, Any],
*,
@@ -905,15 +969,18 @@ class ScheduledTaskService:
scheduled_for: datetime,
run_id: str,
) -> dict[str, Any]:
"""Dispatch a single execution attempt by task kind.
"""Dispatch a single execution attempt by the task's output requirements.
``html_page`` tasks render an HTML document via GLM5; everything else
runs the regular lead-agent attempt. The kind lives in
``execution_context.task_kind`` so no schema migration is needed.
A task carries two independent switches in ``execution_context``:
``require_html`` (必须生成 HTML) and ``require_markdown`` (必须生成
Markdown). When ``require_html`` is set the run goes through the HTML
render pipeline (which also enforces the Markdown requirement when both
switches are on); otherwise it runs the regular lead-agent attempt.
Both flags live in ``execution_context`` so no schema migration is
needed, and old ``task_kind == "html_page"`` tasks still resolve to the
HTML pipeline via :func:`_task_requires_html`.
"""
ctx = task.get("execution_context")
kind = ctx.get("task_kind") if isinstance(ctx, dict) else None
if kind == "html_page":
if _task_requires_html(task):
return await self._run_html_page_attempt(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
@@ -956,17 +1023,21 @@ class ScheduledTaskService:
runtime_thread_id: str,
scheduled_for: datetime,
run_id: str,
) -> str:
) -> tuple[str, list[dict[str, Any]]]:
"""Run the lead agent to collect source data for the HTML page.
Reuses the regular agent-run primitives (with the task's search query
folded into the prompt) and returns the agent's final text, which feeds
the GLM5 render step as ``本次数据``. Returns an empty string if the
agent produced nothing.
folded into the prompt) and returns ``(text, files)`` — the agent's
final text (feeds the render step as ``本次数据``) plus any files it
produced. The files are only captured/needed when the task also has the
Markdown switch on (both 必须生成 HTML and 必须生成 Markdown), so the run
can validate and surface the ``.md`` file alongside the HTML page;
otherwise the list is empty.
"""
cfg = self._html_config(task)
page_prompt = str(cfg.get("page_prompt") or "").strip()
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
also_markdown = _task_requires_markdown(task)
gather_instruction = (
"请运用你自身的检索/搜索等技能,围绕下述页面需求获取最新、真实的资料,"
"并整理成结构化的中文要点(可用列表/小标题),供后续生成 HTML 页面使用,本步骤不要生成 HTML:\n"
@@ -982,12 +1053,22 @@ class ScheduledTaskService:
# which sources/data the agent collects, otherwise it silently leaks in.
if negative_prompt:
parts.append(f"限制条件(最高优先级,必须严格遵守):{negative_prompt}")
gather_task = {**task, "prompt": "\n".join(parts)}
# Never apply the markdown gate to the gather pass.
gather_prompt = "\n".join(parts)
# When the Markdown switch is also on, this same gather pass must save a
# .md file. We do NOT append _MARKDOWN_INSTRUCTION here — `_start_agent_run`
# already appends it exactly once when `require_markdown` is present in the
# execution context (kept below). Appending it here too made the prompt
# carry the 【输出要求】 block twice when both HTML and Markdown were on.
gather_task = {**task, "prompt": gather_prompt}
gather_ctx = dict(task.get("execution_context") or {})
gather_ctx.pop("require_markdown", None)
# Strip the HTML-specific flags so the gather pass is a plain agent run.
gather_ctx.pop("require_html", None)
gather_ctx.pop("task_kind", None)
gather_ctx.pop("html_config", None)
# Keep ``require_markdown`` only when both switches are on, so the agent
# is told (via _start_agent_run) to produce the .md file.
if not also_markdown:
gather_ctx.pop("require_markdown", None)
# Use the task's selected model for the data-gathering agent too, so the
# whole HTML pipeline honors ``html_config.model_name`` instead of
# silently falling back to the default model for this step.
@@ -997,13 +1078,22 @@ class ScheduledTaskService:
gather_task["execution_context"] = gather_ctx
await self._ensure_thread_meta(task["user_id"], runtime_thread_id)
files: list[dict[str, Any]] = []
async with self._agent_run_lock:
output_snapshot = self._snapshot_result_files(runtime_thread_id, task["user_id"]) if also_markdown else None
record = await self._start_agent_run(gather_task, scheduled_for=scheduled_for, runtime_thread_id=runtime_thread_id)
await self.store.update_run(run_id, {"agent_run_id": record.run_id})
if record.task is not None:
await record.task
text = await self._extract_latest_ai_text(runtime_thread_id)
return text or ""
if also_markdown:
files = await self._extract_result_files(
runtime_thread_id,
task["user_id"],
run_id=run_id,
output_snapshot=output_snapshot,
)
return text or "", files
def _collect_html_skill_instructions(self) -> str:
"""Concatenate the bodies of enabled skills marked for HTML pages.
@@ -1072,10 +1162,17 @@ class ScheduledTaskService:
negative_prompt = str(cfg.get("negative_prompt") or "").strip()
reference_favorite_id = cfg.get("reference_favorite_id") or None
# 1) Gather source data (real, via the agent's search tools).
data_text = await self._gather_html_data(
# 1) Gather source data (real, via the agent's search tools). When the
# Markdown switch is also on, the gather pass additionally saves the .md
# file, returned here so we can validate + surface it next to the page.
also_markdown = _task_requires_markdown(task)
data_text, gathered_files = await self._gather_html_data(
task, runtime_thread_id=runtime_thread_id, scheduled_for=scheduled_for, run_id=run_id
)
if also_markdown and not _has_valid_markdown_file(gathered_files):
raise MarkdownRequirementError(
f"未满足输出要求:需生成 Markdown(.md) 文件,且文件正文不少于 {_MARKDOWN_MIN_CHARS} 字"
)
# 2) Load reference style/layout from a favorite if one was selected.
reference_style = None
@@ -1119,6 +1216,7 @@ class ScheduledTaskService:
# 4) Sanitize + validate, with one repair pass on failure.
html, _removed = html_page_utils.sanitize_html(html)
html, _link_changes = html_page_utils.constrain_same_page_anchor_links(html)
errors = html_page_utils.validate_html(html)
if errors:
logger.info("Generated HTML failed validation (%s); attempting repair", errors)
@@ -1131,6 +1229,7 @@ class ScheduledTaskService:
fixed = repaired.get("fixed_html")
if isinstance(fixed, str) and "<" in fixed:
fixed_clean, _ = html_page_utils.sanitize_html(fixed)
fixed_clean, _ = html_page_utils.constrain_same_page_anchor_links(fixed_clean)
if not html_page_utils.validate_html(fixed_clean):
html = fixed_clean
except Exception:
@@ -1158,6 +1257,13 @@ class ScheduledTaskService:
archive_path = self._archive_html_index(run_id, html)
files: list[dict[str, Any]] = []
# When both switches are on, surface the gathered Markdown file(s)
# alongside the generated page so the run delivers both artifacts.
if also_markdown:
for item in gathered_files:
name = str(item.get("name") or "") if isinstance(item, dict) else ""
if name.lower().endswith((".md", ".markdown")):
files.append(item)
if archive_path is not None:
files.append(
{
@@ -1483,6 +1589,14 @@ class ScheduledTaskService:
index += 1
shutil.copy2(path, candidate)
if candidate.suffix.lower() in {".md", ".markdown"}:
try:
original = candidate.read_text(encoding="utf-8", errors="ignore")
constrained = _constrain_markdown_same_page_anchor_links(original)
if constrained != original:
candidate.write_text(constrained, encoding="utf-8", newline="")
except Exception:
logger.debug("Failed to constrain scheduled markdown links in %s", candidate, exc_info=True)
return candidate
def _build_file_result(self, *, virtual_path: str, source_path: Path, archive_path: Path) -> dict[str, Any]:
@@ -36,6 +36,7 @@ _DEFAULT_ENTRY: dict = {
"created_at": None,
"state": STATE_ACTIVE,
"pinned": False,
"featured": False,
"archived_at": None,
"source": SOURCE_UPLOAD,
}
@@ -141,6 +142,11 @@ def set_pinned(name: str, pinned: bool) -> None:
_update_entry(name, {"pinned": pinned})
def set_featured(name: str, featured: bool) -> None:
"""置顶/取消置顶。与 pinned(防归档)正交,仅影响列表排序展示。"""
_update_entry(name, {"featured": featured})
def forget(name: str) -> None:
try:
with _usage_lock: