sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -10,7 +10,7 @@ import httpx
import pytest
import pytest_asyncio
from fastapi import FastAPI
from sqlalchemy import update
from sqlalchemy import event, select, update
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway import deps, langgraph_auth
@@ -25,7 +25,7 @@ from app.gateway.auth.repositories.base import CredentialConflictError
from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
from app.gateway.auth_middleware import AuthMiddleware
from app.gateway.csrf_middleware import CSRFMiddleware
from app.gateway.routers import auth, parallel_agents, system_settings
from app.gateway.routers import auth, parallel_agents, service_tokens, system_settings
from deerflow.config import system_settings as settings
from deerflow.config.login_config import AuthLoginConfig
from deerflow.persistence.user.model import AuthInitializationRow, AuthUsernameRow, UserRow
@@ -44,6 +44,7 @@ async def ctx(monkeypatch, tmp_path):
provider = LocalAuthProvider(repository)
monkeypatch.setattr(settings, "runtime_home", lambda: tmp_path)
monkeypatch.setattr(auth, "get_local_provider", lambda: provider)
monkeypatch.setattr(service_tokens, "get_local_provider", lambda: provider)
monkeypatch.setattr(deps, "get_local_provider", lambda: provider)
monkeypatch.setattr(langgraph_auth, "get_local_provider", lambda: provider)
monkeypatch.setattr(auth_config, "_auth_config", auth_config.AuthConfig(jwt_secret="test-auth-key-with-sufficient-entropy-0123456789"))
@@ -57,19 +58,159 @@ async def ctx(monkeypatch, tmp_path):
monkeypatch.setattr(auth, "_SETUP_STATUS_COOLDOWN", {})
app = FastAPI()
app.include_router(auth.router)
app.include_router(service_tokens.router)
app.include_router(system_settings.router)
app.include_router(parallel_agents.router)
app.add_middleware(CSRFMiddleware)
app.add_middleware(AuthMiddleware)
counter = SimpleNamespace(count_owned=AsyncMock(return_value=0))
app.state.skill_store = counter
app.state.agent_store = counter
app.dependency_overrides[deps.get_skill_store] = lambda: counter
app.dependency_overrides[deps.get_agent_store] = lambda: counter
monkeypatch.setattr(auth, "get_run_store", lambda _: SimpleNamespace(count_by_user=AsyncMock(return_value={})))
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
yield SimpleNamespace(client=client, provider=provider, repository=repository, sf=sf, login_config=login_config)
yield SimpleNamespace(client=client, app=app, provider=provider, repository=repository, sf=sf, login_config=login_config)
await engine.dispose()
@pytest.mark.asyncio
async def test_service_exchange_requires_shared_token(ctx):
user = await ctx.provider.create_user("alice@cm.com", password=PASSWORD)
denied = await ctx.client.post(
"/api/v1/auth/service/exchange",
json={"username": "alice", "scope": ["llmwiki:sync"]},
)
assert denied.status_code == 401
assert denied.json()["detail"] == "Invalid service token"
wrong = await ctx.client.post(
"/api/v1/auth/service/exchange",
json={"username": "alice", "scope": ["llmwiki:sync"]},
headers={"X-Service-Token": "wrong-token"},
)
assert wrong.status_code == 401
response = await ctx.client.post(
"/api/v1/auth/service/exchange",
json={"username": "alice", "scope": ["llmwiki:sync"]},
headers={"X-Service-Token": service_tokens._SERVICE_TOKEN_SECRET},
)
assert response.status_code == 200, response.text
token = decode_token(response.json()["access_token"])
assert token.sub == str(user.id)
assert token.auth_method == "service"
assert token.client_id is None
assert token.scope == ["llmwiki:sync"]
assert response.json()["expires_in"] == 300
repeated = await ctx.client.post(
"/api/v1/auth/service/exchange",
json={"username": "alice", "scope": ["llmwiki:sync"]},
headers={"X-Service-Token": service_tokens._SERVICE_TOKEN_SECRET},
)
assert repeated.status_code == 200
@pytest.mark.asyncio
async def test_service_exchange_rejects_unsupported_scope(ctx):
response = await ctx.client.post(
"/api/v1/auth/service/exchange",
json={"username": "alice", "scope": ["admin"]},
headers={"X-Service-Token": service_tokens._SERVICE_TOKEN_SECRET},
)
assert response.status_code == 403
@pytest.mark.asyncio
async def test_user_owner_lookup_batches_ids(ctx):
first = await ctx.provider.create_user("first-owner@cm.com", password=PASSWORD)
second = await ctx.provider.create_user("second-owner@cm.com", password=PASSWORD)
users = await ctx.provider.get_users_by_ids([
str(first.id), str(second.id), str(first.id), "missing-user",
])
assert {str(user.id) for user in users} == {str(first.id), str(second.id)}
@pytest.mark.asyncio
async def test_username_login_uses_exact_indexed_key_and_backfills_legacy_rows(ctx):
user = await ctx.provider.create_user("quick-login@legacy-domain.com", password=PASSWORD)
statements: list[str] = []
engine = ctx.sf.kw["bind"]
def capture_sql(_conn, _cursor, statement, _parameters, _context, _executemany):
if statement.lstrip().upper().startswith("SELECT") and "users" in statement.lower():
statements.append(statement.lower())
event.listen(engine.sync_engine, "before_cursor_execute", capture_sql)
try:
found = await ctx.provider.find_user_by_email_local_part("quick-login")
finally:
event.remove(engine.sync_engine, "before_cursor_execute", capture_sql)
assert found is not None and found.id == user.id
assert statements and "username_key =" in statements[0]
assert "lower(users.email)" not in statements[0]
async with ctx.sf() as session:
await session.execute(update(UserRow).where(UserRow.id == str(user.id)).values(username_key=None))
await session.commit()
found_after_legacy_fallback = await ctx.provider.find_user_by_email_local_part("QUICK-LOGIN")
assert found_after_legacy_fallback is not None and found_after_legacy_fallback.id == user.id
async with ctx.sf() as session:
saved_key = await session.scalar(select(UserRow.username_key).where(UserRow.id == str(user.id)))
assert saved_key == "quick-login"
@pytest.mark.parametrize(
("method", "path", "allowed"),
[
("GET", "/api/llmwiki/knowledge-bases", True),
("POST", "/api/llmwiki/knowledge-bases/kb/documents", True),
("POST", "/api/llmwiki/knowledge-bases/kb/documents/manual", True),
("POST", "/api/llmwiki/search", False),
("POST", "/api/threads/thread/runs", False),
("GET", "/api/v1/auth/me", False),
],
)
def test_service_scope_path_allowlist(method, path, allowed):
from app.gateway.auth_middleware import _is_llmwiki_sync_request
assert _is_llmwiki_sync_request(method, path) is allowed
@pytest.mark.parametrize(
("scopes", "method", "path", "all_requested", "allowed"),
[
(["recommended-questions:read"], "GET", "/api/recommended-questions", False, True),
(["recommended-questions:read"], "GET", "/api/recommended-questions", True, False),
(["recommended-questions:read"], "POST", "/api/recommended-questions", False, False),
(["recommended-questions:read"], "GET", "/api/recommended-questions-reorder", False, False),
(["recommended-questions:read"], "GET", "/api/llmwiki/knowledge-bases", False, False),
(["llmwiki:sync"], "GET", "/api/recommended-questions", False, False),
],
)
def test_recommended_question_service_scope_is_read_only(scopes, method, path, all_requested, allowed):
from app.gateway.auth_middleware import _is_service_scope_request
assert _is_service_scope_request(scopes, method, path, all_requested=all_requested) is allowed
@pytest.mark.asyncio
async def test_service_token_is_rejected_outside_llmwiki_sync(ctx):
user = await ctx.provider.create_user("alice@cm.com", password=PASSWORD)
token = create_access_token(
str(user.id),
expires_delta=service_tokens.timedelta(seconds=300),
token_version=user.token_version,
auth_method="service",
client_id="surfsense",
scope=["llmwiki:sync"],
)
response = await ctx.client.get("/api/v1/auth/me", headers={"Authorization": f"Bearer {token}"})
assert response.status_code == 403
def enable_registration():
current = settings.load_system_settings()
settings.save_system_settings(current.model_copy(update={"authentication": settings.AuthenticationSettings(password_login_enabled=True, registration_enabled=True)}))
@@ -83,6 +224,65 @@ async def make_admin(ctx, *, password=PASSWORD):
return await ctx.provider.create_user("operator@cm.com", password=password, system_role="admin")
@pytest.mark.asyncio
async def test_login_identity_and_user_management_do_not_use_checkpointer(ctx, monkeypatch):
def forbidden(*args, **kwargs):
raise AssertionError("Authentication accessed the chat checkpoint backend")
monkeypatch.setattr(deps, "get_checkpointer", forbidden)
ctx.app.state.checkpointer = SimpleNamespace(aget_tuple=forbidden, alist=forbidden, aput=forbidden)
user = await make_admin(ctx)
ctx.login_config.token_login.enabled = True
ctx.login_config.token_login.mock_tokens = {"audit-token": "operator"}
assert (await ctx.client.get("/api/v1/auth/options")).status_code == 200
for route, kwargs in [
("local", {"data": {"username": "operator", "password": PASSWORD}}),
("username", {"json": {"username": "operator"}, "params": {"password": PASSWORD}}),
("token", {"params": {"token": "audit-token"}}),
]:
response = await ctx.client.post(f"/api/v1/auth/login/{route}", **kwargs)
assert response.status_code == 200, response.text
headers = {"Authorization": f"Bearer {response.json()['access_token']}"}
assert (await ctx.client.get("/api/v1/auth/me", headers=headers)).json()["id"] == str(user.id)
assert (await ctx.client.get("/api/v1/auth/users", headers=headers)).status_code == 200
@pytest.mark.asyncio
async def test_list_users_paged_shape_and_search(ctx):
"""传 limit → UserListPage(items/total/approved_count);不传 → 旧全量列表。"""
admin = await make_admin(ctx)
for name in ("bob@cm.com", "carol@cm.com"):
await ctx.provider.create_user(name, password=PASSWORD)
await ctx.provider.create_user("dave@cm.com", password=PASSWORD) # 将被取消放行
dave = await ctx.provider.get_user_by_email("dave@cm.com")
dave.approved = False
await ctx.provider.update_user(dave)
headers = bearer(admin)
legacy = await ctx.client.get("/api/v1/auth/users", headers=headers)
assert legacy.status_code == 200
assert isinstance(legacy.json(), list) and len(legacy.json()) == 4
paged = await ctx.client.get("/api/v1/auth/users", headers=headers, params={"limit": 2, "offset": 0, "include_counts": "false"})
assert paged.status_code == 200, paged.text
body = paged.json()
assert body["total"] == 4 and body["approved_count"] == 3 # admin+bob+carol 可登录,dave 被拦
assert body["limit"] == 2 and body["offset"] == 0
assert len(body["items"]) == 2
assert all(item["qa_count"] == 0 and item["skill_count"] == 0 for item in body["items"])
page2 = await ctx.client.get("/api/v1/auth/users", headers=headers, params={"limit": 2, "offset": 2})
assert page2.status_code == 200 and len(page2.json()["items"]) == 2
searched = await ctx.client.get("/api/v1/auth/users", headers=headers, params={"limit": 10, "q": "dave"})
assert searched.status_code == 200
assert searched.json()["total"] == 1 and searched.json()["items"][0]["email"] == "dave@cm.com"
assert searched.json()["approved_count"] == 0 # 搜索范围内唯一用户已被取消放行
miss = await ctx.client.get("/api/v1/auth/users", headers=headers, params={"limit": 10, "offset": 999})
assert miss.status_code == 200 and miss.json()["items"] == [] and miss.json()["total"] == 4
@pytest.mark.asyncio
async def test_anonymous_options_defaults_and_closed_registration(ctx):
settings.save_system_settings(settings.SystemSettings())
@@ -172,9 +372,13 @@ async def test_policy_admin_only_and_lockout_protection(ctx):
for path, kwargs in [
("/api/v1/auth/login/username", {"json": {"username": "newbie"}}),
("/api/v1/auth/login/token", {"params": {"token": "upstream"}}),
("/api/parallel-agents/auth/token", {"json": {"username": "newbie"}}),
]:
assert (await ctx.client.post(path, **kwargs)).status_code == 403
# Detail integrations use the legacy token endpoint as an unconditional
# preflight. Policy rejection returns a non-JWT compatibility value.
compat = await ctx.client.post("/api/parallel-agents/auth/token", json={"username": "newbie"})
assert compat.status_code == 200
assert compat.json()["access_token"] == "force-situation-detail-compat-newbie"
assert await ctx.provider.count_users() == 2
assert (await ctx.client.get("/api/v1/auth/oauth/github", headers=bearer(admin))).status_code == 403
@@ -247,13 +451,31 @@ async def test_historical_plaintext_passphrase_wins_and_changes_without_old_hash
@pytest.mark.asyncio
async def test_parallel_token_endpoint_cannot_bypass_personal_password(ctx):
async def test_parallel_token_endpoint_returns_non_jwt_compat_value_when_login_is_rejected(ctx):
await ctx.provider.create_user("alice@cm.com", password=PASSWORD)
endpoint = "/api/parallel-agents/auth/token"
assert (await ctx.client.post(endpoint, json={"username": "alice"})).status_code == 401
fallback = await ctx.client.post(endpoint, json={"username": "alice"})
assert fallback.status_code == 200
assert fallback.json()["access_token"] == "force-situation-detail-compat-alice"
assert (await ctx.client.post(endpoint, params={"password": PASSWORD}, json={"username": "alice"})).status_code == 200
@pytest.mark.asyncio
async def test_parallel_token_endpoint_returns_compat_value_for_any_rejected_detail_preflight(ctx):
"""Every old detail client passes its preflight without a real login session."""
from app.gateway.auth import TokenError, decode_token
endpoint = "/api/parallel-agents/auth/token"
for username in ("fs-collect", "fs-detail-read", "fs-detail-submit", "fs-detail-update", "task-detail-agent"):
response = await ctx.client.post(endpoint, json={"username": username})
assert response.status_code == 200
payload = response.json()
assert payload["access_token"] == f"force-situation-detail-compat-{username}"
assert payload["user_id"] == f"external:{username}"
# The compatibility value cannot authenticate unrelated APIs.
assert isinstance(decode_token(payload["access_token"]), TokenError)
@pytest.mark.asyncio
async def test_approval_gate_applies_to_password_login(ctx):
await ctx.provider.create_user("blocked@cm.com", password=PASSWORD, approved=False)
@@ -313,6 +535,19 @@ async def test_concurrent_registration_email_domains_cannot_claim_same_username(
assert await ctx.provider.count_users() == 1
@pytest.mark.asyncio
async def test_username_login_accepts_form_body(ctx):
response = await ctx.client.post(
"/api/v1/auth/login/username",
data={"username": "firefox-legacy"},
params={"password": "123ewq"},
)
assert response.status_code == 200, response.text
assert response.json()["username"] == "firefox-legacy"
blank = await ctx.client.post("/api/v1/auth/login/username", data={"username": ""})
assert blank.status_code == 422
@pytest.mark.asyncio
async def test_username_sso_rechecks_password_if_registration_won_a_race(ctx, monkeypatch):
await ctx.provider.create_user("alice@cm.com", password=PASSWORD)
@@ -371,8 +606,10 @@ async def test_first_personal_password_requires_current_shared_passphrase(ctx):
async def test_enabled_password_login_blocks_bare_username_and_parallel_logins(ctx):
await ctx.provider.create_user("legacy@cm.com")
for username in ("legacy", "not-created"):
for endpoint in ("/api/v1/auth/login/username", "/api/parallel-agents/auth/token"):
assert (await ctx.client.post(endpoint, json={"username": username})).status_code == 401
assert (await ctx.client.post("/api/v1/auth/login/username", json={"username": username})).status_code == 401
compat = await ctx.client.post("/api/parallel-agents/auth/token", json={"username": username})
assert compat.status_code == 200
assert compat.json()["access_token"] == f"force-situation-detail-compat-{username}"
assert await ctx.provider.count_users() == 1
@@ -536,3 +773,54 @@ async def test_first_account_login_obeys_single_session_setting(ctx, method):
assert second.status_code == 200
assert decode_token(second.json()["access_token"]).ver == 2
assert (await ctx.client.get("/api/v1/auth/me", headers={"Authorization": f"Bearer {old_token}"})).status_code == 401
@pytest.mark.asyncio
async def test_whitelist_is_independent_of_statistics_stores(ctx, monkeypatch):
admin = await make_admin(ctx)
def forbidden(*args, **kwargs):
raise AssertionError("Whitelist must not resolve statistics stores")
for name in ("get_skill_store", "get_agent_store", "get_run_store"):
monkeypatch.setattr(auth, name, forbidden)
response = await ctx.client.get("/api/v1/auth/users", headers=bearer(admin), params={"limit": 20, "include_counts": "false"})
assert response.status_code == 200, response.text
assert response.json()["total"] == 1
assert response.json()["items"][0]["login_password"] is None
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["list_users_page", "count_users_filtered"])
async def test_whitelist_timeout_cancels_account_queries(ctx, monkeypatch, method):
admin = await make_admin(ctx)
cancelled = asyncio.Event()
async def stalled(*args, **kwargs):
try:
await asyncio.Event().wait()
finally:
cancelled.set()
monkeypatch.setattr(ctx.provider, method, stalled)
monkeypatch.setattr(auth, "_USER_LIST_TIMEOUT_SECONDS", 0.05)
response = await asyncio.wait_for(ctx.client.get("/api/v1/auth/users", headers=bearer(admin), params={"limit": 20, "include_counts": "false"}), 5)
assert response.status_code == 503
assert "超时" in response.json()["detail"]
assert cancelled.is_set()
@pytest.mark.asyncio
async def test_whitelist_paged_is_admin_only(ctx):
user = await ctx.provider.create_user("normal-whitelist@cm.com", password=PASSWORD)
params = {"limit": 20, "include_counts": "false"}
assert (await ctx.client.get("/api/v1/auth/users", headers=bearer(user), params=params)).status_code == 403
assert (await ctx.client.get("/api/v1/auth/users", params=params)).status_code == 401
@pytest.mark.asyncio
async def test_whitelist_loads_when_resource_stores_unavailable(ctx):
admin = await make_admin(ctx)
del ctx.app.state.skill_store
del ctx.app.state.agent_store
response = await ctx.client.get("/api/v1/auth/users", headers=bearer(admin), params={"limit": 20, "include_counts": "false"})
assert response.status_code == 200
assert response.json()["total"] == 1
# The full statistics endpoint still requires the resource stores.
assert (await ctx.client.get("/api/v1/auth/users", headers=bearer(admin))).status_code == 503
@@ -0,0 +1,69 @@
"""Leaderboard reads must not expose the removed per-user analytics queries."""
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from httpx import ASGITransport, AsyncClient
from app.gateway.routers import admin_users
def _app() -> FastAPI:
app = FastAPI()
app.include_router(admin_users.router)
return app
@pytest.mark.asyncio
async def test_removed_analytics_url_never_opens_a_database_session(monkeypatch):
def unexpected_session():
pytest.fail("The retired analytics endpoint must not touch the database")
monkeypatch.setattr(admin_users, "get_session_factory", unexpected_session)
app = _app()
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
response = await client.get("/api/admin/users/sample-user/analytics?days=7")
assert response.status_code == 404
assert not any(path.endswith("/analytics") for path in app.openapi()["paths"])
@pytest.mark.asyncio
async def test_leaderboard_still_returns_aggregate_snapshot(monkeypatch):
async def current_admin(_request):
return SimpleNamespace(system_role="admin")
async def snapshot(_factory, _settings, since, until, **_kwargs):
return admin_users.AdminLeaderboardResponse(
range_days=7,
since=since.isoformat(),
until=until.isoformat(),
overview=admin_users.AdminLeaderboardOverview(total_users=2, total_runs=12),
)
monkeypatch.setattr(admin_users, "get_current_user_from_request", current_admin)
monkeypatch.setattr(admin_users, "get_session_factory", lambda: object())
monkeypatch.setattr(admin_users, "load_system_settings", lambda: SimpleNamespace(leaderboard=object()))
monkeypatch.setattr(admin_users, "_load_or_schedule_daily_leaderboard", snapshot)
async with AsyncClient(transport=ASGITransport(app=_app()), base_url="http://test") as client:
response = await client.get("/api/admin/users/leaderboard?days=7")
assert response.status_code == 200
assert response.json()["overview"]["total_runs"] == 12
@pytest.mark.asyncio
async def test_leaderboard_keeps_admin_authorization(monkeypatch):
async def ordinary_user(_request):
return SimpleNamespace(system_role="user")
def unexpected_session():
pytest.fail("Authorization must precede database access")
monkeypatch.setattr(admin_users, "get_current_user_from_request", ordinary_user)
monkeypatch.setattr(admin_users, "get_session_factory", unexpected_session)
async with AsyncClient(transport=ASGITransport(app=_app()), base_url="http://test") as client:
response = await client.get("/api/admin/users/leaderboard?days=7")
assert response.status_code == 403
@@ -0,0 +1,67 @@
from __future__ import annotations
import pytest
from app.gateway.admin_audit_retention import sweep_admin_audit_retention
from app.gateway.admin_audit_retention_control import (
is_paused,
read_status,
set_paused,
write_status,
)
class _MetricsStore:
def __init__(self) -> None:
self.calls = 0
async def purge_before(self, _cutoff, *, batch_size: int) -> int:
self.calls += 1
return batch_size
@pytest.mark.asyncio
async def test_sweep_stops_after_committed_batch_and_reports_progress() -> None:
store = _MetricsStore()
stop_requested = False
progress: list[dict] = []
async def report(value: dict) -> None:
nonlocal stop_requested
progress.append(value)
if value["deleted"]["llm_metrics"] > 0:
stop_requested = True
deleted = await sweep_admin_audit_retention(
session_factory=None,
llm_metrics_store=store,
retention_days=90,
batch_size=3,
max_batches_per_store=4,
inter_batch_pause_seconds=0,
max_sweep_seconds=5,
should_stop=lambda: stop_requested,
on_progress=report,
)
assert store.calls == 1
assert deleted == {
"llm_metrics": 3,
"tool_metrics": 0,
"run_summaries": 0,
"leaderboard_snapshots": 0,
}
assert progress[-1]["status"] == "paused"
assert progress[-1]["deleted"]["llm_metrics"] == 3
def test_pause_marker_and_progress_survive_process_restart(tmp_path) -> None:
assert not is_paused(tmp_path)
set_paused(tmp_path, True)
write_status(tmp_path, {"status": "running", "deleted": {"llm_metrics": 12}})
assert is_paused(tmp_path)
assert read_status(tmp_path) == {"status": "running", "deleted": {"llm_metrics": 12}}
set_paused(tmp_path, False)
assert not is_paused(tmp_path)
@@ -1,6 +1,8 @@
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
from uuid import uuid4
import pytest
@@ -10,13 +12,19 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers.agents import (
AgentsListResponse,
_agent_list_cache,
_agent_list_inflight,
_agent_owner_name_cache,
_backfill_listing_cache,
_cached_paginated_agent_list,
_load_config_extras,
_resolve_usernames,
_write_agent_config,
)
from app.gateway.services import build_run_config
from deerflow.config import paths as paths_module
from deerflow.config.agents_config import AgentConfig
from deerflow.config.agents_config import AgentConfig, normalize_agent_chat_link
from deerflow.persistence.agents import AgentRepository, MemoryAgentStore
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.base import Base
@@ -65,6 +73,30 @@ async def _add_sql_agent(sf, *, agent_id: str, user_id: str | None) -> None:
await session.commit()
@pytest.mark.asyncio
async def test_sidebar_agent_query_returns_only_lightweight_visible_fields(sql_session_factory):
await _add_sql_agent(sql_session_factory, agent_id="owned-agent", user_id="user-1")
await _add_sql_agent(sql_session_factory, agent_id="public-agent", user_id=None)
agents = await AgentRepository(sql_session_factory).list_sidebar("user-1", limit=3)
assert agents == [
{"id": "owned-agent", "name": "owned-agent"},
{"id": "public-agent", "name": "public-agent"},
]
assert all(set(item) == {"id", "name"} for item in agents)
@pytest.mark.asyncio
async def test_memory_sidebar_query_returns_only_lightweight_fields():
store = MemoryAgentStore()
await store.create({"id": "owned-agent", "name": "我的智能体", "user_id": "user-1"})
agents = await store.list_sidebar("user-1", limit=3)
assert agents == [{"id": "owned-agent", "name": "我的智能体"}]
async def _add_sql_favorite(sf, *, user_id: str, agent_id: str, origin: str) -> None:
async with sf() as session:
session.add(
@@ -267,6 +299,274 @@ async def test_manual_favorite_upgrades_existing_auto_builtin_favorite(sql_sessi
assert [row["id"] for row in rows] == ["builtin-agent"]
async def _add_sql_hidden_agent(sf, *, agent_id: str, user_id: str) -> None:
"""他人创建、普通列表不可见的条目(未发布 / 私密广场)。"""
async with sf() as session:
session.add(
AgentRow(
id=agent_id,
name=agent_id,
description="",
user_id=user_id,
published=not agent_id.startswith("hidden-unpublished"),
is_private=agent_id.startswith("hidden-private"),
square_id="",
)
)
await session.commit()
@pytest.mark.asyncio
async def test_sql_list_paginated_admin_pages_every_agent(sql_session_factory) -> None:
"""管理端分页(GET /api/agents?admin=true&page=N 的 store 支撑):
admin=True 跳过可见性过滤 — 内置 / 他人未发布 / 私密广场条目全部参与分页,
搜索在服务端过滤,越界页返回空页而非报错。"""
await _add_sql_agent(sql_session_factory, agent_id="builtin-1", user_id=None)
await _add_sql_agent(sql_session_factory, agent_id="owned-a", user_id="user-a")
await _add_sql_hidden_agent(sql_session_factory, agent_id="hidden-unpublished", user_id="user-b")
await _add_sql_hidden_agent(sql_session_factory, agent_id="hidden-private", user_id="user-b")
store = AgentRepository(sql_session_factory)
# 非 admin 对照:user-a 只看得到自己 + 内置(user-b 的两条不可见)。
_, visible_total = await store.list_paginated(
user_id="user-a",
scope=None,
search=None,
tag_ids=None,
page=1,
page_size=20,
)
assert visible_total == 2
page1, total = await store.list_paginated(
user_id="user-a",
scope=None,
search=None,
tag_ids=None,
page=1,
page_size=2,
admin=True,
sort="time_asc",
)
page2, total2 = await store.list_paginated(
user_id="user-a",
scope=None,
search=None,
tag_ids=None,
page=2,
page_size=2,
admin=True,
sort="time_asc",
)
assert total == 4 and total2 == 4
assert len(page1) == 2 and len(page2) == 2
assert {r["id"] for r in page1} | {r["id"] for r in page2} == {
"builtin-1",
"owned-a",
"hidden-unpublished",
"hidden-private",
}
# 越界页:空页但 total 不变(前端据此收敛回末页)。
page3, total3 = await store.list_paginated(
user_id="user-a",
scope=None,
search=None,
tag_ids=None,
page=3,
page_size=2,
admin=True,
)
assert total3 == 4 and page3 == []
# 服务端搜索:管理端目录同样按名称过滤。
hits, hits_total = await store.list_paginated(
user_id="user-a",
scope=None,
search="hidden",
tag_ids=None,
page=1,
page_size=20,
admin=True,
)
assert hits_total == 2
assert {r["id"] for r in hits} == {"hidden-unpublished", "hidden-private"}
@pytest.mark.asyncio
async def test_memory_store_admin_pagination() -> None:
"""内存版 store 的 admin 分页行为与 SQL 版一致(单元测试/回退路径)。"""
store = MemoryAgentStore()
await store.ensure_builtin({"id": "builtin-1", "name": "Built In"})
await store.create({"id": "owned-a", "name": "Owned A", "user_id": "user-a"})
await store.create(
{"id": "hidden-b", "name": "Hidden B", "user_id": "user-b", "published": False}
)
await store.create(
{
"id": "private-b",
"name": "Private B",
"user_id": "user-b",
# 内存版 _is_visible 不看 is_private,这里保持未发布,
# 使「非 admin 对照 = 内置 + 自己」在两种 store 语义下都成立。
"published": False,
"is_private": True,
}
)
_, visible_total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None, page=1, page_size=20
)
assert visible_total == 2
page1, total = await store.list_paginated(
user_id="user-a",
scope=None,
search=None,
tag_ids=None,
page=1,
page_size=3,
admin=True,
sort="time_asc",
)
page2, total2 = await store.list_paginated(
user_id="user-a",
scope=None,
search=None,
tag_ids=None,
page=2,
page_size=3,
admin=True,
sort="time_asc",
)
assert total == 4 and total2 == 4
assert len(page1) == 3 and len(page2) == 1
assert {r["id"] for r in page1} | {r["id"] for r in page2} == {
"builtin-1",
"owned-a",
"hidden-b",
"private-b",
}
async def _add_sql_hidden_flagged_agent(sf, *, agent_id: str, user_id: str | None) -> None:
"""创建时即 hidden=True 的条目(绕过 API,直接落库)。"""
async with sf() as session:
session.add(
AgentRow(
id=agent_id,
name=agent_id,
description="",
user_id=user_id,
published=True,
is_private=False,
square_id="",
hidden=True,
)
)
await session.commit()
@pytest.mark.asyncio
async def test_sql_hidden_agent_vanishes_from_non_admin_listings_but_stays_usable(
sql_session_factory,
) -> None:
"""管理端隐藏开关的 store 语义:hidden=True 后从所有非 admin 列表消失
(list_visible / 各分页 scope / selector / 私密广场),但单条访问
(get_visible)不受影响,admin 列表仍可见且可按 hidden 筛选。"""
await _add_sql_agent(sql_session_factory, agent_id="builtin-1", user_id=None)
await _add_sql_agent(sql_session_factory, agent_id="owned-a", user_id="user-a")
await _add_sql_agent(sql_session_factory, agent_id="published-b", user_id="user-b")
await _add_sql_hidden_flagged_agent(sql_session_factory, agent_id="flagged-1", user_id="user-b")
store = AgentRepository(sql_session_factory)
# list_visible(默认列表):flagged-1 消失
visible_ids = {r["id"] for r in await store.list_visible("user-a")}
assert visible_ids == {"builtin-1", "owned-a", "published-b"}
# 非_admin 分页(默认 scope 与 square/mine scope 一致排除)
for scope in (None, "square", "mine"):
rows, total = await store.list_paginated(
user_id="user-a", scope=scope, search=None, tag_ids=None, page=1, page_size=20,
)
assert "flagged-1" not in {r["id"] for r in rows}, scope
# 属主自己也看不到自己被隐藏的智能体(非 admin 一律过滤)
rows, _ = await store.list_paginated(
user_id="user-b", scope="mine", search=None, tag_ids=None, page=1, page_size=20,
)
assert "flagged-1" not in {r["id"] for r in rows}
# selector(featured)与私密广场同样排除
await store.set_featured("flagged-1", 1)
featured_ids = {r["id"] for r in await store.list_featured("user-a")}
assert "flagged-1" not in featured_ids
assert await store.update_any("flagged-1", {"is_private": True}) is not None
private_ids = {r["id"] for r in await store.list_private()}
assert "flagged-1" not in private_ids
# 单条访问不受影响(隐藏不是禁用:既有会话/详情继续可用)
assert await store.get_visible("flagged-1", "user-a") is not None
assert await store.get_visible("flagged-1", "user-b") is not None
# admin:全量可见,且可按 hidden 筛选
rows, total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None,
page=1, page_size=20, admin=True,
)
assert total == 4 and "flagged-1" in {r["id"] for r in rows}
rows, total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None,
page=1, page_size=20, admin=True, hidden=True,
)
assert total == 1 and {r["id"] for r in rows} == {"flagged-1"}
rows, total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None,
page=1, page_size=20, admin=True, hidden=False,
)
assert total == 3 and "flagged-1" not in {r["id"] for r in rows}
# 开关往返:取消隐藏后回到列表。注:flagged-1 中途被挪进了私密广场
#(is_private=True),普通列表本就不含私密条目,先移回公共广场再断言。
assert (await store.update_any("flagged-1", {"hidden": False}))["hidden"] is False
await store.update_any("flagged-1", {"is_private": False})
visible_ids = {r["id"] for r in await store.list_visible("user-a")}
assert "flagged-1" in visible_ids
@pytest.mark.asyncio
async def test_memory_hidden_agent_filtered_from_non_admin_listings() -> None:
"""内存版 store 的 hidden 行为与 SQL 版一致。"""
store = MemoryAgentStore()
await store.ensure_builtin({"id": "builtin-1", "name": "Built In"})
await store.create({"id": "owned-a", "name": "Owned A", "user_id": "user-a"})
await store.create(
{"id": "flagged-1", "name": "Flagged", "user_id": "user-b", "published": True, "hidden": True}
)
assert {r["id"] for r in await store.list_visible("user-a")} == {"builtin-1", "owned-a"}
rows, total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None, page=1, page_size=20,
)
assert total == 2 and "flagged-1" not in {r["id"] for r in rows}
# 单条访问 + admin 可见 + hidden 筛选
assert await store.get_visible("flagged-1", "user-a") is not None
rows, total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None,
page=1, page_size=20, admin=True,
)
assert total == 3 and "flagged-1" in {r["id"] for r in rows}
rows, total = await store.list_paginated(
user_id="user-a", scope=None, search=None, tag_ids=None,
page=1, page_size=20, admin=True, hidden=True,
)
assert total == 1 and {r["id"] for r in rows} == {"flagged-1"}
# 开关往返
assert (await store.update_any("flagged-1", {"hidden": False}))["hidden"] is False
assert "flagged-1" in {r["id"] for r in await store.list_visible("user-a")}
@pytest.mark.asyncio
async def test_agent_store_only_owner_can_update_or_delete() -> None:
store = MemoryAgentStore()
@@ -390,9 +690,21 @@ def test_write_agent_config_persists_skills_into_config_yaml(isolated_paths) ->
assert raw["description"] == "desc"
assert raw["skills"] == ["s1", "s2"]
assert raw["model"] == "gpt-x"
assert raw["model_prompt_suffix_enabled"] is True
assert raw["tool_groups"] == ["tg-a"]
def test_write_agent_config_persists_prompt_suffix_switch(isolated_paths) -> None:
_write_agent_config(
"agent-suffix-off",
name="助手",
description="",
model_prompt_suffix_enabled=False,
)
assert _load_config_extras("agent-suffix-off")["model_prompt_suffix_enabled"] is False
def test_load_config_extras_returns_values_written_to_yaml(isolated_paths) -> None:
agent_id = "agent-002"
_write_agent_config(
@@ -408,11 +720,13 @@ def test_load_config_extras_returns_values_written_to_yaml(isolated_paths) -> No
assert extras == {
"skills": ["s1"],
"model": None,
"model_prompt_suffix_enabled": True,
"tool_groups": None,
"common_questions": None,
"template_name": None,
"seat_skill_directive": None,
"llmwiki_knowledge_base_ids": None,
"chat_link": None,
}
@@ -421,11 +735,13 @@ def test_load_config_extras_missing_directory_returns_none_triple(isolated_paths
assert extras == {
"skills": None,
"model": None,
"model_prompt_suffix_enabled": True,
"tool_groups": None,
"common_questions": None,
"template_name": None,
"seat_skill_directive": None,
"llmwiki_knowledge_base_ids": None,
"chat_link": None,
}
@@ -495,6 +811,56 @@ def test_write_agent_config_omits_skills_when_none(isolated_paths) -> None:
assert "skills" not in raw
assert "model" not in raw
assert "tool_groups" not in raw
assert "chat_link" not in raw
def test_write_agent_config_persists_chat_link_and_drops_blank(isolated_paths) -> None:
agent_id = "agent-chat-link"
link = {
"url": "https://example.test/desk",
"open_mode": "self",
"route_params": [
{"key": "userName", "paramType": "login", "source": "userName"},
{"key": "theme", "paramType": "theme", "themeValue": "red=red;dark-blue=dark"},
],
}
_write_agent_config(agent_id, name="助手", description="", chat_link=link)
extras = _load_config_extras(agent_id)
assert extras["chat_link"]["url"] == "https://example.test/desk"
assert extras["chat_link"]["open_mode"] == "self"
assert extras["chat_link"]["route_params"][0]["key"] == "userName"
assert extras["chat_link"]["route_params"][1]["paramType"] == "theme"
_write_agent_config(agent_id, name="助手", description="", chat_link=None)
raw = yaml.safe_load((paths_module.get_paths().agent_dir(agent_id) / "config.yaml").read_text(encoding="utf-8"))
assert "chat_link" not in raw
assert _load_config_extras(agent_id)["chat_link"] is None
def test_normalize_agent_chat_link_rejects_unsafe_urls() -> None:
assert normalize_agent_chat_link({"url": ""}) is None
assert normalize_agent_chat_link({"url": "javascript:alert(1)"}, strict=False) is None
with pytest.raises(ValueError):
normalize_agent_chat_link({"url": "javascript:alert(1)"}, strict=True)
kept = normalize_agent_chat_link({"url": "/page/workspace/agents", "open_mode": "blank"})
assert kept is not None
assert kept["url"] == "/page/workspace/agents"
loaded = AgentConfig(name="x", chat_link={"url": "javascript:alert(1)"})
assert loaded.chat_link is None
@pytest.mark.asyncio
async def test_backfill_listing_cache_mirrors_chat_link() -> None:
store = MemoryAgentStore()
link = {"url": "https://example.test/a", "open_mode": "blank", "route_params": []}
cfg = AgentConfig(id="a1", name="Agent", chat_link=link)
await _backfill_listing_cache(store, [("a1", cfg)])
extras = await store.get_extras_for(["a1"])
assert extras["a1"]["chat_link"]["url"] == "https://example.test/a"
await _backfill_listing_cache(store, [("a1", cfg)])
assert (await store.get_extras_for(["a1"]))["a1"]["chat_link"]["url"] == "https://example.test/a"
@pytest.mark.asyncio
@@ -538,3 +904,51 @@ async def test_backfill_listing_cache_is_idempotent_and_skips_unchanged(monkeypa
cfg2 = AgentConfig(id="a1", name="Agent", skills=["s1", "s2"], tool_groups=None, model=None)
await _backfill_listing_cache(store, [("a1", cfg2)])
assert calls["replace"] == 1
@pytest.mark.asyncio
async def test_paginated_agent_list_coalesces_concurrent_reads_and_returns_copies() -> None:
_agent_list_cache.clear()
_agent_list_inflight.clear()
calls = 0
async def loader() -> AgentsListResponse:
nonlocal calls
calls += 1
await asyncio.sleep(0.01)
return AgentsListResponse(agents=[], total=3, page=1, page_size=20)
first, second = await asyncio.gather(
_cached_paginated_agent_list(("user", 1), loader),
_cached_paginated_agent_list(("user", 1), loader),
)
assert calls == 1
assert {first[1], second[1]} == {"未命中", "合并等待"}
first[0].total = 99
cached, cache_status = await _cached_paginated_agent_list(("user", 1), loader)
assert cache_status == "命中"
assert cached.total == 3
assert calls == 1
@pytest.mark.asyncio
async def test_agent_owner_usernames_are_batched_and_cached(monkeypatch) -> None:
from app.gateway import deps as deps_module
_agent_owner_name_cache.clear()
class Provider:
def __init__(self):
self.calls = []
async def get_users_by_ids(self, user_ids):
self.calls.append(list(user_ids))
return [SimpleNamespace(id="owner-1", email="alice@example.com")]
provider = Provider()
monkeypatch.setattr(deps_module, "get_local_provider", lambda: provider)
assert await _resolve_usernames({"owner-1", "deleted-user"}) == {"owner-1": "alice"}
assert await _resolve_usernames({"owner-1", "deleted-user"}) == {"owner-1": "alice"}
assert provider.calls == [["deleted-user", "owner-1"]]
@@ -110,6 +110,17 @@ def _state(material_source="knowledge_base"):
}
def _patch_whitelist(monkeypatch, skills):
"""把素材收集专家(ai-writing-researcher)的 skills 白名单打成固定值。"""
import deerflow.config.agents_config as agents_config_module
monkeypatch.setattr(
agents_config_module,
"load_agent_config",
lambda _agent_id: SimpleNamespace(skills=list(skills)),
)
def test_general_skill_candidates_do_not_resurrect_explicitly_offline_knowledge_search(
tmp_path,
monkeypatch,
@@ -120,6 +131,7 @@ def test_general_skill_candidates_do_not_resurrect_explicitly_offline_knowledge_
encoding="utf-8",
)
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
_patch_whitelist(monkeypatch, ["knowledge-base-search", "other-search"])
other = SimpleNamespace(name="other-search", description="other")
@@ -140,6 +152,7 @@ def test_general_skill_candidates_keep_missing_catalog_fallback(tmp_path, monkey
extensions_path = tmp_path / "extensions_config.json"
extensions_path.write_text('{"mcpServers": {}, "skills": {}}', encoding="utf-8")
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
_patch_whitelist(monkeypatch, [])
class _Storage:
def load_skills(self, *, enabled_only=False):
@@ -154,6 +167,67 @@ def test_general_skill_candidates_keep_missing_catalog_fallback(tmp_path, monkey
assert [skill.name for skill in R._fallback_candidate_skills(set())] == [R.KNOWLEDGE_SEARCH_SKILL_NAME]
def test_general_skill_candidates_limited_to_researcher_whitelist(tmp_path, monkeypatch) -> None:
"""300+ 技能部署:候选池只取白名单里的启用技能,其余一概不进池。"""
extensions_path = tmp_path / "extensions_config.json"
extensions_path.write_text('{"mcpServers": {}, "skills": {}}', encoding="utf-8")
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
_patch_whitelist(monkeypatch, ["custom-a", "knowledge-base-search", "custom-b", "missing-skill"])
def _skill(name: str):
return SimpleNamespace(name=name, description=f"desc-{name}")
catalog = [_skill(f"bulk-{i}") for i in range(300)] + [
_skill("custom-b"),
_skill("custom-a"),
_skill(R.KNOWLEDGE_SEARCH_SKILL_NAME),
]
class _Storage:
def load_skills(self, *, enabled_only=False):
return catalog
import deerflow.skills.storage as storage_module
monkeypatch.setattr(storage_module, "get_or_new_skill_storage", lambda: _Storage())
# 白名单顺序保留,但内置知识库检索始终提到最前;目录里不存在的
# missing-skill 与 300 个 bulk-* 全部排除。
assert [skill.name for skill in R._load_candidate_skills()] == [
R.KNOWLEDGE_SEARCH_SKILL_NAME,
"custom-a",
"custom-b",
]
def test_fallback_candidates_stay_inside_researcher_whitelist(tmp_path, monkeypatch) -> None:
"""容错兜底也不回到全量技能目录:仍限定在白名单内。"""
extensions_path = tmp_path / "extensions_config.json"
extensions_path.write_text('{"mcpServers": {}, "skills": {}}', encoding="utf-8")
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
_patch_whitelist(monkeypatch, ["knowledge-base-search", "custom-a"])
def _skill(name: str):
return SimpleNamespace(name=name, description=f"desc-{name}")
class _Storage:
def load_skills(self, *, enabled_only=False):
return [_skill("custom-a"), _skill("bulk-other"), _skill(R.KNOWLEDGE_SEARCH_SKILL_NAME)]
import deerflow.skills.storage as storage_module
monkeypatch.setattr(storage_module, "get_or_new_skill_storage", lambda: _Storage())
assert [skill.name for skill in R._fallback_candidate_skills(set())] == [
R.KNOWLEDGE_SEARCH_SKILL_NAME,
"custom-a",
]
# 已试过的技能不再进入兜底池
assert [skill.name for skill in R._fallback_candidate_skills({"custom-a"})] == [
R.KNOWLEDGE_SEARCH_SKILL_NAME,
]
@pytest.mark.asyncio
async def test_knowledge_base_hits_intranet_directly(monkeypatch):
_patch(monkeypatch)
@@ -0,0 +1,126 @@
"""Integration coverage for persisted, administrator-owned performance controls."""
from types import SimpleNamespace
from unittest.mock import AsyncMock
import httpx
import pytest
import pytest_asyncio
from fastapi import FastAPI, HTTPException
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from test_api_read_cache import FakeRedis, _config
from app.gateway import api_read_cache as cache
from app.gateway.deps import get_config
from app.gateway.routers import browser_context, system_settings
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
@pytest_asyncio.fixture
async def controls(monkeypatch, tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'controls.db'}")
async with engine.begin() as connection:
await connection.run_sync(ApiReadCacheSettingRow.__table__.create)
factory = async_sessionmaker(engine, expire_on_commit=False)
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: factory)
monkeypatch.setattr(cache, "_api_read_cache_settings_cache", None)
redis = FakeRedis()
monkeypatch.setattr(cache, "_client", lambda *_args, **_kwargs: (redis, "test:controls"))
config = _config()
config.browser_context = BrowserContextConfig(enabled=True)
auth = AsyncMock(return_value=SimpleNamespace(id="admin", system_role="admin"))
monkeypatch.setattr(system_settings, "get_current_user_from_request", auth)
app = FastAPI()
app.state.config = config
app.dependency_overrides[get_config] = lambda: config
app.include_router(system_settings.router)
app.include_router(browser_context.router)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
yield client, config, redis, auth
await engine.dispose()
@pytest.mark.asyncio
async def test_partial_updates_persist_without_resetting_other_switches(controls, monkeypatch):
client, config, _, _ = controls
endpoint = "/api/system-settings/api-read-cache"
response = await client.put(endpoint, json={"enabled": True, "ttl_seconds": 777, "collection_overrides": {"agents": False}})
assert response.status_code == 200
assert {row["id"]: row["enabled"] for row in response.json()["collections"]}["agents"] is False
response = await client.put(endpoint, json={"browser_status_enabled": False, "collection_overrides": {"skills": False}})
assert response.status_code == 200
monkeypatch.setattr(cache, "_api_read_cache_settings_cache", None)
result = (await client.get(endpoint)).json()
assert result["enabled"] is True and result["ttl_seconds"] == 777
assert result["browser_status_enabled"] is False
rows = {row["id"]: row for row in result["collections"]}
assert not rows["agents"]["enabled"] and not rows["skills"]["enabled"]
assert rows["light_apps"]["enabled"]
assert all(row["paths"] and row["scope"] and row["ttl_seconds"] == 777 for row in rows.values())
assert await cache.is_browser_status_enabled(config) is False
@pytest.mark.asyncio
async def test_per_collection_disable_bypasses_reads_and_writes(controls):
client, config, redis, _ = controls
await cache.set_cached_json(config, "agents", "u:1", {"old": True})
await client.put("/api/system-settings/api-read-cache", json={"collection_overrides": {"agents": False}})
snapshot = dict(redis.values)
assert await cache.get_cached_json(config, "agents", "u:1", version="0") is None
assert await cache.get_cache_version(config, "agents") is None
assert await cache.wait_for_cached_json_or_claim_fill(config, "agents", "u:1", version="0") == (None, None)
await cache.set_cached_json(config, "agents", "u:1", {"new": True}, expected_version="0")
assert redis.values == snapshot
await cache.set_cached_json(config, "skills", "u:1", {"enabled": True})
assert await cache.get_cached_json(config, "skills", "u:1") == {"enabled": True}
await client.put("/api/system-settings/api-read-cache", json={"collection_overrides": {"agents": True}})
assert await cache.get_cached_json(config, "agents", "u:1") is None
@pytest.mark.asyncio
async def test_browser_access_gate_precedes_warm_cache_and_recovers(controls):
client, _, redis, _ = controls
assert (await client.get("/api/browser-context/status")).status_code == 200
assert any(":browser_status:v" in key for key in redis.values)
response = await client.put("/api/system-settings/api-read-cache", json={"browser_status_enabled": False})
assert response.status_code == 200
response = await client.get("/api/browser-context/status")
assert response.status_code == 403
assert response.json()["detail"]["code"] == "browser_status_disabled"
assert response.headers["Retry-After"] == "300"
await client.put("/api/system-settings/api-read-cache", json={"browser_status_enabled": True})
assert (await client.get("/api/browser-context/status")).json()["enabled"] is True
@pytest.mark.asyncio
@pytest.mark.parametrize("payload", [{"ttl_seconds": 0}, {"ttl_seconds": 3601}, {"collection_overrides": {"typo": True}}])
async def test_invalid_settings_are_rejected_without_persisting(controls, payload):
client, _, _, _ = controls
assert (await client.put("/api/system-settings/api-read-cache", json=payload)).status_code == 422
assert (await client.get("/api/system-settings/api-read-cache")).json()["ttl_seconds"] == 300
@pytest.mark.asyncio
@pytest.mark.parametrize("role,status", [("user", 403), ("anonymous", 401)])
async def test_only_admin_can_read_or_update_controls(controls, role, status):
client, _, _, auth = controls
if role == "anonymous":
auth.side_effect = HTTPException(status_code=401, detail="Login required")
else:
auth.return_value = SimpleNamespace(system_role=role)
assert (await client.get("/api/system-settings/api-read-cache")).status_code == status
assert (await client.put("/api/system-settings/api-read-cache", json={"browser_status_enabled": False})).status_code == status
@pytest.mark.asyncio
async def test_redis_failure_falls_back_and_global_off_preserves_individual_choices(controls, monkeypatch):
client, config, redis, _ = controls
await client.put("/api/system-settings/api-read-cache", json={"enabled": False, "collection_overrides": {"agents": False}})
assert await cache.get_cache_version(config, "skills") is None
response = await client.put("/api/system-settings/api-read-cache", json={"enabled": True})
assert next(row for row in response.json()["collections"] if row["id"] == "agents")["enabled"] is False
monkeypatch.setattr(redis, "get", AsyncMock(side_effect=ConnectionError("offline")))
assert await cache.get_cached_json(config, "skills", "u:1") is None
await cache.set_cached_json(config, "skills", "u:1", {"ok": True})
assert (await client.get("/api/browser-context/status")).status_code == 200
@@ -0,0 +1,201 @@
"""Concurrent changes must not lose controls or serve invalidated catalogue data."""
import asyncio
import importlib.util
from pathlib import Path
import pytest
import pytest_asyncio
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import create_engine, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from test_api_read_cache import FakeRedis, _config
from app.gateway import api_read_cache as cache
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
@pytest_asyncio.fixture
async def settings_database(monkeypatch, tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'edge-controls.db'}")
async with engine.begin() as connection:
await connection.run_sync(ApiReadCacheSettingRow.__table__.create)
class SlowReadSession(AsyncSession):
async def get(self, *args, **kwargs):
result = await super().get(*args, **kwargs)
# Simulate real database latency between reading settings and writing
# an independently edited interface toggle from another admin tab.
await asyncio.sleep(0.025)
return result
factory = async_sessionmaker(engine, class_=SlowReadSession, expire_on_commit=False)
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: factory)
monkeypatch.setattr(cache, "_api_read_cache_settings_cache", None)
yield factory
await engine.dispose()
@pytest.mark.asyncio
async def test_concurrent_first_saves_keep_both_individual_choices(settings_database):
config = _config()
results = await asyncio.gather(
cache.save_api_read_cache_settings(None, config=config, collection_overrides={"agents": False}),
cache.save_api_read_cache_settings(None, config=config, collection_overrides={"skills": False}),
return_exceptions=True,
)
assert not [result for result in results if isinstance(result, Exception)]
async with settings_database() as session:
row = await session.get(ApiReadCacheSettingRow, "default")
assert row.collection_overrides == {"agents": False, "skills": False}
@pytest.mark.asyncio
async def test_concurrent_updates_do_not_replace_other_interface_choices(settings_database):
config = _config()
await cache.save_api_read_cache_settings(True, 777, config=config)
await asyncio.gather(
cache.save_api_read_cache_settings(None, config=config, collection_overrides={"agents": False}),
cache.save_api_read_cache_settings(None, config=config, collection_overrides={"skills": False}),
)
async with settings_database() as session:
row = await session.get(ApiReadCacheSettingRow, "default")
assert row.collection_overrides == {"agents": False, "skills": False}
assert row.enabled is True
assert row.ttl_seconds == 777
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["disable", "invalidate"])
async def test_waiting_requests_reject_fill_changed_before_wakeup(monkeypatch, change):
config = _config()
redis = FakeRedis()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None)
monkeypatch.setattr(cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(cache, "_fill_events", {})
monkeypatch.setattr(cache, "_fill_local_payloads", {})
monkeypatch.setattr(cache, "_client", lambda *_args, **_kwargs: (redis, "test:edge"))
version = await cache.get_cache_version(config, "agents")
_, owner = await cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=version)
assert owner
waiter = asyncio.create_task(cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=version, wait_ms=1000))
await asyncio.sleep(0.01)
await cache.set_cached_json(config, "agents", "u1", {"old": True}, expected_version=version)
if change == "disable":
cache.set_api_read_cache_settings_local(True, 300, collection_overrides={"agents": False})
else:
await cache.invalidate_cached_collection(config, "agents", force=True)
await cache.release_cache_fill(config, "agents", "u1", owner)
value, new_owner = await waiter
if new_owner:
await cache.release_cache_fill(config, "agents", "u1", new_owner)
assert value is None, "A waiting request must not use the old local payload after controls/data changed"
def test_controls_migration_preserves_existing_ttl_and_defaults_status_to_enabled(tmp_path):
migration_path = Path(__file__).parents[1] / "packages/harness/deerflow/persistence/migrations/versions/20261029_02_api_performance_controls.py"
spec = importlib.util.spec_from_file_location("performance_controls_migration", migration_path)
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)
engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}")
with engine.begin() as connection:
connection.execute(text("CREATE TABLE api_read_cache_settings (key VARCHAR(32) PRIMARY KEY, enabled BOOLEAN NOT NULL, ttl_seconds INTEGER NOT NULL)"))
connection.execute(text("INSERT INTO api_read_cache_settings VALUES ('default', 1, 777)"))
with Operations.context(MigrationContext.configure(connection)):
migration.upgrade()
migration.upgrade() # Existing/create_all schemas are safe to revisit.
row = connection.execute(text("SELECT * FROM api_read_cache_settings")).mappings().one()
assert row["ttl_seconds"] == 777
assert bool(row["enabled"]) is True
assert bool(row["browser_status_enabled"]) is True
assert row["collection_overrides"] is None
assert row["cache_epoch"] == "0"
engine.dispose()
@pytest.mark.asyncio
async def test_saved_epoch_survives_redis_outage_and_process_restart(settings_database, monkeypatch):
config = _config()
class RecoverableRedis(FakeRedis):
online = True
async def get(self, key):
if not self.online:
raise ConnectionError("cache offline")
return await super().get(key)
async def incr(self, key):
if not self.online:
raise ConnectionError("cache offline")
return await super().incr(key)
redis = RecoverableRedis()
# Keep the real _client prefix construction, while using an in-memory
# Redis connection shared by the simulated old/new Gateway processes.
monkeypatch.setattr(cache, "_clients", {})
monkeypatch.setattr("redis.asyncio.Redis.from_url", lambda *args, **kwargs: redis)
await cache.save_api_read_cache_settings(True, config=config)
await cache.set_cached_json(config, "agents", "u1", {"old": True})
assert await cache.get_cached_json(config, "agents", "u1") == {"old": True}
old_keys = set(redis.values)
old_version = await cache.get_cache_version(config, "agents")
redis.online = False
await cache.save_api_read_cache_settings(False, config=config)
await cache.invalidate_cached_collection(config, "agents", force=True)
await cache.save_api_read_cache_settings(True, config=config)
await cache.invalidate_cached_collection(config, "agents", force=True)
# Restart clears every process-local control/client; DB and Redis survive.
monkeypatch.setattr(cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(cache, "_clients", {})
redis.online = True
assert await cache.get_cached_json(config, "agents", "u1") is None
assert old_keys.issubset(redis.values), "Old keys still exist but must be unreachable"
assert await cache.get_cache_version(config, "agents") != old_version
await cache.set_cached_json(config, "agents", "u1", {"fresh": True})
assert await cache.get_cached_json(config, "agents", "u1") == {"fresh": True}
@pytest.mark.asyncio
async def test_save_fences_fills_started_in_the_previous_epoch(settings_database, monkeypatch):
config = _config()
redis = FakeRedis()
monkeypatch.setattr(cache, "_clients", {})
monkeypatch.setattr("redis.asyncio.Redis.from_url", lambda *args, **kwargs: redis)
await cache.save_api_read_cache_settings(True, config=config)
version = await cache.get_cache_version(config, "agents")
_, owner = await cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=version)
waiter = asyncio.create_task(cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=version, wait_ms=1000))
await asyncio.sleep(0.01)
await cache.save_api_read_cache_settings(True, config=config, collection_overrides={"skills": False})
await cache.set_cached_json(config, "agents", "u1", {"stale": True}, expected_version=version)
await cache.release_cache_fill(config, "agents", "u1", owner)
value, token = await waiter
assert (value, token) == (None, None)
assert await cache.get_cached_json(config, "agents", "u1") is None
@pytest.mark.asyncio
async def test_old_epoch_owner_release_does_not_wake_new_epoch_waiters(settings_database, monkeypatch):
config = _config()
redis = FakeRedis()
monkeypatch.setattr(cache, "_clients", {})
monkeypatch.setattr("redis.asyncio.Redis.from_url", lambda *args, **kwargs: redis)
await cache.save_api_read_cache_settings(True, config=config)
old_version = await cache.get_cache_version(config, "agents")
_, old_owner = await cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=old_version)
await cache.save_api_read_cache_settings(True, config=config)
version = await cache.get_cache_version(config, "agents")
_, owner = await cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=version)
waiter = asyncio.create_task(
cache.wait_for_cached_json_or_claim_fill(config, "agents", "u1", version=version, wait_ms=1000)
)
await asyncio.sleep(0.01)
await cache.release_cache_fill(config, "agents", "u1", old_owner)
await asyncio.sleep(0.01)
assert not waiter.done()
await cache.set_cached_json(config, "agents", "u1", {"fresh": True}, expected_version=version)
await cache.release_cache_fill(config, "agents", "u1", owner)
assert await waiter == ({"fresh": True}, None)
@@ -0,0 +1,453 @@
from __future__ import annotations
import asyncio
from pathlib import Path
from types import SimpleNamespace
import pytest
from app.gateway import api_read_cache
class FakeRedis:
def __init__(self) -> None:
self.values: dict[str, bytes] = {}
self.expirations: dict[str, int] = {}
async def get(self, key: str) -> bytes | None:
return self.values.get(key)
async def set(self, key: str, value: bytes | str, *, ex: int, nx: bool = False) -> bool | None:
assert ex > 0
if nx and key in self.values:
return None
self.values[key] = value if isinstance(value, bytes) else value.encode("utf-8")
self.expirations[key] = ex
return True
async def incr(self, key: str) -> int:
value = int(self.values.get(key, b"0")) + 1
self.values[key] = str(value).encode("ascii")
return value
async def eval(self, _script: str, _num_keys: int, key: str, token: str) -> int:
if self.values.get(key) == token.encode("utf-8"):
del self.values[key]
return 1
return 0
def _config(*, enabled: bool = True) -> SimpleNamespace:
return SimpleNamespace(
api_read_cache=SimpleNamespace(
enabled=enabled,
url=None,
password=None,
namespace="test:homepage",
ttl_seconds=300,
max_value_bytes=524288,
socket_connect_timeout_ms=50,
socket_timeout_ms=50,
),
concurrency=SimpleNamespace(
redis=SimpleNamespace(
enabled=enabled,
url="redis://cache.test/0",
password=None,
namespace="test:deerflow",
socket_connect_timeout_ms=50,
socket_timeout_ms=50,
)
)
)
def test_redis_cache_ttl_defaults_to_five_minutes_and_stays_bounded() -> None:
assert api_read_cache._cache_ttl_seconds(SimpleNamespace(api_read_cache=SimpleNamespace())) == 300
assert api_read_cache._cache_ttl_seconds(
SimpleNamespace(api_read_cache=SimpleNamespace(ttl_seconds=9999))
) == 3600
@pytest.mark.asyncio
async def test_cache_round_trip_and_generation_invalidation(monkeypatch: pytest.MonkeyPatch) -> None:
redis = FakeRedis()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None)
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(api_read_cache, "_client", lambda _config, **_kwargs: (redis, "test:cache"))
config = _config()
await api_read_cache.set_cached_json(config, "agents", "user:u1", {"items": [1, 2]})
assert await api_read_cache.get_cached_json(config, "agents", "user:u1") == {"items": [1, 2]}
payload = await api_read_cache.get_cached_json_payload(config, "agents", "user:u1")
assert payload == b'{"items":[1,2]}'
await api_read_cache.invalidate_cached_collection(config, "agents")
assert await api_read_cache.get_cached_json(config, "agents", "user:u1") is None
@pytest.mark.asyncio
async def test_concurrent_cache_misses_wait_for_single_fill(monkeypatch: pytest.MonkeyPatch) -> None:
redis = FakeRedis()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None)
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(api_read_cache, "_client", lambda _config, **_kwargs: (redis, "test:cache"))
config = _config()
_, owner_token = await api_read_cache.wait_for_cached_json_or_claim_fill(
config, "agents", "user:u1", version="0"
)
assert owner_token is not None
waiter = asyncio.create_task(
api_read_cache.wait_for_cached_json_or_claim_fill(
config, "agents", "user:u1", version="0", wait_ms=500
)
)
await asyncio.sleep(0.05)
await api_read_cache.set_cached_json(
config, "agents", "user:u1", {"items": ["shared fill"]}, expected_version="0"
)
await api_read_cache.release_cache_fill(config, "agents", "user:u1", owner_token)
cached, waiter_token = await waiter
assert cached == {"items": ["shared fill"]}
assert waiter_token is None
assert not any(":fill:" in key for key in redis.values)
stale_version = await api_read_cache.get_cache_version(config, "agents")
await api_read_cache.invalidate_cached_collection(config, "agents")
await api_read_cache.set_cached_json(
config,
"agents",
"user:u1",
{"items": ["stale"]},
expected_version=stale_version,
)
assert await api_read_cache.get_cached_json(config, "agents", "user:u1") is None
@pytest.mark.asyncio
async def test_concurrent_http_payload_misses_wait_for_single_fill(monkeypatch: pytest.MonkeyPatch) -> None:
redis = FakeRedis()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None)
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(api_read_cache, "_client", lambda _config, **_kwargs: (redis, "test:cache"))
config = _config()
_, owner_token = await api_read_cache.wait_for_cached_json_payload_or_claim_fill(
config, "skills", "user:u1", version="0"
)
assert owner_token is not None
waiter = asyncio.create_task(
api_read_cache.wait_for_cached_json_payload_or_claim_fill(
config, "skills", "user:u1", version="0", wait_ms=500
)
)
await asyncio.sleep(0.05)
await api_read_cache.set_cached_json(
config,
"skills",
"user:u1",
{"skills": [{"name": "one"}]},
expected_version="0",
)
await api_read_cache.release_cache_fill(config, "skills", "user:u1", owner_token)
payload, waiter_token = await waiter
assert payload == b'{"skills":[{"name":"one"}]}'
assert waiter_token is None
@pytest.mark.asyncio
async def test_disabled_redis_uses_the_database_path() -> None:
config = _config(enabled=True)
config.api_read_cache = SimpleNamespace(enabled=False)
assert api_read_cache._redis_settings(config) is None
api_read_cache.set_api_read_cache_enabled_local(False)
assert await api_read_cache.get_cached_json(config, "skills", "user:u1") is None
await api_read_cache.set_cached_json(config, "skills", "user:u1", {"items": []})
def test_homepage_cache_can_reuse_a_separate_redis_connection() -> None:
config = _config(enabled=False)
config.api_read_cache = SimpleNamespace(
enabled=True,
url="redis://homepage-cache.test/2",
password="secret",
namespace="test:homepage",
)
assert api_read_cache._redis_settings(config) == (
"redis://homepage-cache.test/2",
"secret",
"test:homepage",
)
def test_enabled_homepage_cache_reuses_concurrency_redis() -> None:
config = _config(enabled=True)
config.api_read_cache = SimpleNamespace(
enabled=True,
url=None,
password=None,
namespace="test:homepage",
)
assert api_read_cache._redis_settings(config) == (
"redis://cache.test/0",
None,
"test:homepage",
)
def test_separate_cache_redis_does_not_inherit_an_unrelated_password() -> None:
config = _config(enabled=True)
config.concurrency.redis.password = "concurrency-secret"
config.api_read_cache = SimpleNamespace(
enabled=True,
url="redis://homepage-cache.test/2",
password=None,
namespace="test:homepage",
)
assert api_read_cache._redis_settings(config) == (
"redis://homepage-cache.test/2",
None,
"test:homepage",
)
@pytest.mark.asyncio
async def test_admin_cache_switch_and_ttl_are_persisted_and_survive_process_refresh(monkeypatch: pytest.MonkeyPatch) -> None:
from deerflow.persistence.api_read_cache_settings.model import ApiReadCacheSettingRow
class FakeSession:
def __init__(self) -> None:
self.row: ApiReadCacheSettingRow | None = None
self.added: list[ApiReadCacheSettingRow] = []
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return False
async def get(self, _model, key: str, **_kwargs):
assert key == "default"
return self.row
def add(self, row: ApiReadCacheSettingRow) -> None:
self.added.append(row)
self.row = row
async def commit(self) -> None:
return None
session = FakeSession()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: lambda: session)
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
config = _config(enabled=False)
assert await api_read_cache.get_api_read_cache_settings(config, force_refresh=True) == (False, 300)
await api_read_cache.save_api_read_cache_settings(True, 900, config=config)
assert session.row is not None and session.row.enabled is True
assert session.row.ttl_seconds == 900
assert await api_read_cache.get_api_read_cache_settings(config) == (True, 900)
# A new process has no in-memory value and reads the persisted MySQL row.
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
assert await api_read_cache.get_api_read_cache_settings(config, force_refresh=True) == (True, 900)
await api_read_cache.save_api_read_cache_enabled(False)
assert session.row.enabled is False
assert session.row.ttl_seconds == 900
@pytest.mark.asyncio
async def test_cache_toggle_mysql_read_is_singleflight_under_concurrency(monkeypatch: pytest.MonkeyPatch) -> None:
class FakeSession:
def __init__(self) -> None:
self.get_calls = 0
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return False
async def get(self, _model, _key):
self.get_calls += 1
await asyncio.sleep(0.01)
return None
session = FakeSession()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: lambda: session)
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
config = _config(enabled=False)
results = await asyncio.gather(
*(api_read_cache.is_api_read_cache_enabled(config) for _ in range(70))
)
assert results == [False] * 70
assert session.get_calls == 1
@pytest.mark.asyncio
async def test_configured_mysql_ttl_is_used_for_agent_and_skill_cache_writes(monkeypatch: pytest.MonkeyPatch) -> None:
redis = FakeRedis()
class FakeSession:
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return False
async def get(self, _model, _key):
return SimpleNamespace(enabled=True, ttl_seconds=900)
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: lambda: FakeSession())
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(api_read_cache, "_client", lambda _config, **_kwargs: (redis, "test:cache"))
config = _config()
for collection in ("agents", "skills"):
await api_read_cache.set_cached_json(config, collection, "user:u1", {"items": []})
expirations = [
expiry
for key, expiry in redis.expirations.items()
if f":{collection}:v0:" in key
]
assert expirations == [900]
def test_existing_api_cache_setting_receives_five_minute_ttl(tmp_path: Path) -> None:
import sqlalchemy as sa
from alembic import command
from alembic.config import Config as AlembicConfig
database = tmp_path / "api-read-cache.db"
engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}")
with engine.begin() as connection:
connection.exec_driver_sql(
"CREATE TABLE api_read_cache_settings ("
"`key` VARCHAR(32) PRIMARY KEY, enabled BOOLEAN NOT NULL, updated_at DATETIME NOT NULL)"
)
connection.exec_driver_sql(
"INSERT INTO api_read_cache_settings (`key`, enabled, updated_at) "
"VALUES ('default', 1, CURRENT_TIMESTAMP)"
)
connection.exec_driver_sql(
"CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL PRIMARY KEY)"
)
connection.exec_driver_sql(
"INSERT INTO alembic_version (version_num) VALUES ('20261026_02')"
)
engine.dispose()
migrations = Path(__file__).parents[1] / "packages/harness/deerflow/persistence/migrations"
config = AlembicConfig(str(migrations / "alembic.ini"))
config.set_main_option(
"sqlalchemy.url",
f"sqlite+aiosqlite:///{database.resolve().as_posix()}",
)
command.upgrade(config, "head")
engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}")
with engine.connect() as connection:
setting = connection.execute(
sa.text("SELECT enabled, ttl_seconds, collection_overrides, browser_status_enabled, cache_epoch FROM api_read_cache_settings WHERE `key`='default'")
).one()
revision = connection.execute(sa.text("SELECT version_num FROM alembic_version")).scalar_one()
engine.dispose()
assert bool(setting.enabled) is True
assert setting.ttl_seconds == 300
assert setting.collection_overrides is None
assert bool(setting.browser_status_enabled) is True
assert setting.cache_epoch == "0"
assert revision == "20261102_02"
@pytest.mark.asyncio
async def test_skill_catalog_projection_uses_compact_rows_without_long_fields(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
from sqlalchemy import update
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway import skill_catalog_cache
from deerflow.persistence.skills.model import SkillRow
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(SkillRow.__table__.create)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
monkeypatch.setattr(skill_catalog_cache, "get_session_factory", lambda: session_factory)
monkeypatch.setattr(skill_catalog_cache, "get_or_new_skill_storage", lambda **_kwargs: SimpleNamespace())
monkeypatch.setattr(skill_catalog_cache, "_load_usage_entries", lambda: {})
skill_file = tmp_path / "SKILL.md"
skill_file.write_text("---\nname: catalog-demo\n---\n", encoding="utf-8")
skill = SimpleNamespace(
name="catalog-demo",
description="compact metadata",
license="MIT",
category="public",
enabled=True,
always_on=False,
skill_file=skill_file,
)
assert await skill_catalog_cache.refresh_skill_catalog_projection(skills=[skill]) == 1
async with session_factory() as session:
await session.execute(update(SkillRow).where(SkillRow.name == "catalog-demo").values(detail="x" * 5000))
await session.commit()
rows = await skill_catalog_cache.load_skill_catalog_projection(user_id="user-1")
assert rows is not None and len(rows) == 1
assert rows[0]["name"] == "catalog-demo"
assert rows[0]["description"] == "compact metadata"
assert "detail" not in rows[0]
assert "detail" not in {column.key for column in skill_catalog_cache._SKILL_LIST_COLUMNS}
await engine.dispose()
@pytest.mark.asyncio
async def test_skill_catalog_schema_adds_columns_to_existing_table_without_losing_data(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from sqlalchemy import inspect
from sqlalchemy.ext.asyncio import create_async_engine
from app.gateway import skill_catalog_cache
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.exec_driver_sql(
"CREATE TABLE skills ("
"name VARCHAR(64) PRIMARY KEY, owner_user_id VARCHAR(64), "
"published BOOLEAN NOT NULL DEFAULT 0, detail TEXT, name_zh VARCHAR(128), "
"always_on BOOLEAN NOT NULL DEFAULT 0, square_id VARCHAR(64) NOT NULL DEFAULT '', "
"created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL)"
)
await connection.exec_driver_sql(
"INSERT INTO skills (name, detail, created_at, updated_at) "
"VALUES ('existing-skill', 'keep this annotation', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
)
monkeypatch.setattr(skill_catalog_cache, "get_engine", lambda: engine)
await skill_catalog_cache.ensure_skill_catalog_schema()
# The migration must be safe to retry after process restarts.
await skill_catalog_cache.ensure_skill_catalog_schema()
async with engine.connect() as connection:
columns = await connection.run_sync(
lambda sync_connection: {column["name"] for column in inspect(sync_connection).get_columns("skills")}
)
preserved = (
await connection.exec_driver_sql(
"SELECT detail, available, category, detail_preview FROM skills WHERE name='existing-skill'"
)
).one()
assert "detail_preview" in columns
assert preserved[0] == "keep this annotation"
assert bool(preserved[1]) is True
assert preserved[2] == "custom"
assert preserved[3] == "keep this annotation"
await engine.dispose()
@@ -0,0 +1,151 @@
from io import BytesIO
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from starlette.datastructures import UploadFile
from app.gateway.routers import app_identity as app_identity_router
from deerflow.config.system_settings import AppIdentitySettings, SystemSettings
PNG = bytes.fromhex(
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4"
"890000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082"
)
def _store(monkeypatch: pytest.MonkeyPatch, tmp_path):
current = {"settings": SystemSettings()}
def load():
return current["settings"]
def save(settings: SystemSettings) -> SystemSettings:
current["settings"] = settings
return settings
async def admin_user(_request):
return SimpleNamespace(system_role="admin")
monkeypatch.setattr(app_identity_router, "runtime_home", lambda: tmp_path)
monkeypatch.setattr("deerflow.config.system_settings.runtime_home", lambda: tmp_path)
monkeypatch.setattr(app_identity_router, "load_system_settings", load)
monkeypatch.setattr(app_identity_router, "save_system_settings", save)
monkeypatch.setattr("deerflow.config.system_settings.load_system_settings", load)
monkeypatch.setattr(
"app.gateway.routers.system_settings.get_optional_user_from_request",
admin_user,
)
return current
def test_new_settings_leave_app_identity_empty() -> None:
assert SystemSettings().app_identity == AppIdentitySettings()
@pytest.mark.asyncio
async def test_non_admin_cannot_change_app_identity(monkeypatch: pytest.MonkeyPatch) -> None:
async def ordinary_user(_request):
return SimpleNamespace(system_role="user")
monkeypatch.setattr(
"app.gateway.routers.system_settings.get_optional_user_from_request",
ordinary_user,
)
with pytest.raises(HTTPException) as error:
await app_identity_router.update_app_identity_settings(
SimpleNamespace(),
system_name="新系统",
remove_logo=False,
logo=None,
)
assert error.value.status_code == 403
@pytest.mark.asyncio
async def test_admin_saves_name_without_replacing_the_static_logo(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
current = _store(monkeypatch, tmp_path)
response = await app_identity_router.update_app_identity_settings(
SimpleNamespace(),
system_name=" 研判中心 ",
remove_logo=False,
logo=None,
)
assert response.system_name == "研判中心"
assert response.has_logo is False
assert response.logo_url is None
assert current["settings"].app_identity.logo_filename is None
@pytest.mark.asyncio
async def test_uploaded_logo_is_served_separately_from_generated_pages(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
current = _store(monkeypatch, tmp_path)
upload = UploadFile(file=BytesIO(PNG), filename="mark.png")
saved = await app_identity_router.update_app_identity_settings(
SimpleNamespace(),
system_name="研判中心",
remove_logo=False,
logo=upload,
)
assert saved.has_logo is True
assert saved.logo_url is not None
assert saved.logo_url.startswith("/api/public/app-identity/logo?v=")
logo_file = tmp_path / "app-identity" / current["settings"].app_identity.logo_filename
assert logo_file.is_file()
assert not (tmp_path / "generated-page-branding").exists()
served = await app_identity_router.get_app_identity_logo()
assert served.media_type == "image/png"
@pytest.mark.asyncio
async def test_invalid_logo_is_rejected(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
_store(monkeypatch, tmp_path)
upload = UploadFile(file=BytesIO(b"not-an-image"), filename="mark.gif")
with pytest.raises(HTTPException) as error:
await app_identity_router.update_app_identity_settings(
SimpleNamespace(),
system_name="研判中心",
remove_logo=False,
logo=upload,
)
assert error.value.status_code == 422
@pytest.mark.asyncio
async def test_clearing_app_identity_removes_only_its_logo(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
) -> None:
current = _store(monkeypatch, tmp_path)
upload = UploadFile(file=BytesIO(PNG), filename="mark.png")
await app_identity_router.update_app_identity_settings(
SimpleNamespace(),
system_name="研判中心",
remove_logo=False,
logo=upload,
)
logo_file = tmp_path / "app-identity" / current["settings"].app_identity.logo_filename
cleared = await app_identity_router.delete_app_identity_settings(SimpleNamespace())
assert cleared.system_name == ""
assert cleared.has_logo is False
assert logo_file.exists() is False
with pytest.raises(HTTPException) as error:
await app_identity_router.get_app_identity_logo()
assert error.value.status_code == 404
@@ -0,0 +1,152 @@
"""Actual async entry points must not block another request's MCP discovery.
Run each scenario in a child process: the regression blocks the event loop, so
an asyncio timeout alone cannot fail the test or clean up the waiting threads.
"""
from __future__ import annotations
import asyncio
import contextvars
import os
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
@pytest.mark.parametrize("entrypoint", ["open_chat", "open_chat_stream", "workflow"])
def test_async_entrypoint_coexists_with_cold_mcp_discovery(entrypoint):
backend_root = Path(__file__).resolve().parents[1]
env = dict(os.environ)
env["PYTHONPATH"] = os.pathsep.join(filter(None, (str(backend_root), env.get("PYTHONPATH"))))
result = subprocess.run(
[sys.executable, str(Path(__file__).resolve()), entrypoint],
cwd=backend_root,
env=env,
capture_output=True,
text=True,
timeout=40,
check=False,
)
assert result.returncode == 0, result.stdout + result.stderr
assert "concurrent MCP initialization completed" in result.stdout
async def _exercise_entrypoint(entrypoint: str) -> None:
import httpx
from fastapi import FastAPI
from langchain_core.messages import AIMessage
from app.gateway.routers import open_chat
from app.gateway.workflow_agent_runner import WorkflowAgentRunner
from deerflow.agents.lead_agent import agent as lead_agent
from deerflow.config import app_config
from deerflow.mcp import cache, tools
from deerflow.runtime.runs.worker import build_agent_with_timing
owner_loop = asyncio.get_running_loop()
discovery_started = asyncio.Event()
release_discovery = asyncio.Event()
built_thread = contextvars.ContextVar("entrypoint_built_thread", default=None)
discovery_calls = 0
async def discover():
nonlocal discovery_calls
assert asyncio.get_running_loop() is owner_loop
discovery_calls += 1
discovery_started.set()
await release_discovery.wait()
return ["discovered tool"]
class Agent:
def __init__(self, thread_id):
self.thread_id = thread_id
async def ainvoke(self, state, *, config):
assert built_thread.get() == self.thread_id
return {"messages": [AIMessage(content="answer")]}
async def astream(self, state, *, config, stream_mode):
assert built_thread.get() == self.thread_id
yield "values", {"messages": [AIMessage(content="answer")]}
def factory(config):
assert cache.get_cached_mcp_tools() == ["discovered tool"]
thread_id = config["configurable"]["thread_id"]
built_thread.set(thread_id)
return Agent(thread_id)
factory._deerflow_background_build = True
app = FastAPI()
app.include_router(open_chat.router)
@app.get("/probe")
async def probe():
return {"ok": True}
with pytest.MonkeyPatch.context() as patch:
patch.setattr(cache, "_mcp_tools_cache", None)
patch.setattr(cache, "_cache_initialized", False)
patch.setattr(cache, "_config_mtime", None)
patch.setattr(cache, "_initialization_lock", asyncio.Lock())
patch.setattr(cache, "_get_config_mtime", lambda: 1.0)
patch.setattr(tools, "get_mcp_tools", discover)
patch.setattr(open_chat, "make_lead_agent", factory)
patch.setattr(lead_agent, "make_lead_agent", factory)
patch.setattr(
app_config,
"get_app_config",
lambda: SimpleNamespace(workflows=SimpleNamespace(agent_recursion_limit=100)),
)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
# This is the same background construction path used by run_agent.
first = asyncio.create_task(
build_agent_with_timing(factory, {"configurable": {"thread_id": "normal-chat"}}, None, "normal-chat")
)
second = None
try:
await asyncio.wait_for(discovery_started.wait(), timeout=5)
if entrypoint == "workflow":
second = asyncio.create_task(
WorkflowAgentRunner(app).run_agent(
run_id="test-run", node_id="test-node", owner_id="test-owner", agent_id="test-agent", prompt="hello"
)
)
else:
path = "/api/open/chat/stream" if entrypoint == "open_chat_stream" else "/api/open/chat"
second = asyncio.create_task(client.post(path, json={"message": "hello", "include_files": False}))
# Let the real HTTP handler / workflow runner enter construction
# while discovery holds its lock and is awaiting network work.
await asyncio.sleep(0.05)
assert not first.done() and not second.done()
for _ in range(5):
response = await asyncio.wait_for(client.get("/probe"), timeout=1)
assert response.json() == {"ok": True}
await asyncio.sleep(0)
finally:
release_discovery.set()
results = await asyncio.wait_for(asyncio.gather(first, *([second] if second else [])), timeout=5)
assert discovery_calls == 1
assert built_thread.get() is None # Other request contexts remain isolated.
result = results[1]
if entrypoint == "workflow":
assert result["text"] == "answer"
assert result["thread_id"] == "wf-test-run-test-node"
elif entrypoint == "open_chat_stream":
assert result.status_code == 200
assert "event: result" in result.text
assert '"answer": "answer"' in result.text
assert "event: error" not in result.text
else:
assert result.status_code == 200
assert result.json()["answer"] == "answer"
print("concurrent MCP initialization completed", flush=True)
if __name__ == "__main__":
asyncio.run(_exercise_entrypoint(sys.argv[1]))
@@ -0,0 +1,176 @@
"""Async tool consumers must leave their MCP discovery loop runnable."""
import asyncio
import contextvars
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from deerflow.mcp import cache
from deerflow.mcp import tools as mcp_tools
from deerflow.tools import get_available_tools_async
from deerflow.tools import tools as tool_factory
from deerflow.tools.builtins.tool_search import (
DeferredToolRegistry,
get_deferred_registry,
set_deferred_registry,
)
from deerflow.utils.background_init import run_background_init
@pytest.mark.asyncio
async def test_async_tools_preserve_options_and_deferred_registry(monkeypatch):
owner_thread = threading.get_ident()
user = contextvars.ContextVar("async_tool_test_user")
user.set("caller")
registry = DeferredToolRegistry()
expected = [object()]
options = {
"groups": ["files"],
"include_mcp": False,
"model_name": "test-model",
"subagent_enabled": True,
"is_scheduled_run": True,
"excluded_tools": ["bash"],
"allowed_tools": ["read_file"],
"allow_agent_orchestration": True,
"app_config": object(),
"write_file_compatibility": True,
}
def load(**kwargs):
assert threading.get_ident() != owner_thread
assert user.get() == "caller"
assert kwargs == options
set_deferred_registry(registry)
return expected
monkeypatch.setattr(tool_factory, "get_available_tools", load)
assert await get_available_tools_async(**options) is expected
assert get_deferred_registry() is registry
@pytest.mark.asyncio
async def test_tool_loading_during_agent_mcp_discovery_keeps_owner_loop_free(monkeypatch):
loop = asyncio.get_running_loop()
started = asyncio.Event()
release = asyncio.Event()
discovery_calls = 0
expected = [object()]
identity = contextvars.ContextVar("async_tool_concurrent_identity")
registries = {name: DeferredToolRegistry() for name in ("first", "second")}
monkeypatch.setattr(cache, "_mcp_tools_cache", None)
monkeypatch.setattr(cache, "_cache_initialized", False)
monkeypatch.setattr(cache, "_config_mtime", None)
monkeypatch.setattr(cache, "_initialization_lock", asyncio.Lock())
monkeypatch.setattr(cache, "_get_config_mtime", lambda: 1.0)
async def discover():
nonlocal discovery_calls
discovery_calls += 1
assert asyncio.get_running_loop() is loop
started.set()
await release.wait()
return expected
def load(**kwargs):
name = kwargs["model_name"]
assert identity.get() == name
result = cache.get_cached_mcp_tools()
set_deferred_registry(registries[name])
return result
async def request(name):
identity.set(name)
result = await get_available_tools_async(model_name=name)
assert get_deferred_registry() is registries[name]
return result
monkeypatch.setattr(mcp_tools, "get_mcp_tools", discover)
monkeypatch.setattr(tool_factory, "get_available_tools", load)
# An agent initializer already owns MCP discovery when two tool consumers arrive.
agent = asyncio.create_task(run_background_init(cache.get_cached_mcp_tools))
consumers = []
try:
await asyncio.wait_for(started.wait(), timeout=3)
consumers = [asyncio.create_task(request(name)) for name in registries]
await asyncio.sleep(0.02)
assert not agent.done() and all(not task.done() for task in consumers)
assert discovery_calls == 1
finally:
release.set()
results = await asyncio.wait_for(asyncio.gather(agent, *consumers), timeout=3)
assert len(results) == 3 and all(result is expected for result in results)
assert discovery_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("entry_point", ["capabilities", "executor"])
async def test_deep_research_provider_initialization_leaves_loop_free(monkeypatch, entry_point):
from app.gateway import deep_research_job_executor as executor_module
from app.gateway.routers import deep_research as router
from deerflow.agents.deep_research.adapters import artifacts, context, event_sink, image, llm, material_provider
from deerflow.agents.deep_research.cancellation import CancellationToken
from deerflow.agents.deep_research.config import DeepResearchConfig
from deerflow.agents.deep_research.types import DeepResearchResult
loop = asyncio.get_running_loop()
owner_thread = threading.get_ident()
started = asyncio.Event()
release = threading.Event()
def load_tools(_model_name):
assert threading.get_ident() != owner_thread
loop.call_soon_threadsafe(started.set)
assert release.wait(3), "the owner loop did not release initialization"
return {"web_search"}
monkeypatch.setattr(material_provider, "_load_question_answer_tool_names", load_tools)
monkeypatch.setattr(material_provider, "_load_question_answer_research_skills", lambda: [])
monkeypatch.setattr(image.ImageGenerationSettings, "from_app_config", lambda: SimpleNamespace(is_configured=False))
monkeypatch.setattr(image, "build_image_provider", lambda: None)
monkeypatch.setattr(llm, "DeerFlowCompletionBackend", lambda *args, **kwargs: object())
monkeypatch.setattr(context, "DeerFlowContextCompressor", lambda **kwargs: object())
monkeypatch.setattr(artifacts, "ThreadOutputsArtifactWriter", lambda *args, **kwargs: object())
sink = SimpleNamespace(diagnostics=[], publish_live=AsyncMock())
monkeypatch.setattr(event_sink, "PersistingEventSink", lambda *args, **kwargs: sink)
monkeypatch.setattr(executor_module, "RecordingEventSink", lambda *args, **kwargs: sink)
engine_run = AsyncMock(return_value=DeepResearchResult(report_markdown="completed"))
monkeypatch.setattr(executor_module, "DeepResearchEngine", lambda: SimpleNamespace(run=engine_run))
executor = executor_module.DeepResearchJobExecutor(None, None, event_store=None)
monkeypatch.setattr(executor, "_ensure_runtime_thread", AsyncMock(return_value="open-test-research"))
monkeypatch.setattr(executor, "_load_interrupted_plan", AsyncMock(return_value=None))
if entry_point == "capabilities":
pending = router.get_capabilities()
else:
pending = executor._execute(
job_id="test-job",
session_id="test-session",
user_id="test-user",
query="test query",
title="test title",
config=DeepResearchConfig(),
lease_owner=None,
cancel_token=CancellationToken(),
)
task = asyncio.create_task(pending)
try:
await asyncio.wait_for(started.wait(), timeout=3)
await asyncio.sleep(0)
assert not task.done()
finally:
release.set()
result = await asyncio.wait_for(task, timeout=3)
if entry_point == "capabilities":
assert "materialChannels" in result
else:
provider = engine_run.call_args.kwargs["adapters"].materials
assert isinstance(provider, material_provider.DeerFlowMaterialProvider)
assert provider._question_answer_tool_names == {"web_search"}
assert provider._online_search_semaphore is None
assert provider._user_id == "test-user"
assert provider._runtime_thread_id == "open-test-research"
assert result.report_markdown == "completed"
@@ -0,0 +1,164 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "bbc-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_bbc_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"bbc_page_renderer",
SCRIPT_PATH,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "bbc-page-generator"
assert "BBC" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["bbc-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["bbc-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/bbc-page-generator/SKILL.md" in lead_prompt_source
assert "专用 BBC 路由" in lead_prompt_source
def test_default_json_renders_editable_bbc_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(
encoding="utf-8"
)
)
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "bbc-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "bbc-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["focus"] >= 1
assert renderer.DATA_MARKER not in html
assert renderer.RUNTIME_MARKER not in html
assert 'id="bbcPageData"' in html
assert 'id="leadLayout"' in html
assert 'id="mediaGrid"' in html
assert 'id="audioPlay"' in html
assert 'id="liveModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert 'data-video-id="' in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert '"brandLetters": [' in html
assert "<title>BBC 新闻 — 看见正在发生的世界</title>" in html
assert '"bbc-page-data-v1:" + pageFingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "const cachedData = localStorage.getItem" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
data["site"]["pageTitle"] = "BBC · 智能体公共新闻"
titled_html = renderer.render_document(data, template, runtime)
assert "<title>BBC · 智能体公共新闻</title>" in titled_html
assert "<title>BBC 新闻 — 看见正在发生的世界</title>" not in titled_html
data["lead"]["title"] = "</script><script>alert('x')</script>"
escaped_html = renderer.render_document(data, template, runtime)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "bbc-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(
renderer.DataValidationError,
match="Refusing to overwrite",
):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_and_remote_visual() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(
encoding="utf-8"
)
)
data.pop("footer")
data["focus"]["items"][1]["id"] = data["focus"]["items"][0]["id"]
data["focus"]["items"][0]["visual"] = "remote-image"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: footer" in message
assert "must be unique" in message
assert "visual must be one of" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/news.jpg", "alt": "新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/news.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "新闻原文", "embed": True},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["lead"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "bbc-runtime.js").read_text(encoding="utf-8")
assert 'media.type === "webpage"' in runtime
assert 'sandbox="allow-forms allow-popups allow-scripts"' in runtime
@@ -14,6 +14,7 @@ from deerflow.persistence.business_mapping.sql import DEFAULT_BUSINESS_ROWS
class _MappingStore:
def __init__(self) -> None:
self.calls: list[tuple[str, dict[str, object]]] = []
self.create_calls: list[dict[str, str]] = []
async def update_mapping(self, business_code: str, **fields: object) -> dict[str, object]:
self.calls.append((business_code, fields))
@@ -27,6 +28,27 @@ class _MappingStore:
"sort_order": 3,
}
async def create_custom_mapping(
self,
*,
business_code: str,
mapping_type: str,
) -> dict[str, object] | None:
self.create_calls.append(
{"business_code": business_code, "mapping_type": mapping_type}
)
return {
"business_code": business_code,
"label": business_code,
"mapping_type": mapping_type,
"is_custom": True,
"agent_id": None,
"agent_name": None,
"chain_id": None,
"chain_title": None,
"sort_order": 99,
}
class _ChainStore:
def __init__(self, chain: dict[str, object] | None) -> None:
@@ -68,11 +90,70 @@ def _request(
def test_seed_includes_independent_3q_slots() -> None:
codes = [row["business_code"] for row in DEFAULT_BUSINESS_ROWS]
assert codes[:5] == ["3Q", "6BF", "7BF", "8BF", "AFX"]
assert codes[5:12] == ["3q-0", "3q-1", "3q-2", "3q-3", "3q-4", "3q-5", "3q-6"]
assert codes[:6] == ["3Q", "6BF", "7BF", "8BF", "AFX", "b5-1"]
assert codes[6:13] == ["3q-0", "3q-1", "3q-2", "3q-3", "3q-4", "3q-5", "3q-6"]
assert DEFAULT_BUSINESS_ROWS[codes.index("b5-1")]["label"] == "任务分析"
assert "6-1" in codes and "7-4" in codes
assert codes.index("3q-0") < codes.index("3q-1")
assert codes.index("3q-6") < codes.index("6-1")
custom_codes = {row["business_code"] for row in DEFAULT_BUSINESS_ROWS if row.get("is_custom")}
assert custom_codes == {"3q-1", "3q-2", "3q-3", "3q-4", "3q-5", "3q-6"}
for code in custom_codes:
seed = DEFAULT_BUSINESS_ROWS[codes.index(code)]
assert seed["mapping_type"] == "自定义类型"
assert DEFAULT_BUSINESS_ROWS[codes.index("3q-0")].get("is_custom") is not True
@pytest.mark.asyncio
async def test_admin_can_create_a_custom_mapping() -> None:
request = _request("admin")
response = await business_mapping_router.create_mapping(
request,
business_mapping_router.BusinessMappingCreateRequest(
name="区域研判",
),
)
assert response.business_code == "区域研判"
assert response.label == "区域研判"
assert response.mapping_type == "自定义类型"
assert response.is_custom is True
assert request.app.state.business_mapping_store.create_calls == [
{"business_code": "区域研判", "mapping_type": "自定义类型"}
]
@pytest.mark.asyncio
async def test_non_admin_cannot_create_a_custom_mapping() -> None:
request = _request("user")
with pytest.raises(HTTPException) as error:
await business_mapping_router.create_mapping(
request,
business_mapping_router.BusinessMappingCreateRequest(
name="区域研判",
),
)
assert error.value.status_code == 403
assert request.app.state.business_mapping_store.create_calls == []
@pytest.mark.asyncio
async def test_custom_mapping_name_cannot_consume_the_a_mode_suffix() -> None:
request = _request("admin")
with pytest.raises(HTTPException) as error:
await business_mapping_router.create_mapping(
request,
business_mapping_router.BusinessMappingCreateRequest(
name="区域研判-A",
),
)
assert error.value.status_code == 422
assert request.app.state.business_mapping_store.create_calls == []
@pytest.mark.asyncio
@@ -0,0 +1,119 @@
from __future__ import annotations
from time import perf_counter
from types import SimpleNamespace
import pytest
from starlette.requests import Request
from app.gateway.routers import agents as agents_router
from app.gateway.routers import skills as skills_router
from deerflow.persistence.agents import MemoryAgentStore
from deerflow.persistence.skills import MemorySkillStore
from deerflow.persistence.tags import MemoryTagStore
from deerflow.skills.types import SkillCategory
def _request(path: str) -> Request:
request = Request(
{
"type": "http",
"method": "GET",
"path": path,
"headers": [],
"query_string": b"",
"server": ("test", 80),
"client": ("127.0.0.1", 12345),
"scheme": "http",
"root_path": "",
}
)
request.scope["app"] = SimpleNamespace(
state=SimpleNamespace(
config=SimpleNamespace(
concurrency=SimpleNamespace(
redis=SimpleNamespace(enabled=False, url=None, password=None)
)
)
)
)
request.state.user = SimpleNamespace(id="benchmark-user", system_role="user")
return request
@pytest.mark.asyncio
async def test_catalog_endpoint_assembly_with_300_agents_and_300_skills(monkeypatch) -> None:
agent_store = MemoryAgentStore()
for index in range(300):
await agent_store.ensure_builtin(
{
"id": f"benchmark-agent-{index:03d}",
"name": f"Benchmark Agent {index:03d}",
"description": "synthetic homepage benchmark record",
"published": True,
}
)
agent_tags = MemoryTagStore()
monkeypatch.setattr(agents_router, "_require_agents_api_enabled", lambda: None)
monkeypatch.setattr(agents_router, "get_agent_store", lambda _request: agent_store)
monkeypatch.setattr(agents_router, "get_tag_store", lambda _request: agent_tags)
monkeypatch.setattr(agents_router, "get_skill_store", lambda _request: None)
started = perf_counter()
agents_response = await agents_router.list_agents(
_request("/api/agents"),
search=None,
tag_ids=None,
scope=None,
page=None,
page_size=20,
sort="time_desc",
owner=None,
square=None,
admin=False,
hidden=None,
)
agents_ms = (perf_counter() - started) * 1000
assert len(agents_response.agents) == 300
skill_store = MemorySkillStore()
catalog = []
for index in range(300):
name = f"benchmark-skill-{index:03d}"
await skill_store.ensure_legacy({"name": name, "owner_user_id": None, "published": True})
catalog.append(
{
"name": name,
"description": "synthetic homepage benchmark record",
"license": None,
"category": SkillCategory.PUBLIC.value,
"enabled": True,
"created_at": None,
"last_activity_at": None,
}
)
monkeypatch.setattr(skills_router, "load_or_warm_skill_catalog_snapshot", lambda _config: _resolved(catalog))
monkeypatch.setattr(skills_router, "_list_skill_display_adapters", lambda _names: _resolved({}))
started = perf_counter()
skills_response = await skills_router.list_skills(
_request("/api/skills"),
config=SimpleNamespace(),
skill_store=skill_store,
tag_store=MemoryTagStore(),
search=None,
tag_ids=None,
square=None,
scope=None,
sort="time_desc",
admin=False,
)
skills_ms = (perf_counter() - started) * 1000
assert len(skills_response.skills) == 300
assert agents_ms < 3_000
assert skills_ms < 3_000
print(f"synthetic 300+300 catalogue assembly: agents={agents_ms:.1f}ms skills={skills_ms:.1f}ms")
async def _resolved(value):
return value
@@ -0,0 +1,194 @@
"""Configuration-cache hits, isolation, revocation and cross-catalog invalidation."""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from fastapi import HTTPException
from test_api_read_cache import FakeRedis, _config
from app.gateway import api_read_cache
from app.gateway.catalog_cache import cached_catalog
from app.gateway.routers import compact_menu_overrides, html_templates, light_apps, menu_overrides, models, tags, task_buttons
from deerflow.persistence.compact_menu_overrides import MemoryCompactMenuOverrideStore
from deerflow.persistence.light_apps import MemoryLightAppStore
from deerflow.persistence.menu_overrides import MemoryMenuOverrideStore
from deerflow.persistence.tags import MemoryTagStore
from deerflow.persistence.task_buttons import MemoryTaskButtonStore
@pytest.fixture
def cached_request(monkeypatch):
redis = FakeRedis()
monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None)
monkeypatch.setattr(api_read_cache, "_api_read_cache_settings_cache", None)
monkeypatch.setattr(api_read_cache, "_client", lambda *_args, **_kwargs: (redis, "test:catalog"))
request = SimpleNamespace(
app=SimpleNamespace(state=SimpleNamespace(config=_config())),
state=SimpleNamespace(user=SimpleNamespace(id="user-1", system_role="user")),
)
async def current_user(req):
return req.state.user
for router in (menu_overrides, compact_menu_overrides, light_apps, task_buttons, html_templates):
monkeypatch.setattr(router, "get_optional_user_from_request", current_user)
for router in (light_apps, task_buttons):
monkeypatch.setattr(router, "get_current_user", AsyncMock(return_value="admin"))
return request, redis
@pytest.mark.asyncio
async def test_personal_menu_cache_isolates_users_roles_and_checks_auth(cached_request):
request, _ = cached_request
store = request.app.state.menu_override_store = MemoryMenuOverrideStore()
await store.replace_all([{"node_id": "n", "label": "default"}])
await store.replace_user_overrides("user-1", [{"node_id": "n", "label": "private"}])
store.list_overrides = AsyncMock(wraps=store.list_overrides)
assert (await menu_overrides.list_menu_overrides(request)).overrides[0].label == "private"
await menu_overrides.list_menu_overrides(request)
assert store.list_overrides.await_count == 1
request.state.user = SimpleNamespace(id="user-2", system_role="user")
assert (await menu_overrides.list_menu_overrides(request)).overrides[0].label == "default"
request.state.user = SimpleNamespace(id="user-1", system_role="admin")
assert (await menu_overrides.list_menu_overrides(request)).editable_scope == "default"
request.state.user = None
with pytest.raises(HTTPException) as exc:
await menu_overrides.list_menu_overrides(request)
assert exc.value.status_code == 401
@pytest.mark.asyncio
async def test_default_menu_edit_invalidates_all_personal_views(cached_request):
request, _ = cached_request
store = request.app.state.menu_override_store = MemoryMenuOverrideStore()
await store.replace_all([{"node_id": "n", "label": "old"}])
assert (await menu_overrides.list_menu_overrides(request)).overrides[0].label == "old"
request.state.user = SimpleNamespace(id="admin", system_role="admin")
await menu_overrides.replace_menu_overrides(request, menu_overrides.MenuOverrideReplaceRequest(
overrides=[menu_overrides.MenuOverride(node_id="n", label="new")],
hierarchy=[menu_overrides.MenuHierarchyNode(node_id="n")],
))
request.state.user = SimpleNamespace(id="user-1", system_role="user")
assert (await menu_overrides.list_menu_overrides(request)).overrides[0].label == "new"
@pytest.mark.asyncio
async def test_compact_menu_cache_isolates_variant_and_reset(cached_request):
request, _ = cached_request
store = request.app.state.compact_menu_override_store = MemoryCompactMenuOverrideStore()
await store.replace_all([{"node_id": "n", "label": "A"}], menu_type="A")
await store.replace_all([{"node_id": "n", "label": "B"}], menu_type="B")
assert (await compact_menu_overrides.list_compact_menu_overrides(request, "A")).overrides[0].label == "A"
assert (await compact_menu_overrides.list_compact_menu_overrides(request, "B")).overrides[0].label == "B"
await compact_menu_overrides.replace_compact_menu_overrides(request, compact_menu_overrides.CompactMenuOverrideReplaceRequest(
menu_type="B", overrides=[compact_menu_overrides.CompactMenuOverride(node_id="n", label="personal")],
))
assert (await compact_menu_overrides.list_compact_menu_overrides(request, "B")).overrides[0].label == "personal"
await compact_menu_overrides.reset_compact_menu_overrides(request, "B")
assert (await compact_menu_overrides.list_compact_menu_overrides(request, "B")).overrides[0].label == "B"
@pytest.mark.asyncio
async def test_light_app_filters_detail_and_tags_are_invalidated(cached_request):
request, _ = cached_request
request.state.user.system_role = "admin"
request.app.state.light_app_store = MemoryLightAppStore()
tag_store = request.app.state.tag_store = MemoryTagStore()
app = await light_apps.create_light_app(request, light_apps.LightAppCreateRequest(
name="One", app_type="iframe", url="https://example.test", mount_location="root",
))
query = dict(app_type=None, mount_location=None, enabled_only=False, search=None, tag_ids=None)
assert len((await light_apps.list_light_apps(request, **query)).apps) == 1
assert not (await light_apps.list_light_apps(request, **{**query, "search": "missing"})).apps
assert (await light_apps.get_light_app(request, app.app_id)).name == "One"
tag = await tags.create_tag(tags.TagCreateRequest(name="Topic", tag_type="light_app"), request, tag_store)
await tags.set_resource_tags("light_app", app.app_id, tags.TagAssignmentRequest(tag_ids=[tag.id]), request, tag_store)
assert (await light_apps.get_light_app(request, app.app_id)).tags[0]["name"] == "Topic"
await tags.update_tag(tag.id, tags.TagUpdateRequest(name="Renamed"), request, tag_store)
assert (await light_apps.list_light_apps(request, **query)).apps[0].tags[0]["name"] == "Renamed"
await light_apps.delete_light_app(request, app.app_id)
assert not (await light_apps.list_light_apps(request, **query)).apps
with pytest.raises(HTTPException) as exc:
await light_apps.get_light_app(request, app.app_id)
assert exc.value.status_code == 404
@pytest.mark.asyncio
async def test_writes_while_disabled_do_not_resurrect_old_buttons(cached_request):
request, _ = cached_request
request.state.user.system_role = "admin"
store = request.app.state.task_button_store = MemoryTaskButtonStore()
await store.replace_all([{"id": "b", "business": "rwfx", "label": "old"}])
assert (await task_buttons.list_task_buttons(request)).buttons[0].label == "old"
api_read_cache.set_api_read_cache_settings_local(False, 300)
await task_buttons.replace_task_buttons(request, task_buttons.TaskButtonReplaceRequest(
buttons=[task_buttons.TaskButton(id="b", business="rwfx", label="new")],
))
api_read_cache.set_api_read_cache_settings_local(True, 300)
assert (await task_buttons.list_task_buttons(request)).buttons[0].label == "new"
@pytest.mark.asyncio
async def test_html_edit_invalidates_metadata_content_and_list(cached_request, monkeypatch):
request, _ = cached_request
request.state.user.system_role = "admin"
state = {"html": "old"}
monkeypatch.setattr(html_templates.template_store, "read_template_html", lambda _id: state["html"])
monkeypatch.setattr(html_templates.template_store, "list_templates", lambda: [{"name": state["html"]}])
monkeypatch.setattr(html_templates.template_store, "get_template_meta", lambda _id: {"name": state["html"]})
def update(_id, **kwargs):
state["html"] = kwargs["html"]
return {"name": state["html"]}
monkeypatch.setattr(html_templates.template_store, "update_template", update)
assert (await html_templates.get_html_template_content("one", request))["html"] == "old"
await html_templates.list_html_templates(request)
await html_templates.get_html_template("one", request)
await html_templates.update_html_template("one", html_templates.TemplateUpdateRequest(html="new"), request)
assert (await html_templates.get_html_template_content("one", request))["html"] == "new"
assert (await html_templates.get_html_template("one", request))["name"] == "new"
assert (await html_templates.list_html_templates(request))["templates"][0]["name"] == "new"
@pytest.mark.asyncio
async def test_models_cache_capability_reads_and_do_not_cache_failures(cached_request):
request, _ = cached_request
store = request.app.state.model_capability_store = SimpleNamespace(list_capabilities=AsyncMock(
side_effect=[RuntimeError("unavailable"), [{"model_name": "one", "supports_reasoning_effort": True}]],
))
assert await models._reasoning_capabilities(request) == {}
assert (await models._reasoning_capabilities(request))["one"]["supports_reasoning_effort"] is True
await models._reasoning_capabilities(request)
assert store.list_capabilities.await_count == 2
@pytest.mark.asyncio
async def test_redis_outage_falls_back_to_config_store(cached_request, monkeypatch):
request, redis = cached_request
request.app.state.task_button_store = MemoryTaskButtonStore()
monkeypatch.setattr(redis, "get", AsyncMock(side_effect=ConnectionError("offline")))
assert (await task_buttons.list_task_buttons(request)).buttons == []
@pytest.mark.asyncio
async def test_catalogue_burst_shares_one_database_fill(cached_request):
request, _ = cached_request
calls = 0
async def load():
nonlocal calls
calls += 1
await asyncio.sleep(0.03)
return {"buttons": ["shared"]}
results = await asyncio.gather(*(
cached_catalog(request, "task_buttons", "all", load) for _ in range(30)
))
assert calls == 1
assert all(result == {"buttons": ["shared"]} for result in results)
@@ -0,0 +1,252 @@
"""A bad thread must not prevent healthy conversations from being archived."""
from __future__ import annotations
import errno
import sqlite3
from contextlib import asynccontextmanager
from datetime import datetime
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from langgraph.checkpoint.base import empty_checkpoint
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers import chat_archive as archive_api
from deerflow.persistence.base import Base
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.checkpointer.chat_archive import (
archive_db_path,
ensure_catalog,
try_acquire_archive_job_lock,
)
MONTH = "2025-02"
THREADS = ("a-first", "b-middle", "c-last")
async def checkpoint_ids(saver, thread_id):
return [
item.config["configurable"]["checkpoint_id"]
async for item in saver.alist({"configurable": {"thread_id": thread_id}})
]
async def add_checkpoint(saver, thread_id):
checkpoint = empty_checkpoint()
await saver.aput(
{"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}},
checkpoint, {"source": "archive-failure-test"}, checkpoint["channel_versions"],
)
return checkpoint["id"]
@asynccontextmanager
async def archive_fixture(tmp_path):
root = tmp_path / "archives"
ensure_catalog(root)
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'metadata.db'}")
try:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all, tables=[ThreadMetaRow.__table__])
sessions = async_sessionmaker(engine, expire_on_commit=False)
created_at = datetime(2025, 2, 12, 12)
async with sessions() as session:
for thread_id in THREADS:
session.add(ThreadMetaRow(
thread_id=thread_id, assistant_id="assistant", user_id="owner",
display_name=thread_id, status="idle", metadata_json={},
created_at=created_at, updated_at=created_at,
))
await session.commit()
async with AsyncSqliteSaver.from_conn_string(str(tmp_path / "primary.db")) as primary:
await primary.setup()
for thread_id in THREADS:
await add_checkpoint(primary, thread_id)
yield SimpleNamespace(
root=root, primary=primary,
runner=archive_api.ChatArchiveRunner(root, primary, sessions),
)
finally:
await engine.dispose()
async def run_job(fixture, action):
job = await fixture.runner.start(MONTH, action)
await fixture.runner._tasks[job["job_id"]]
return archive_api._job_dict(fixture.root, job["job_id"])
def assert_partial(job, failed_thread, total=3):
assert job["status"] == "partial"
assert job["processed_threads"] == total
assert job["failed_threads"] == 1
assert failed_thread in job["error"]
assert job["current_thread_id"] is None
@pytest.mark.asyncio
@pytest.mark.parametrize("failed_thread", THREADS[:2])
async def test_copy_failure_continues_and_never_exposes_unverified_thread_to_purge(tmp_path, monkeypatch, failed_thread):
real_copy = archive_api.copy_thread_state
async def fail_one(source, target, thread_id):
if thread_id == failed_thread:
raise ValueError("unsupported checkpoint payload")
return await real_copy(source, target, thread_id)
monkeypatch.setattr(archive_api, "copy_thread_state", fail_one)
monkeypatch.setattr(archive_api, "_PURGE_THREAD_DELAY_SECONDS", 0)
async with archive_fixture(tmp_path) as fixture:
result = await run_job(fixture, "archive")
assert_partial(result, failed_thread)
assert "unsupported checkpoint payload" in result["error"]
with sqlite3.connect(fixture.root / "catalog.db") as conn:
rows = conn.execute("SELECT thread_id FROM archive_threads").fetchall()
assert {row[0] for row in rows} == set(THREADS) - {failed_thread}
async with AsyncSqliteSaver.from_conn_string(str(archive_db_path(fixture.root, MONTH))) as archived:
assert await checkpoint_ids(archived, "c-last")
purge = await run_job(fixture, "purge")
assert purge["status"] == "completed"
assert purge["processed_threads"] == 2
for thread_id in THREADS:
assert bool(await checkpoint_ids(fixture.primary, thread_id)) is (thread_id == failed_thread)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_mode", ["copy_error", "verification_error"])
async def test_purge_keeps_unverified_pg_data_and_continues_other_threads(tmp_path, monkeypatch, failure_mode):
monkeypatch.setattr(archive_api, "_PURGE_THREAD_DELAY_SECONDS", 0)
async with archive_fixture(tmp_path) as fixture:
assert (await run_job(fixture, "archive"))["status"] == "completed"
await add_checkpoint(fixture.primary, "a-first")
real_copy = archive_api.copy_thread_state
async def fail_one(source, target, thread_id):
if thread_id == "a-first":
if failure_mode == "copy_error":
raise ValueError("cannot copy new checkpoint")
return False # Deliberately leave the new PG checkpoint absent from SQLite.
return await real_copy(source, target, thread_id)
monkeypatch.setattr(archive_api, "copy_thread_state", fail_one)
purge = await run_job(fixture, "purge")
assert_partial(purge, "a-first")
assert len(await checkpoint_ids(fixture.primary, "a-first")) == 2
assert await checkpoint_ids(fixture.primary, "b-middle") == []
assert await checkpoint_ids(fixture.primary, "c-last") == []
@pytest.mark.asyncio
async def test_rollback_continues_after_one_thread_fails(tmp_path, monkeypatch):
monkeypatch.setattr(archive_api, "_PURGE_THREAD_DELAY_SECONDS", 0)
async with archive_fixture(tmp_path) as fixture:
await run_job(fixture, "archive")
await run_job(fixture, "purge")
real_copy = archive_api.copy_thread_state
async def fail_one(source, target, thread_id):
if thread_id == "a-first":
raise ValueError("cannot restore this checkpoint")
return await real_copy(source, target, thread_id)
monkeypatch.setattr(archive_api, "copy_thread_state", fail_one)
result = await run_job(fixture, "rollback")
assert_partial(result, "a-first")
assert await checkpoint_ids(fixture.primary, "a-first") == []
assert await checkpoint_ids(fixture.primary, "b-middle")
assert await checkpoint_ids(fixture.primary, "c-last")
async with AsyncSqliteSaver.from_conn_string(str(archive_db_path(fixture.root, MONTH))) as archived:
assert await checkpoint_ids(archived, "a-first")
@pytest.mark.asyncio
async def test_disk_exhaustion_stops_instead_of_attempting_more_threads(tmp_path, monkeypatch):
visited = []
async def disk_full(source, target, thread_id):
visited.append(thread_id)
raise OSError(errno.ENOSPC, "disk full")
monkeypatch.setattr(archive_api, "copy_thread_state", disk_full)
async with archive_fixture(tmp_path) as fixture:
job = await run_job(fixture, "archive")
assert job["status"] == "failed"
assert visited == ["a-first"]
for thread_id in THREADS:
assert await checkpoint_ids(fixture.primary, thread_id)
def persisted_job(job_id, *, status="running"):
return {
"job_id": job_id, "month": MONTH, "action": "archive", "status": status,
"total_threads": 3, "processed_threads": 0, "failed_threads": 0,
"current_thread_id": None, "message": "", "error": None,
"created_at": "2026-10-25T00:00:00+08:00", "updated_at": "2026-10-25T00:00:00+08:00",
}
def test_recovery_leaves_live_older_process_alone_then_recovers_after_lock_release(tmp_path):
root = tmp_path / "recovery"
archive_api._save_job(root, persisted_job("older-live-process"))
archive_api._upsert_month(root, MONTH, status="copying", total=3, job_id="older-live-process")
operation_lock = try_acquire_archive_job_lock(root)
assert operation_lock is not None
try:
archive_api._recover_jobs(root, started_before="2026-10-26T00:00:00+08:00")
assert archive_api._job_dict(root, "older-live-process")["status"] == "running"
finally:
operation_lock.close()
archive_api._recover_jobs(root, started_before="2026-10-26T00:00:00+08:00")
recovered = archive_api._job_dict(root, "older-live-process")
assert recovered["status"] == "interrupted"
assert recovered["error"]
assert archive_api._catalog_months(root)[MONTH]["status"] == "interrupted"
@pytest.mark.asyncio
async def test_another_process_operation_lock_prevents_start(tmp_path):
async with archive_fixture(tmp_path) as fixture:
lock = try_acquire_archive_job_lock(fixture.root)
assert lock is not None
try:
with pytest.raises(HTTPException) as error:
await fixture.runner.start(MONTH, "archive")
assert error.value.status_code == 409
finally:
lock.close()
def test_latest_job_uses_insertion_order_when_timestamps_tie(tmp_path):
root = tmp_path / "tie"
archive_api._save_job(root, persisted_job("z-earlier", status="completed"))
archive_api._save_job(root, persisted_job("a-newer", status="partial"))
assert archive_api._catalog_months(root)[MONTH]["latest_job"]["job_id"] == "a-newer"
@pytest.mark.asyncio
async def test_inventory_rechecks_owner_after_previously_live_process_exits(tmp_path):
root = tmp_path / "later-exit"
archive_api._save_job(root, persisted_job("later-exit"))
request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace()))
lock = try_acquire_archive_job_lock(root)
assert lock is not None
try:
await archive_api._ensure_archive_recovery_async(request, root)
assert archive_api._job_dict(root, "later-exit")["status"] == "running"
finally:
lock.close()
await archive_api._ensure_archive_recovery_async(request, root)
assert archive_api._job_dict(root, "later-exit")["status"] == "interrupted"
@pytest.mark.asyncio
async def test_new_action_recovers_unowned_stale_job_without_gateway_restart(tmp_path):
async with archive_fixture(tmp_path) as fixture:
archive_api._save_job(fixture.root, persisted_job("orphaned"))
result = await run_job(fixture, "archive")
assert result["status"] == "completed"
assert archive_api._job_dict(fixture.root, "orphaned")["status"] == "interrupted"
@@ -0,0 +1,122 @@
"""Opt-in end-to-end archive test for a disposable PostgreSQL database.
Run with DEERFLOW_TEST_POSTGRES_URL set to an isolated database. This test
archives, purges, transparently reads, and restores one uniquely named thread.
"""
from __future__ import annotations
import os
import uuid
from datetime import datetime
import pytest
from langgraph.checkpoint.base import empty_checkpoint
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers.chat_archive import ChatArchiveRunner, _save_job, _upsert_month
from deerflow.persistence.base import Base
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.checkpointer.chat_archive import archive_db_path, ensure_catalog
from deerflow.runtime.checkpointer.checkpoint_migration import DualReadCheckpointer
async def _checkpoint_ids(saver, thread_id: str) -> list[str]:
return [
item.config["configurable"]["checkpoint_id"]
async for item in saver.alist({"configurable": {"thread_id": thread_id}})
]
@pytest.mark.asyncio
async def test_postgres_month_archive_purge_fallback_and_rollback(tmp_path):
conninfo = os.getenv("DEERFLOW_TEST_POSTGRES_URL")
if not conninfo:
pytest.skip("set DEERFLOW_TEST_POSTGRES_URL to an isolated PostgreSQL database")
thread_id = f"archive-qa-{uuid.uuid4().hex}"
month = "2025-02"
pool = AsyncConnectionPool(
conninfo=conninfo,
min_size=1,
max_size=3,
kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row},
open=False,
)
business_engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'business.db'}")
try:
await pool.open(wait=True)
primary = AsyncPostgresSaver(pool)
await primary.setup()
async with business_engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all, tables=[ThreadMetaRow.__table__])
session_factory = async_sessionmaker(business_engine, expire_on_commit=False)
created_at = datetime(2025, 2, 12, 12, 0)
async with session_factory() as session:
session.add(ThreadMetaRow(
thread_id=thread_id,
assistant_id="archive-qa",
user_id="archive-qa",
display_name="isolated PostgreSQL archive test",
status="idle",
metadata_json={},
created_at=created_at,
updated_at=created_at,
))
await session.commit()
checkpoint = empty_checkpoint()
await primary.aput(
{"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}},
checkpoint,
{"source": "isolated-postgres-qa"},
checkpoint["channel_versions"],
)
assert await _checkpoint_ids(primary, thread_id) == [checkpoint["id"]]
root = tmp_path / "chat-archives"
ensure_catalog(root)
job_id = f"archive-qa-{uuid.uuid4().hex}"
now = "2026-10-25T00:00:00+08:00"
_save_job(root, {
"job_id": job_id,
"month": month,
"action": "archive",
"status": "running",
"total_threads": 1,
"processed_threads": 0,
"failed_threads": 0,
"current_thread_id": None,
"message": "",
"error": None,
"created_at": now,
"updated_at": now,
})
_upsert_month(root, month, status="copying", total=1, job_id=job_id)
runner = ChatArchiveRunner(root, primary, session_factory)
await runner._archive_month(job_id, month)
archived_path = archive_db_path(root, month)
async with AsyncSqliteSaver.from_conn_string(str(archived_path)) as archived:
await archived.setup()
assert await _checkpoint_ids(archived, thread_id) == [checkpoint["id"]]
await runner._purge_month(job_id, month)
assert await _checkpoint_ids(primary, thread_id) == []
proxy = DualReadCheckpointer(primary, [], archive_dir=root)
recovered = await proxy.aget_tuple({"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}})
assert recovered is not None
assert recovered.checkpoint["id"] == checkpoint["id"]
await proxy.aclose_sources()
await runner._rollback_month(job_id, month)
assert await _checkpoint_ids(primary, thread_id) == [checkpoint["id"]]
finally:
await business_engine.dispose()
await pool.close()
@@ -0,0 +1,136 @@
"""Step-budget recovery must preserve history and allow a real next turn."""
from unittest.mock import AsyncMock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import END, START, MessagesState, StateGraph
from pydantic import ValidationError
from deerflow.config.system_settings import ConversationContextSettings
from deerflow.runtime.runs.manager import RunManager
from deerflow.runtime.runs.schemas import RunStatus
from deerflow.runtime.runs.worker import RunContext, run_agent
@pytest.mark.parametrize("value", [19, 10001, 25.5, True, "100"])
def test_invalid_budget(value):
from app.gateway.routers.system_settings import ConversationContextUpdateRequest
with pytest.raises(ValidationError):
ConversationContextSettings(max_run_steps=value)
with pytest.raises(ValidationError):
ConversationContextUpdateRequest(max_run_steps=value)
@pytest.mark.asyncio
@pytest.mark.parametrize("budget", [3, 4])
async def test_limit_is_saved_without_error_and_continue_works(budget):
calls = []
def model(state):
messages = state["messages"]
if any(isinstance(m, HumanMessage) and m.content == "继续" for m in messages):
# A provider requires every earlier call to have a result.
tool_ids = {m.tool_call_id for m in messages if isinstance(m, ToolMessage)}
assert all(c["id"] in tool_ids for m in messages if isinstance(m, AIMessage) for c in m.tool_calls)
return {"messages": [AIMessage(content="已继续完成")]}
return {
"messages": [
AIMessage(
content="正在处理",
tool_calls=[
{
"id": f"call-{len(messages)}",
"name": "search",
"args": {},
}
],
)
]
}
def tool(state):
call = state["messages"][-1].tool_calls[0]
calls.append(call["id"])
return {"messages": [ToolMessage(content="已有检索结果", tool_call_id=call["id"])]}
graph = StateGraph(MessagesState)
graph.add_node("model", model)
graph.add_node("tools", tool)
graph.add_edge(START, "model")
graph.add_conditional_edges("model", lambda s: "tools" if s["messages"][-1].tool_calls else END)
graph.add_edge("tools", "model")
saver = InMemorySaver()
agent = graph.compile(checkpointer=saver)
config = {"configurable": {"thread_id": "step-test"}, "recursion_limit": budget}
manager = RunManager()
record = await manager.create("step-test", "lead_agent")
bridge = AsyncMock()
await run_agent(bridge, manager, record, ctx=RunContext(checkpointer=saver), agent_factory=lambda config: agent, graph_input={"messages": [HumanMessage(content="研究")]}, config=config, stream_modes=["messages", "values"])
assert record.status == RunStatus.interrupted
assert record.error is None
assert not any(c.args[1] == "error" for c in bridge.publish.call_args_list)
events = {c.args[1]: c.args[2] for c in bridge.publish.call_args_list}
assert "步骤上限" in events["messages"][0]["content"]
assert "步骤上限" in events["values"]["messages"][-1]["content"]
bridge.publish_end.assert_awaited_once()
state = await agent.aget_state(config)
assert not state.next
assert "已有检索结果" in [m.content for m in state.values["messages"]]
assert "步骤上限" in state.values["messages"][-1].content
count = len(calls)
result = await agent.ainvoke({"messages": [HumanMessage(content="继续")]}, config)
assert result["messages"][-1].content == "已继续完成"
assert len(calls) == count
def test_server_budget_overrides_client_and_preserves_other_graphs(monkeypatch):
from app.gateway.services import build_run_config
monkeypatch.setattr("deerflow.config.system_settings.get_conversation_context_settings", lambda: ConversationContextSettings(max_run_steps=800))
for assistant in (None, "lead_agent", "custom-agent"):
assert build_run_config("t", {"recursion_limit": 99999}, None, assistant_id=assistant)["recursion_limit"] == 800
assert build_run_config("t", {"recursion_limit": 250}, None, assistant_id="canvas_agent")["recursion_limit"] == 250
@pytest.mark.parametrize("client_limit", [None, 1, 300])
@pytest.mark.parametrize("server_limit", [20, 800])
def test_open_chat_uses_server_budget(monkeypatch, client_limit, server_limit):
from app.gateway.routers.open_chat import OpenChatRequest, _build_run_context
monkeypatch.setattr("deerflow.config.system_settings.get_conversation_context_settings", lambda: ConversationContextSettings(max_run_steps=server_limit))
body = OpenChatRequest(message="hello", **({"recursion_limit": client_limit} if client_limit is not None else {}))
config, _, _ = _build_run_context(body)
assert config["recursion_limit"] == server_limit
assert config["configurable"]["ephemeral"] is True
@pytest.mark.asyncio
async def test_setting_requires_admin_and_roundtrips(monkeypatch):
from fastapi import HTTPException
from app.gateway.routers import system_settings as routes
from deerflow.config.system_settings import SystemSettings
current = SystemSettings()
monkeypatch.setattr(routes, "load_system_settings", lambda: current)
def save(value):
nonlocal current
current = value
return value
monkeypatch.setattr(routes, "save_system_settings", save)
guard = AsyncMock(side_effect=HTTPException(status_code=403))
monkeypatch.setattr(routes, "_require_admin", guard)
body = routes.ConversationContextUpdateRequest(max_run_steps=600)
with pytest.raises(HTTPException):
await routes.update_conversation_context_setting(body, None)
assert current.conversation_context.max_run_steps == 1000
guard.side_effect = None
result = await routes.update_conversation_context_setting(body, None)
assert result.max_run_steps == 600
assert (await routes.get_conversation_context_setting()).max_run_steps == 600
assert current.conversation_context.recent_turn_window_enabled is False
@@ -0,0 +1,240 @@
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "cnn-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_cnn_page.py"
FAST_SCRIPT_PATH = SKILL_ROOT / "scripts" / "fast_page.py"
LEAD_PROMPT_PATH = BACKEND_ROOT / "packages" / "harness" / "deerflow" / "agents" / "lead_agent" / "prompt.py"
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"cnn_page_renderer",
SCRIPT_PATH,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "cnn-page-generator"
assert "CNN" in skill.description
discovered = LocalSkillStorage(host_path=str(SKILL_ROOT.parent.parent)).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert discovered_by_name["cnn-page-generator"].category == SkillCategory.PUBLIC
assert discovered_by_name["cnn-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/cnn-page-generator/SKILL.md" in lead_prompt_source
assert "专用 CNN 路由" in lead_prompt_source
def test_default_json_renders_editable_cnn_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "cnn-template.html").read_text(encoding="utf-8")
runtime = (SKILL_ROOT / "assets" / "cnn-runtime.js").read_text(encoding="utf-8")
html = renderer.render_document(data, template, runtime)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["latest"] >= 1
assert renderer.DATA_MARKER not in html
assert renderer.RUNTIME_MARKER not in html
assert 'id="cnnPageData"' in html
assert 'id="leadGrid"' in html
assert 'id="videoGrid"' in html
assert 'id="liveModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert '"brand": "CNN"' in html
assert "<title>CNN — 全球新闻现场</title>" in html
assert '"cnn-page-data-v1:" + pageFingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "const cachedData = localStorage.getItem" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
data["site"]["pageTitle"] = "CNN · 智能体新闻中心"
titled_html = renderer.render_document(data, template, runtime)
assert "<title>CNN · 智能体新闻中心</title>" in titled_html
assert "<title>CNN — 全球新闻现场</title>" not in titled_html
data["lead"]["primary"]["title"] = "</script><script>alert('x')</script>"
escaped_html = renderer.render_document(data, template, runtime)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "cnn-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(
renderer.DataValidationError,
match="Refusing to overwrite",
):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_and_remote_visual() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data.pop("footer")
data["latest"]["items"][1]["id"] = data["latest"]["items"][0]["id"]
data["latest"]["items"][0]["visual"] = "remote-image"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: footer" in message
assert "must be unique" in message
assert "visual must be one of" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
media_values = (
{"type": "image", "mediaUrl": "https://cdn.example.com/news.jpg", "alt": "新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/news.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "新闻原文", "embed": False},
)
for media in media_values:
data = json.loads(json.dumps(source, ensure_ascii=False))
data["lead"]["primary"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "cnn-runtime.js").read_text(encoding="utf-8")
assert 'media.type === "webpage"' in runtime
assert 'rel="noopener noreferrer"' in runtime
def test_high_level_fast_spec_builds_all_modules_in_one_call(tmp_path: Path) -> None:
spec_path = tmp_path / "cnn-page-spec.json"
spec_path.write_text(
json.dumps({"topic": "茂达蛋 IP"}, ensure_ascii=False),
encoding="utf-8",
)
command = [
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
]
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stdout + completed.stderr
result = json.loads(completed.stdout)
assert result["status"] == "ok"
assert result["action"] == "fast-cnn-page"
data_path = Path(result["data"])
html_path = Path(result["html"])
assert data_path.name == "cnn-page-data.json"
assert html_path.name == "cnn-page.html"
data = json.loads(data_path.read_text(encoding="utf-8"))
assert data["site"]["pageTitle"] == "CNN 中文专题 · 茂达蛋 IP"
assert data["site"]["date"] == "中文示意页面 · 非实时新闻"
assert data["lead"]["primary"]["title"].startswith("茂达蛋 IP发布最新计划")
assert data["lead"]["primary"]["visual"] == "summit"
assert data["latest"]["items"][0]["visual"] == "coast"
assert data["video"]["items"][0]["visual"] == "night-city"
assert data["world"]["items"][0]["visual"] == "trade"
assert data["markets"]["title"] == "趋势快照 · 示意数据"
assert data["markets"]["items"][0]["change"] == "非实时"
html = html_path.read_text(encoding="utf-8")
assert "茂达蛋 IP发布最新计划" in html
assert "本页面内容均为中文示意" in html
second = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert second.returncode == 0, second.stdout + second.stderr
second_result = json.loads(second.stdout)
assert Path(second_result["data"]).name == "cnn-page-2-data.json"
assert Path(second_result["html"]).name == "cnn-page-2.html"
def test_high_level_fast_spec_rejects_english_only_headline(tmp_path: Path) -> None:
spec_path = tmp_path / "invalid-spec.json"
spec_path.write_text(
json.dumps(
{"topic": "测试专题", "headline": "This headline must be Chinese."},
ensure_ascii=False,
),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 2
result = json.loads(completed.stdout)
assert result["status"] == "error"
assert "must contain Simplified Chinese" in result["error"]
def test_cnn_fast_workflow_forbids_probe_and_deep_patch_paths() -> None:
skill_text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
prompt_text = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert '{"topic":"茂达蛋 IP"}' in skill_text
assert "不得写十模块 JSON、`$set` 或深层字段路径" in skill_text
assert "禁止探路渲染" in skill_text
assert "禁止探路和二次渲染" in prompt_text
@@ -55,7 +55,13 @@ def test_storage_keys_keep_type_a_unprefixed():
async def test_replace_all_roundtrips(store: CompactMenuOverrideRepository):
out = await store.replace_all(
[
{"node_id": "workspace-chats", "placement": "settings", "sort_order": 1, "label": "会话"},
{
"node_id": "workspace-chats",
"placement": "settings",
"sort_order": 1,
"label": "会话",
"disabled": True,
},
{"node_id": "workspace-agents", "placement": "sidebar", "sort_order": 0, "label": None},
{"node_id": "workspace-skills", "placement": "toolbox", "sort_order": 0, "label": "技能工具"},
],
@@ -66,6 +72,7 @@ async def test_replace_all_roundtrips(store: CompactMenuOverrideRepository):
assert rows["workspace-chats"]["label"] == "会话"
assert rows["workspace-chats"]["placement"] == "settings"
assert rows["workspace-chats"]["menu_type"] == "A"
assert rows["workspace-chats"]["disabled"] is True
assert rows["workspace-agents"]["updated_by"] == "admin-1"
assert rows["workspace-skills"]["placement"] == "toolbox"
@@ -164,7 +171,15 @@ async def test_memory_store_parity():
async def test_personal_compact_snapshots_are_isolated_by_user_and_type(store: CompactMenuOverrideRepository):
await store.replace_user_overrides(
"user-1",
[{"node_id": "chat", "placement": "toolbox", "sort_order": 0, "label": "我的对话"}],
[
{
"node_id": "chat",
"placement": "toolbox",
"sort_order": 0,
"label": "我的对话",
"disabled": True,
}
],
menu_type="A",
updated_by="user-1",
)
@@ -176,6 +191,7 @@ async def test_personal_compact_snapshots_are_isolated_by_user_and_type(store: C
)
assert (await store.list_user_overrides("user-1", "A"))[0]["placement"] == "toolbox"
assert (await store.list_user_overrides("user-1", "A"))[0]["disabled"] is True
assert (await store.list_user_overrides("user-1", "B"))[0]["placement"] == "settings"
assert await store.list_user_overrides("user-2", "A") is None
@@ -190,11 +206,18 @@ def test_personal_compact_nodes_win_while_new_default_nodes_flow_through():
{"node_id": "new", "placement": "toolbox", "sort_order": 1, "label": "新增菜单"},
]
personal_rows = [
{"node_id": "existing", "placement": "sidebar", "sort_order": 3, "label": "我的名称"},
{
"node_id": "existing",
"placement": "sidebar",
"sort_order": 3,
"label": "我的名称",
"disabled": True,
},
]
merged = {row["node_id"]: row for row in _merge_overrides(default_rows, personal_rows)}
assert merged["existing"]["label"] == "我的名称"
assert merged["existing"]["placement"] == "sidebar"
assert merged["existing"]["disabled"] is True
assert merged["new"]["label"] == "新增菜单"
@@ -0,0 +1,306 @@
from __future__ import annotations
import asyncio
import contextvars
import os
import threading
from concurrent.futures import Future, ThreadPoolExecutor
from concurrent.futures.process import BrokenProcessPool
from pathlib import Path
import pytest
from deerflow.utils import file_conversion
from deerflow.utils.file_io import FileIOPool
async def _wait_for_event(event: threading.Event) -> None:
async def wait():
while not event.is_set():
await asyncio.sleep(0.005)
await asyncio.wait_for(wait(), timeout=3)
@pytest.fixture
def conversion_pool(monkeypatch):
pool = FileIOPool(2, 2, "test-document-conversion")
monkeypatch.setattr(file_conversion, "_CONVERSION_IO", pool)
monkeypatch.setattr(file_conversion, "_get_pdf_converter", lambda: "auto")
# These tests coordinate parser entry/exit with in-process threading events.
# Spawn isolation has its own real-worker regression below.
monkeypatch.setattr(
file_conversion,
"_run_conversion_in_process",
lambda path, converter: file_conversion._convert_document_worker(str(path), converter),
)
yield pool
pool._pool.shutdown(wait=True)
@pytest.mark.asyncio
async def test_parser_and_sidecar_write_run_off_event_loop(monkeypatch, tmp_path: Path, conversion_pool):
source = tmp_path / "report.docx"
source.write_bytes(b"source")
main_thread = threading.get_ident()
user_context = contextvars.ContextVar("conversion-test-user", default="missing")
token = user_context.set("alice")
observed = []
write_text = Path.write_text
def parse(_path, _converter):
observed.append(("parse", threading.get_ident(), user_context.get()))
return "# Complete report"
def write(path, text, **kwargs):
observed.append(("write", threading.get_ident(), user_context.get()))
return write_text(path, text, **kwargs)
monkeypatch.setattr(file_conversion, "_do_convert_unbounded", parse)
monkeypatch.setattr(Path, "write_text", write)
try:
result = await file_conversion.convert_file_to_markdown(source)
finally:
user_context.reset(token)
assert result == source.with_name("report.docx.md")
assert result.read_text(encoding="utf-8") == "# Complete report"
assert [operation for operation, _, _ in observed] == ["parse", "write"]
assert all(worker != main_thread and user == "alice" for _, worker, user in observed)
@pytest.mark.asyncio
async def test_saturated_conversion_does_not_use_default_executor(monkeypatch, tmp_path: Path, conversion_pool):
release = threading.Event()
saturated = threading.Event()
lock = threading.Lock()
active = peak = started = 0
def parse(_path, _converter):
nonlocal active, peak, started
with lock:
started += 1
active += 1
peak = max(peak, active)
if active == 2:
saturated.set()
try:
assert release.wait(5), "test did not release conversion workers"
return "converted"
finally:
with lock:
active -= 1
monkeypatch.setattr(file_conversion, "_do_convert_unbounded", parse)
# One general worker makes starvation observable without starting dozens of
# threads. Conversion must not consume it, even when its own pool is full.
default_pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="test-default")
asyncio.get_running_loop().set_default_executor(default_pool)
jobs = [asyncio.create_task(file_conversion.extract_file_to_markdown(tmp_path / f"{index}.pdf")) for index in range(6)]
try:
await _wait_for_event(saturated)
assert await asyncio.wait_for(asyncio.to_thread(lambda: "ordinary request"), timeout=1) == "ordinary request"
await asyncio.sleep(0.02)
assert started == 2
finally:
release.set()
results = await asyncio.gather(*jobs)
default_pool.shutdown(wait=True)
assert results == ["converted"] * 6
assert peak == 2
assert not list(tmp_path.iterdir()), "extraction must not create sidecars"
@pytest.mark.asyncio
@pytest.mark.parametrize("extract_only", [False, True])
async def test_cancelled_conversion_finishes_before_input_cleanup(monkeypatch, tmp_path: Path, conversion_pool, extract_only):
source = tmp_path / "report.pdf"
source.write_bytes(b"input must remain available")
started = threading.Event()
release = threading.Event()
finished = threading.Event()
def parse(path, _converter):
started.set()
assert release.wait(5), "test did not release conversion worker"
assert path.read_bytes() == b"input must remain available"
finished.set()
return "complete output"
monkeypatch.setattr(file_conversion, "_do_convert_unbounded", parse)
convert = file_conversion.extract_file_to_markdown if extract_only else file_conversion.convert_file_to_markdown
job = asyncio.create_task(convert(source))
try:
await _wait_for_event(started)
job.cancel()
await asyncio.sleep(0.01)
assert not job.done()
job.cancel() # A second cancellation must not release input ownership.
await asyncio.sleep(0.01)
assert not job.done()
finally:
release.set()
with pytest.raises(asyncio.CancelledError):
await job
assert finished.is_set()
sidecar = file_conversion.markdown_sidecar_path(source)
if extract_only:
assert not sidecar.exists()
else:
assert sidecar.read_text(encoding="utf-8") == "complete output"
source.unlink() # The caller may now remove the source safely.
@pytest.mark.asyncio
async def test_cancelled_queued_file_work_never_runs():
pool = FileIOPool(1, 1, "test-cancelled-queue")
started = threading.Event()
release = threading.Event()
cancelled = threading.Event()
ran_queued = threading.Event()
def blocked():
started.set()
assert release.wait(5)
first = asyncio.create_task(pool.run_until_complete(blocked))
queued = None
try:
await _wait_for_event(started)
queued = asyncio.create_task(pool.run_until_complete(ran_queued.set, on_cancel=cancelled.set))
await asyncio.sleep(0.01)
queued.cancel()
with pytest.raises(asyncio.CancelledError):
await queued
finally:
release.set()
await first
pool._pool.shutdown(wait=True)
assert not ran_queued.is_set()
assert not cancelled.is_set(), "a cancellation callback is only needed for submitted work"
@pytest.mark.asyncio
async def test_cancel_callback_drains_failed_worker_and_keeps_admission():
pool = FileIOPool(1, 1, "test-cooperative-cancel")
started = threading.Event()
release = threading.Event()
cancelled = threading.Event()
queued_ran = threading.Event()
def blocked():
started.set()
assert release.wait(5)
raise RuntimeError("worker failed while cancellation was pending")
first = asyncio.create_task(pool.run_until_complete(blocked, on_cancel=cancelled.set))
queued = None
try:
await _wait_for_event(started)
first.cancel()
await _wait_for_event(cancelled)
queued = asyncio.create_task(pool.run_until_complete(queued_ran.set))
await asyncio.sleep(0.01)
assert not queued_ran.is_set()
assert not first.done()
release.set()
with pytest.raises(asyncio.CancelledError):
await first
await queued
finally:
release.set()
await asyncio.gather(first, *([queued] if queued else []), return_exceptions=True)
pool._pool.shutdown(wait=True)
assert queued_ran.is_set()
def test_sync_conversion_produces_complete_sidecar(monkeypatch, tmp_path: Path, conversion_pool):
source = tmp_path / "slides.pptx"
source.write_bytes(b"slides")
monkeypatch.setattr(file_conversion, "_get_pdf_converter", lambda: "auto")
monkeypatch.setattr(file_conversion, "_do_convert_unbounded", lambda *_args: "# Presentation")
result = file_conversion.convert_file_to_markdown_sync(source)
assert result == source.with_name("slides.pptx.md")
assert result.read_text(encoding="utf-8") == "# Presentation"
def test_crashed_conversion_pool_is_replaced_without_retrying_document(monkeypatch, tmp_path: Path):
source = tmp_path / "report.pdf"
source.write_bytes(b"source")
pools = []
class FakePool:
def __init__(self, **kwargs):
assert kwargs["max_workers"] == 2
assert kwargs["mp_context"].get_start_method() == "spawn"
self.failed = not pools
self.calls = 0
self.closed = False
pools.append(self)
def submit(self, *_args):
self.calls += 1
future = Future()
if self.failed:
future.set_exception(BrokenProcessPool("native reader crashed"))
else:
future.set_result("recovered")
return future
def shutdown(self, **_kwargs):
self.closed = True
monkeypatch.setattr(file_conversion, "ProcessPoolExecutor", FakePool)
monkeypatch.setattr(file_conversion, "_PROCESS_POOL", None)
monkeypatch.setattr(file_conversion, "_get_pdf_converter", lambda: "auto")
assert file_conversion.convert_file_to_markdown_sync(source) is None
assert pools[0].calls == 1
assert pools[0].closed
assert file_conversion._PROCESS_POOL is None
assert not file_conversion.markdown_sidecar_path(source).exists()
result = file_conversion.convert_file_to_markdown_sync(source)
assert len(pools) == 2
assert result.read_text(encoding="utf-8") == "recovered"
file_conversion.shutdown_conversion_workers()
assert pools[1].closed
@pytest.mark.asyncio
async def test_real_spawn_conversion_is_isolated_and_bounds_returned_text(monkeypatch, tmp_path: Path):
source = tmp_path / "large.txt"
source.write_text("word " * (file_conversion.MAX_CONVERTED_MARKDOWN_CHARS // 5 + 100), encoding="utf-8")
monkeypatch.setattr(file_conversion, "_get_pdf_converter", lambda: "markitdown")
def reject_parent_parser(*_args):
raise AssertionError("document parsing must execute in a fresh process")
# Spawn imports a clean module; a parser executed in the gateway would hit
# this guard. This test intentionally uses the real installed text reader.
monkeypatch.setattr(file_conversion, "_do_convert_unbounded", reject_parent_parser)
pool = file_conversion._get_conversion_process_pool()
job = None
try:
child_pid = await asyncio.to_thread(lambda: pool.submit(os.getpid).result(timeout=15))
assert child_pid != os.getpid()
assert pool._mp_context.get_start_method() == "spawn"
job = asyncio.create_task(file_conversion.convert_file_to_markdown(source))
# The gateway can execute unrelated coroutine/default-pool work while
# the process imports and runs the real parser.
assert await asyncio.wait_for(asyncio.to_thread(lambda: "available"), timeout=2) == "available"
result = await asyncio.wait_for(job, timeout=30)
assert result == file_conversion.markdown_sidecar_path(source)
converted = result.read_text(encoding="utf-8")
assert len(converted) <= file_conversion.MAX_CONVERTED_MARKDOWN_CHARS
assert "deerflow-conversion-truncated" in converted
finally:
if job is not None:
await asyncio.gather(job, return_exceptions=True)
await asyncio.to_thread(file_conversion.shutdown_conversion_workers)
@@ -0,0 +1,31 @@
import asyncio
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence import engine as database
@pytest.mark.asyncio
async def test_sync_bridge_uses_separate_factory_and_keeps_home_pool(tmp_path, monkeypatch):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'loop.db'}")
factory = async_sessionmaker(engine, expire_on_commit=False)
monkeypatch.setattr(database, "_engine", engine)
monkeypatch.setattr(database, "_session_factory", factory)
monkeypatch.setattr(database, "_engine_loop", asyncio.get_running_loop())
async def read():
isolated = database.get_session_factory()
assert isolated is not factory
async with isolated() as session:
return (await session.execute(text("SELECT 42"))).scalar()
try:
for _ in range(4):
assert database.run_db_blocking(read()) == 42
assert database.get_session_factory() is factory
async with factory() as session:
assert (await session.execute(text("SELECT 1"))).scalar() == 1
finally:
await engine.dispose()
@@ -614,6 +614,9 @@ def test_tool_persists_workbench_session_before_tool_finished(monkeypatch) -> No
_fake_runtime(
"thread-persist",
context={"workbench_session_id": "sess-1"},
messages=[HumanMessage(content=_DEFAULT_TOPIC, additional_kwargs={
"llmwiki_reference_sources": [{"title": _DEFAULT_TOPIC, "content": "新能源汽车市场分析销量证据", "url": "https://example.test/source"}]
})],
),
"call-persist",
_DEFAULT_TOPIC,
@@ -621,6 +624,7 @@ def test_tool_persists_workbench_session_before_tool_finished(monkeypatch) -> No
)
assert updates
assert _FakeEngine.instances[-1].adapters.materials.material_count == 1
assert updates[-1]["session_id"] == "sess-1"
assert updates[-1]["status"] == "completed"
assert updates[-1]["user_id"] is None
@@ -249,6 +249,13 @@ def test_message_visualization_rules_are_offline_safe_and_scoped_to_ordinary_qa(
assert "flow-json" not in ordinary_prompt
assert "chart-json" not in ordinary_prompt
assert "禁止在地图数据中放入 URL" in ordinary_prompt
assert "不要用 `from`/`to`/`coords`" in ordinary_prompt
assert '"coordinates":[[116.4,39.9],[121.47,31.23]]' in ordinary_prompt
assert '"color":"red"' in ordinary_prompt
assert '"arrow":true' in ordinary_prompt
assert '"flow":true' in ordinary_prompt
assert "只有用户**明确要求**连线" in ordinary_prompt
assert "`lines` 必须为 `[]`" in ordinary_prompt
assert "map-json" not in writing_prompt
assert "map-json" not in notebook_prompt
assert "map-json" not in scheduled_prompt
@@ -367,12 +374,13 @@ def test_markdown_file_deliverable_is_scoped_to_ordinary_qa(monkeypatch) -> None
def test_html_runtime_exposes_only_local_geojson_maps(monkeypatch) -> None:
monkeypatch.setattr(
html_runtime,
"get_app_config",
lambda: SimpleNamespace(
html_runtime=SimpleNamespace(
base_url="http://gateway.internal/api/public/html-runtime"
)
),
"html_runtime_base_url",
lambda: "http://gateway.internal/api/public/html-runtime",
)
monkeypatch.setattr(
html_runtime,
"html_map_tile_url",
lambda: "https://mt0.google.com/vt/lyrs=m&hl=zh-CN&x={x}&y={y}&z={z}",
)
app = FastAPI()
app.include_router(html_runtime.router)
@@ -0,0 +1,409 @@
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "facebook-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_facebook_page.py"
FAST_SCRIPT_PATH = SKILL_ROOT / "scripts" / "fast_single_post.py"
LEAD_PROMPT_PATH = BACKEND_ROOT / "packages" / "harness" / "deerflow" / "agents" / "lead_agent" / "prompt.py"
def _load_renderer():
spec = importlib.util.spec_from_file_location("facebook_page_renderer", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "facebook-page-generator"
assert "Facebook" in skill.description
discovered = LocalSkillStorage(host_path=str(SKILL_ROOT.parent.parent)).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert discovered_by_name["facebook-page-generator"].category == SkillCategory.PUBLIC
assert discovered_by_name["facebook-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/facebook-page-generator/SKILL.md" in lead_prompt_source
assert "专用 Facebook 路由" in lead_prompt_source
assert "fast_single_post.py --spec" in lead_prompt_source
assert "动态与评论正文必须使用简体中文" in lead_prompt_source
def test_default_json_renders_editable_facebook_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "facebook-template.html").read_text(encoding="utf-8")
html = renderer.render_document(data, template)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["posts"] >= 1
assert renderer.DATA_MARKER not in html
assert 'id="circleData"' in html
assert 'id="openSettings"' in html
assert 'id="settingsModal"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert "文件不能超过 20MB" not in html
assert '"brand": "Facebook"' in html
assert "<title>Facebook · 社交动态</title>" in html
data["site"]["pageTitle"] = "Facebook · 智能体开发圈"
titled_html = renderer.render_document(data, template)
assert "<title>Facebook · 智能体开发圈</title>" in titled_html
assert "<title>Facebook · 社交动态</title>" not in titled_html
assert "facebook-page-data-v2:${dataFingerprint}" in html
assert "const readStorage = (key)" in html
assert "const savedTheme = readStorage('circle-theme')" in html
assert "writeStorage('circle-theme', next)" in html
assert "const savedTheme = localStorage.getItem('circle-theme')" not in html
assert "const storageKey = 'circle-page-data-v1'" not in html
data["posts"]["items"][0]["text"] = "</script><script>alert('x')</script>"
escaped_html = renderer.render_document(data, template)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "facebook.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="Refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_module_and_duplicate_posts() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data.pop("contacts")
data["posts"]["items"][1]["id"] = data["posts"]["items"][0]["id"]
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: contacts" in message
assert "must be unique" in message
@pytest.mark.parametrize(
"media",
[
{
"type": "image",
"mediaUrl": "https://cdn.example.com/photo.jpg",
"alt": "演示图片",
"pageUrl": "https://example.com/article",
},
{
"type": "video",
"mediaUrl": "https://cdn.example.com/video.mp4",
"posterUrl": "https://cdn.example.com/poster.jpg",
"alt": "演示视频",
},
{
"type": "webpage",
"pageUrl": "https://example.com/article",
"title": "演示网页",
"description": "可在新窗口打开,也可选择内嵌预览。",
"embed": True,
},
],
)
def test_remote_media_contract_renders_safe_editable_markup(media: dict[str, object]) -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["posts"]["items"][0]["media"] = media
renderer.validate_document(data)
html = renderer.render_document(
data,
(SKILL_ROOT / "assets" / "facebook-template.html").read_text(encoding="utf-8"),
)
assert "media.type === 'image'" in html
assert "media.type === 'video'" in html
assert "media.type === 'webpage'" in html
assert 'rel="noopener noreferrer"' in html
assert "safeRemoteUrl" in html
def test_remote_media_contract_rejects_unsafe_urls() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["posts"]["items"][0]["media"] = {
"type": "image",
"mediaUrl": "javascript:alert(1)",
"alt": "不安全图片",
}
with pytest.raises(renderer.DataValidationError, match=r"mediaUrl must use an http\(s\) URL"):
renderer.validate_document(data)
def test_uploaded_data_url_is_accepted_by_media_contract() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["posts"]["items"][0]["media"] = {
"type": "image",
"mediaUrl": "data:image/png;base64,iVBORw0KGgo=",
"alt": "用户上传的图片",
}
renderer.validate_document(data)
def test_single_post_supports_fifty_preloaded_hot_comments() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
data["posts"]["items"] = [post]
post["metrics"]["comments"] = 50
post["commentsExpanded"] = True
post["disclosure"] = "情景模拟页面 · 帖文与热评为演示内容"
post["sourceUrl"] = "https://example.com/source-post"
post["comments"] = [
{
"id": f"comment-{index:02d}",
"author": f"演示用户{index:02d}",
"handle": f"@demo{index:02d}",
"initial": str(index % 10),
"colors": ["#4d6aff", "#8f4bd2"],
"verified": index % 11 == 0,
"time": f"{index + 1} 分钟前",
"text": f"这是第 {index + 1} 条演示热评。",
"likes": 500 - index,
"replyCount": index % 7,
}
for index in range(50)
]
summary = renderer.validate_document(data)
html = renderer.render_document(
data,
(SKILL_ROOT / "assets" / "facebook-template.html").read_text(encoding="utf-8"),
)
assert summary["counts"]["posts"] == 1
assert summary["counts"]["comments"] == 50
assert '"id": "comment-49"' in html
assert "comment-pagination" in html
assert 'data-sort="hot"' in html
assert "showAllLabel" in html
assert "post.disclosure" not in html
assert "post-source" in html
assert "查看原帖" in html
def test_comment_limit_rejects_more_than_one_hundred_items() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
post["metrics"]["comments"] = 101
post["comments"] = [{"author": f"用户{index}", "text": "演示评论"} for index in range(101)]
with pytest.raises(renderer.DataValidationError, match="at most 100"):
renderer.validate_document(data)
def test_fast_single_post_builds_json_and_html_in_one_call(tmp_path: Path) -> None:
command = [
sys.executable,
str(FAST_SCRIPT_PATH),
"--output-dir",
str(tmp_path),
"--author",
"测试账号",
"--text",
"这是一条用于验证快速生成路径的动态。",
"--topic",
"快速生成",
"--comment-count",
"50",
"--media-type",
"image",
"--media-url",
"https://example.com/image.jpg",
"--tag",
"快速生成",
]
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stderr
result = json.loads(completed.stdout)
assert result["status"] == "ok"
assert result["action"] == "fast-single-post"
assert result["counts"]["posts"] == 1
assert result["counts"]["comments"] == 50
data_path = Path(result["data"])
html_path = Path(result["html"])
assert data_path.exists()
assert html_path.exists()
data = json.loads(data_path.read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
assert len(post["comments"]) == 50
assert post["comments"][-1]["id"] == "comment-050"
assert post["media"]["mediaUrl"] == "https://example.com/image.jpg"
assert "disclosure" not in post
html = html_path.read_text(encoding="utf-8")
assert '"id": "comment-050"' in html
assert 'id="openSettings"' in html
second = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert second.returncode == 0, second.stderr
second_result = json.loads(second.stdout)
assert second_result["data"] != result["data"]
assert second_result["html"] != result["html"]
assert Path(second_result["data"]).exists()
assert Path(second_result["html"]).exists()
def test_fast_single_post_spec_separates_total_and_displayed_comments(
tmp_path: Path,
) -> None:
spec_path = tmp_path / "facebook-spec.json"
spec_path.write_text(
json.dumps(
{
"author": "示例账号",
"verified": True,
"text": "今天分享一次重要会晤,双方将继续推进对话与合作。",
"topic": "中美元首会晤",
"tags": ["中美关系"],
"displayCommentCount": 20,
"totalCommentCount": 10000,
"commentTopics": ["中美经济影响", "两岸关系"],
"comments": ["关税调整将直接影响企业成本和供应链安排。"],
"likes": 214000,
"shares": 38600,
},
ensure_ascii=False,
),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stderr
result = json.loads(completed.stdout)
data = json.loads(Path(result["data"]).read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
assert post["metrics"]["comments"] == 10000
assert len(post["comments"]) == 20
assert post["comments"][0]["text"] == "关税调整将直接影响企业成本和供应链安排。"
assert any(topic in post["comments"][1]["text"] for topic in ("中美经济影响", "两岸关系"))
assert post["media"]["type"] == "river"
assert result["counts"]["comments"] == 20
def test_fast_single_post_rejects_english_only_post(tmp_path: Path) -> None:
spec_path = tmp_path / "facebook-english-spec.json"
spec_path.write_text(
json.dumps(
{
"author": "Example",
"text": "This post is written only in English.",
}
),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 2
result = json.loads(completed.stdout)
assert "Simplified Chinese" in result["error"]
spec_path.write_text(
json.dumps(
{
"author": "Example",
"text": "这是一条中文动态。",
"comments": ["This comment is written only in English."],
},
ensure_ascii=False,
),
encoding="utf-8",
)
comment_completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert comment_completed.returncode == 2
assert "Simplified Chinese" in json.loads(comment_completed.stdout)["error"]
@@ -0,0 +1,240 @@
from __future__ import annotations
import json
import re
import subprocess
import sys
from pathlib import Path
import pytest
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILLS_ROOT = BACKEND_ROOT / "skills" / "public"
LEAD_PROMPT_PATH = BACKEND_ROOT / "packages" / "harness" / "deerflow" / "agents" / "lead_agent" / "prompt.py"
FAST_SKILLS = (
"tiktok-page-generator",
"youtube-page-generator",
"threads-page-generator",
"cnn-page-generator",
"bbc-page-generator",
"xinhua-page-generator",
"white-house-page-generator",
"manila-times-page-generator",
"gma-news-page-generator",
"yahoo-japan-page-generator",
"yomiuri-news-page-generator",
)
BRANDED_PAGE_SKILLS = (
"facebook-page-generator",
"x-page-generator",
*FAST_SKILLS,
)
BRAND_HEADER_OFFSET_SNIPPETS = {
"facebook-page-generator": (
".app-header{position:fixed;inset:var(--df-generated-header-height,0px)0auto;",
".sidebar{position:sticky;top:calc(var(--df-generated-header-height,0px)+var(--header-h)+20px);",
),
"x-page-generator": (
".left-fixed{position:fixed;top:var(--df-generated-header-height,0px);",
".feed-header{position:sticky;z-index:30;top:var(--df-generated-header-height,0px);",
),
"tiktok-page-generator": (
".app-header{position:fixed;z-index:80;inset:var(--df-generated-header-height,0px)0auto;",
".channel-tabs{position:fixed;z-index:55;top:calc(var(--df-generated-header-height,0px)+var(--header-h)+10px);",
),
"youtube-page-generator": (
".app-header{position:fixed;z-index:90;inset:var(--df-generated-header-height,0px)0auto;",
".category-bar{position:sticky;z-index:60;top:calc(var(--df-generated-header-height,0px)+var(--header-h));",
),
"threads-page-generator": (
".topbar{position:fixed;z-index:80;inset:var(--df-generated-header-height,0px)0auto;",
".mobile-header{position:fixed;z-index:75;inset:var(--df-generated-header-height,0px)0auto;",
),
"cnn-page-generator": (
".topbar{background:var(--nav);color:white;position:sticky;top:var(--df-generated-header-height,0px);",
),
"bbc-page-generator": (
".settings-trigger{position:fixed;z-index:70;top:calc(var(--df-generated-header-height,0px)+10px);",
),
"xinhua-page-generator": (
".settings-trigger{position:fixed;right:18px;top:calc(var(--df-generated-header-height,0px)+122px);",
),
"white-house-page-generator": (
".site-head{position:absolute;top:calc(var(--df-generated-header-height,0px)+6px);",
".site-head.scrolled{position:fixed;top:var(--df-generated-header-height,0px);",
),
"manila-times-page-generator": (
".primary-nav.stuck{top:var(--df-generated-header-height,0px)}",
".settings-button{top:calc(var(--df-generated-header-height,0px)+18px)}",
),
"gma-news-page-generator": (
"body.sticky.nav{top:var(--df-generated-header-height,0px)}",
".settings-button{top:calc(var(--df-generated-header-height,0px)+58px)}",
),
"yahoo-japan-page-generator": (
".settings-button{top:calc(var(--df-generated-header-height,0px)+20px)}",
),
"yomiuri-news-page-generator": (
".sticky.nav{top:var(--df-generated-header-height,0px)}",
".settings-button{top:calc(var(--df-generated-header-height,0px)+20px)}",
),
}
SPECIALIZED_FAST_SKILLS = {"threads-page-generator", "cnn-page-generator"}
SHARED_FAST_SKILLS = tuple(skill_name for skill_name in FAST_SKILLS if skill_name not in SPECIALIZED_FAST_SKILLS)
@pytest.mark.parametrize("skill_name", BRANDED_PAGE_SKILLS)
def test_platform_chrome_is_offset_below_generated_brand_header(skill_name: str) -> None:
templates = list((SKILLS_ROOT / skill_name / "assets").glob("*-template.html"))
assert len(templates) == 1
compact_template = re.sub(r"\s+", "", templates[0].read_text(encoding="utf-8"))
snippets = BRAND_HEADER_OFFSET_SNIPPETS[skill_name]
for snippet in snippets:
assert snippet in compact_template
@pytest.mark.parametrize("skill_name", BRANDED_PAGE_SKILLS)
def test_platform_modals_render_above_generated_brand_header(skill_name: str) -> None:
templates = list((SKILLS_ROOT / skill_name / "assets").glob("*-template.html"))
assert len(templates) == 1
template = templates[0].read_text(encoding="utf-8")
compact_template = re.sub(r"\s+", "", template)
assert "z-index:1200" in compact_template
assert "z-index:1300" in compact_template
if skill_name == "manila-times-page-generator":
assert ".skip-link:focus{top:calc(var(--df-generated-header-height,0px)+10px)}" in compact_template
else:
assert "translateY(calc(-100%-var(--df-generated-header-height,0px)-24px))" in compact_template
def test_fast_workflow_is_declared_by_all_fixed_template_skills() -> None:
for skill_name in FAST_SKILLS:
skill_text = (SKILLS_ROOT / skill_name / "SKILL.md").read_text(encoding="utf-8")
assert "## 极速工作流(默认)" in skill_text
assert "/scripts/fast_page.py --spec" in skill_text
for skill_name in SHARED_FAST_SKILLS:
skill_text = (SKILLS_ROOT / skill_name / "SKILL.md").read_text(encoding="utf-8")
assert "禁止读取默认数据、契约、模板、运行时或脚本源码" in skill_text
lead_prompt = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "专用固定模板极速规则" in lead_prompt
assert "fast_page.py --spec" in lead_prompt
support_manifest = json.loads((SKILLS_ROOT / "_shared" / ".deerflow-skill-support.json").read_text(encoding="utf-8"))
assert set(support_manifest["required_by"]) == set(SHARED_FAST_SKILLS)
@pytest.mark.parametrize(
("skill_name", "headline_path", "output_stem"),
[
("tiktok-page-generator", "videos.items.0.text", "tiktok-page"),
("youtube-page-generator", "videos.items.0.title", "youtube-page"),
("bbc-page-generator", "lead.title", "bbc-page"),
("xinhua-page-generator", "headline.title", "xinhua-page"),
("white-house-page-generator", "hero.title", "white-house-page"),
("manila-times-page-generator", "lead.main.title", "manila-times-page"),
("gma-news-page-generator", "lead.main.title", "gma-news-page"),
("yahoo-japan-page-generator", "news.feature.title", "yahoo-japan-page"),
("yomiuri-news-page-generator", "lead.main.title", "yomiuri-news-page"),
],
)
def test_compact_spec_renders_complete_page_in_one_call(
tmp_path: Path,
skill_name: str,
headline_path: str,
output_stem: str,
) -> None:
skill_root = SKILLS_ROOT / skill_name
spec_path = tmp_path / "page-spec.json"
spec_path.write_text(
json.dumps(
{
"site": {"pageTitle": f"极速生成 · {skill_name}"},
"$set": {headline_path: "这是通过小型规格 JSON 更新的中文主题内容"},
},
ensure_ascii=False,
),
encoding="utf-8",
)
command = [
sys.executable,
str(skill_root / "scripts" / "fast_page.py"),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
]
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stderr
result = json.loads(completed.stdout)
assert result["status"] == "ok"
assert result["action"] == "fast-page-spec"
data_path = Path(result["data"])
html_path = Path(result["html"])
assert data_path.name == f"{output_stem}-data.json"
assert html_path.name == f"{output_stem}.html"
assert data_path.exists()
assert html_path.exists()
data = json.loads(data_path.read_text(encoding="utf-8"))
assert data["site"]["pageTitle"] == f"极速生成 · {skill_name}"
current: object = data
for part in headline_path.split("."):
current = current[int(part)] if isinstance(current, list) else current[part]
assert current == "这是通过小型规格 JSON 更新的中文主题内容"
html = html_path.read_text(encoding="utf-8")
assert "这是通过小型规格 JSON 更新的中文主题内容" in html
second = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert second.returncode == 0, second.stderr
second_result = json.loads(second.stdout)
assert second_result["data"] != result["data"]
assert second_result["html"] != result["html"]
def test_compact_spec_rejects_unknown_modules(tmp_path: Path) -> None:
skill_root = SKILLS_ROOT / "tiktok-page-generator"
spec_path = tmp_path / "invalid-spec.json"
spec_path.write_text(
json.dumps({"unknownModule": {"title": "不允许"}}, ensure_ascii=False),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(skill_root / "scripts" / "fast_page.py"),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 2
result = json.loads(completed.stdout)
assert result["status"] == "error"
assert "Unknown top-level spec keys" in result["error"]
@@ -3,9 +3,22 @@ from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from deerflow.utils import file_conversion
@pytest.fixture(autouse=True)
def local_parser(monkeypatch):
# Parser-selection unit tests inject readers locally. Production always
# sends this same top-level parser entry point to spawned worker processes.
monkeypatch.setattr(
file_conversion,
"_run_conversion_in_process",
lambda path, converter: file_conversion._convert_document_worker(str(path), converter),
)
def test_legacy_word_is_normalized_before_markitdown(monkeypatch, tmp_path: Path) -> None:
source = tmp_path / "legacy.doc"
source.write_bytes(b"legacy")
@@ -0,0 +1,158 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "gma-news-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_gma_news_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location("gma_news_page_renderer", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "gma-news-page-generator"
assert "GMA News" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert discovered_by_name["gma-news-page-generator"].category == SkillCategory.PUBLIC
assert discovered_by_name["gma-news-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/gma-news-page-generator/SKILL.md" in lead_prompt_source
assert "专用 GMA News 路由" in lead_prompt_source
def test_default_json_renders_editable_gma_news_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
counts = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "gma-news-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "gma-news-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert counts["just_in_items"] == 4
assert counts["lead_stories"] == 3
assert counts["feeds"] == 3
assert counts["weather_locations"] == 4
assert counts["schedule_slots"] == 5
assert counts["videos"] == 3
assert counts["lotto_games"] == 3
assert "__GMA_NEWS_PAGE_DATA__" not in html
assert "__GMA_NEWS_PAGE_RUNTIME__" not in html
assert 'id="gmaNewsPageData"' in html
assert 'id="mainStory"' in html
assert 'id="justPanel"' in html
assert 'id="weatherSelect"' in html
assert 'id="playerModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "external-media::before" in html
assert "data-media-path" in html
assert "exactPathTarget" in html
assert 'const visualHostSelector = "[data-media-path],' in html
assert 'broadcastArt.dataset.mediaPath="broadcast.hero"' in html
assert 'class="video-open"' in html
assert '<button class="video-art' not in html
assert "readAsDataURL" in html
assert '"brandPrimary":"GMA"' in html
assert '"language":"zh-CN"' in html
assert "JSON 编辑器" in html
assert '"gma-news-page-data-v1:" + fingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "localStorage.getItem('metrowave-contrast')" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
output = tmp_path / "gma-news-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_visual_remote_url_and_english_copy() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
data.pop("footer")
data["news"]["topics"][1]["id"] = data["news"]["topics"][0]["id"]
data["lead"]["main"]["visual"] = "remote-image"
data["site"]["description"] = "https://example.com/remote"
data["lead"]["main"]["title"] = "Connections make the capital move faster"
data["site"]["language"] = "en"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "missing top-level module(s): footer" in message
assert "duplicates" in message
assert "visual must be one of" in message
assert "remote URLs" in message
assert "site.language must be zh-CN" in message
assert "lead.main.title must use Simplified Chinese reader-facing copy" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/gma.jpg", "alt": "GMA 新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/gma.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "报道原文", "embed": False},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["lead"]["main"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "gma-news-runtime.js").read_text(encoding="utf-8")
assert "externalMedia" in runtime
assert 'rel="noopener noreferrer"' in runtime
@@ -0,0 +1,164 @@
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from app.gateway.routers import system_settings as system_settings_router
from deerflow.config.html_runtime_config import (
DEFAULT_HTML_MAP_TILE_URL,
DEFAULT_HTML_RUNTIME_BASE_URL,
)
from deerflow.config.system_settings import HtmlRuntimeSettings, SystemSettings
from deerflow.html_runtime.addresses import (
html_map_tile_url,
html_runtime_base_url,
normalize_html_map_tile_url,
normalize_html_runtime_base_url,
)
def test_domain_only_runtime_address_receives_the_public_asset_path() -> None:
assert (
normalize_html_runtime_base_url("https://cm.example.com/")
== "https://cm.example.com/api/public/html-runtime"
)
assert (
normalize_html_runtime_base_url("https://cm.example.com/prefix/api/public/html-runtime/")
== "https://cm.example.com/prefix/api/public/html-runtime"
)
def test_runtime_address_rejects_a_bare_host_and_credentials() -> None:
with pytest.raises(ValueError, match="http://"):
normalize_html_runtime_base_url("cm.example.com")
with pytest.raises(ValueError, match="账号或密码"):
normalize_html_runtime_base_url("https://user:secret@cm.example.com")
def test_tile_address_requires_xyz_placeholders() -> None:
assert normalize_html_map_tile_url(DEFAULT_HTML_MAP_TILE_URL) == DEFAULT_HTML_MAP_TILE_URL
with pytest.raises(ValueError, match="占位符"):
normalize_html_map_tile_url("https://tiles.example/tile.png")
def test_saved_dialog_address_overrides_config(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"deerflow.html_runtime.addresses._saved",
lambda field: {
"base_url": "https://users.example/api/public/html-runtime",
"map_tile_url": "https://tiles.example/{z}/{x}/{y}.png",
}[field],
)
monkeypatch.setattr(
"deerflow.html_runtime.addresses._config_value",
lambda _field: "http://server.internal/api/public/html-runtime",
)
assert html_runtime_base_url(fallback="http://fallback.example/api/public/html-runtime") == (
"https://users.example/api/public/html-runtime"
)
assert html_map_tile_url() == "https://tiles.example/{z}/{x}/{y}.png"
def test_unset_dialog_address_keeps_the_config_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("deerflow.html_runtime.addresses._saved", lambda _field: "")
monkeypatch.setattr(
"deerflow.html_runtime.addresses._config_value",
lambda field: {
"base_url": "http://legacy.example/api/public/html-runtime",
"map_tile_url": "http://legacy.example/tiles/{z}/{x}/{y}.png",
}[field],
)
assert html_runtime_base_url() == "http://legacy.example/api/public/html-runtime"
assert html_map_tile_url() == "http://legacy.example/tiles/{z}/{x}/{y}.png"
def test_missing_config_uses_the_built_in_default(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("deerflow.html_runtime.addresses._saved", lambda _field: "")
monkeypatch.setattr("deerflow.html_runtime.addresses._config_value", lambda _field: "")
assert html_runtime_base_url() == DEFAULT_HTML_RUNTIME_BASE_URL
assert html_map_tile_url() == DEFAULT_HTML_MAP_TILE_URL
def test_new_settings_leave_html_runtime_unset() -> None:
assert SystemSettings().html_runtime == HtmlRuntimeSettings()
@pytest.mark.asyncio
async def test_non_admin_cannot_change_html_runtime_addresses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def ordinary_user(_request):
return SimpleNamespace(system_role="user")
monkeypatch.setattr(
system_settings_router,
"get_optional_user_from_request",
ordinary_user,
)
with pytest.raises(HTTPException) as error:
await system_settings_router.update_html_runtime_settings(
system_settings_router.HtmlRuntimeSettingsUpdateRequest(
base_url="https://users.example",
map_tile_url="https://tiles.example/{z}/{x}/{y}.png",
),
SimpleNamespace(),
)
assert error.value.status_code == 403
@pytest.mark.asyncio
async def test_admin_can_save_html_runtime_addresses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
current = SystemSettings()
saved: list[SystemSettings] = []
async def admin_user(_request):
return SimpleNamespace(system_role="admin")
monkeypatch.setattr(system_settings_router, "get_optional_user_from_request", admin_user)
monkeypatch.setattr(system_settings_router, "load_system_settings", lambda: current)
monkeypatch.setattr(
system_settings_router,
"save_system_settings",
lambda settings: saved.append(settings) or settings,
)
response = await system_settings_router.update_html_runtime_settings(
system_settings_router.HtmlRuntimeSettingsUpdateRequest(
base_url="https://users.example",
map_tile_url="https://tiles.example/{z}/{x}/{y}.png",
),
SimpleNamespace(),
)
assert response.base_url == "https://users.example/api/public/html-runtime"
assert response.map_tile_url == "https://tiles.example/{z}/{x}/{y}.png"
assert saved[0].html_runtime.base_url == response.base_url
assert saved[0].prompt_prefix == current.prompt_prefix
@pytest.mark.asyncio
async def test_invalid_html_runtime_address_is_rejected(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def admin_user(_request):
return SimpleNamespace(system_role="admin")
monkeypatch.setattr(system_settings_router, "get_optional_user_from_request", admin_user)
with pytest.raises(HTTPException) as error:
await system_settings_router.update_html_runtime_settings(
system_settings_router.HtmlRuntimeSettingsUpdateRequest(
base_url="not a url",
map_tile_url="https://tiles.example/{z}/{x}/{y}.png",
),
SimpleNamespace(),
)
assert error.value.status_code == 400
@@ -0,0 +1,856 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from fastapi import HTTPException
from langchain_core.messages import HumanMessage
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.store.memory import InMemoryStore
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway import analysis_projects, services
from app.gateway import knowledge_research as run_policy
from app.gateway.routers import llmwiki as llmwiki_router
from app.gateway.routers.thread_runs import RunCreateRequest
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.persistence.base import Base
from deerflow.persistence.knowledge_research import (
KnowledgeResearchSessionRow,
KnowledgeResearchStore,
)
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
from deerflow.runtime.user_context import reset_current_user, set_current_user
class _MappingStore:
def __init__(self, mapping: dict) -> None:
self.mapping = mapping
self.calls: list[tuple[str, str, bool, bool]] = []
async def get_authorized(
self,
mapping_id: str,
user_id: str,
*,
write: bool,
is_admin: bool,
) -> dict | None:
self.calls.append((mapping_id, user_id, write, is_admin))
if mapping_id != self.mapping["id"]:
return None
if is_admin or user_id == self.mapping["owner_user_id"]:
return dict(self.mapping)
if not write and self.mapping.get("publication_status") == "published":
return dict(self.mapping)
return None
class _WikiClient:
def __init__(self) -> None:
self.created: list[tuple[str, dict]] = []
self.updated: list[tuple[str, str, dict]] = []
self.pages: dict[str, dict] = {}
async def create_wiki_page(self, knowledge_base_id: str, payload: dict) -> dict:
self.created.append((knowledge_base_id, dict(payload)))
page = {"id": "page-one", **payload}
self.pages[str(payload["slug"])] = page
return dict(page)
async def update_wiki_page(
self,
knowledge_base_id: str,
slug: str,
payload: dict,
) -> dict:
self.updated.append((knowledge_base_id, slug, dict(payload)))
page = {"id": "page-one", "slug": slug, **payload}
self.pages[slug] = page
return dict(page)
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
return dict(self.pages[slug])
def _request(**state_values):
return SimpleNamespace(
app=SimpleNamespace(state=SimpleNamespace(**state_values))
)
def test_knowledge_research_model_is_registered_for_startup_create_all() -> None:
from deerflow.persistence import models
assert models.KnowledgeResearchSessionRow is KnowledgeResearchSessionRow
assert "knowledge_research_sessions" in Base.metadata.tables
@pytest.mark.asyncio
async def test_get_by_thread_missing_table_is_not_a_research_session() -> None:
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
try:
store = KnowledgeResearchStore(
async_sessionmaker(engine, expire_on_commit=False)
)
assert await store.get_by_thread("thread-one") is None
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_prepare_run_treats_missing_table_as_ordinary_chat() -> None:
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
try:
store = KnowledgeResearchStore(
async_sessionmaker(engine, expire_on_commit=False)
)
body = RunCreateRequest(
input={"messages": [{"role": "user", "content": "普通问答"}]},
)
prepared, resolved = await run_policy.prepare_knowledge_research_run(
body,
"thread-one",
_request(knowledge_research_store=store),
)
assert prepared is body
assert resolved is None
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_sql_store_keeps_question_title_separate_from_publication_title() -> None:
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
try:
async with engine.begin() as connection:
await connection.run_sync(KnowledgeResearchSessionRow.__table__.create)
store = KnowledgeResearchStore(
async_sessionmaker(engine, expire_on_commit=False)
)
created = await store.create(
knowledge_base_id="kb-one",
user_id="alice",
title="原始问题",
session_id="session-one",
publication_id="publication-one",
thread_id="thread-one",
)
await store.update_owned(
created["id"],
"alice",
draft_markdown="# 答案",
)
saved = await store.save_publication(
created["id"],
"alice",
title="可修改的文件名",
markdown="# 答案",
saved_hash="a" * 64,
wiki_slug="knowledge-research/publication-one",
wiki_page_id="page-one",
wiki_folder_id="folder-one",
)
assert saved is not None
assert saved["title"] == "原始问题"
assert saved["saved_title"] == "可修改的文件名"
assert (await store.get_by_thread("thread-one"))["id"] == "session-one"
assert (await store.get_publication("publication-one"))["id"] == "session-one"
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_memory_store_serializes_draft_update_with_publication() -> None:
store = KnowledgeResearchStore(None)
created = await store.create(
knowledge_base_id="kb-one",
user_id="alice",
title="问题",
session_id="session-one",
publication_id="publication-one",
thread_id="thread-one",
)
await store.update_owned(
created["id"],
"alice",
draft_markdown="# 旧草稿",
)
async with store.locked_publication(created["id"], "alice") as publication:
assert publication is not None
pending_patch = asyncio.create_task(
store.update_owned(
created["id"],
"alice",
draft_markdown="# 新草稿",
)
)
await asyncio.sleep(0)
assert pending_patch.done() is False
publication.save(
title="文件名",
markdown="# 旧草稿",
saved_hash="a" * 64,
wiki_slug="knowledge-research/publication-one",
wiki_page_id="page-one",
wiki_folder_id="folder-one",
)
await pending_patch
current = await store.get(created["id"])
assert current is not None
assert current["draft_markdown"] == "# 新草稿"
assert current["saved_markdown"] == "# 旧草稿"
@pytest.mark.asyncio
async def test_start_run_bypasses_global_fixed_answers_for_research(
monkeypatch,
) -> None:
"""Research must reach scoped KB retrieval, never a global canned answer."""
async def no_project(body, _thread_id, _request):
return body, None
async def research_binding(body, _thread_id, _request):
return body, {"id": "session-one", "mode": "knowledge_research"}
async def forbidden_fixed_answer(**_kwargs):
raise AssertionError("knowledge research must bypass fixed answers")
class _ReachedModelResolution(RuntimeError):
pass
def stop_after_fixed_answer_policy(*_args, **_kwargs):
raise _ReachedModelResolution
monkeypatch.setattr(analysis_projects, "prepare_project_run", no_project)
monkeypatch.setattr(run_policy, "prepare_knowledge_research_run", research_binding)
monkeypatch.setattr(services, "get_stream_bridge", lambda _request: object())
monkeypatch.setattr(services, "get_run_manager", lambda _request: object())
monkeypatch.setattr(
services,
"get_run_context",
lambda _request: SimpleNamespace(app_config=object()),
)
monkeypatch.setattr(services, "get_concurrency_gate", lambda _request: None)
monkeypatch.setattr(services, "get_effective_user_id", lambda: "alice")
monkeypatch.setattr(services, "_find_fixed_answer", forbidden_fixed_answer)
monkeypatch.setattr(
services,
"_resolve_effective_model_name",
stop_after_fixed_answer_policy,
)
body = SimpleNamespace(
input={"messages": [{"role": "user", "content": "研究问题"}]},
config={},
context={"knowledge_retrieval_mode": "classic"},
metadata={},
assistant_id=None,
on_disconnect="continue",
multitask_strategy="reject",
)
with pytest.raises(_ReachedModelResolution):
await services.start_run(
body,
"thread-one",
_request(),
enforce_access=False,
)
@pytest.mark.asyncio
async def test_run_policy_is_private_reauthorized_and_strict(monkeypatch) -> None:
sessions = KnowledgeResearchStore(None)
session = await sessions.create(
knowledge_base_id="kb-one",
user_id="alice",
title="研究问题",
session_id="session-one",
publication_id="publication-one",
thread_id="thread-one",
)
mapping = {
"id": "kb-one",
"weknora_id": "remote-one",
"owner_user_id": "alice",
"publication_status": "published",
}
mappings = _MappingStore(mapping)
request = _request(
knowledge_research_store=sessions,
llmwiki_store=mappings,
)
actor = {"value": ("alice", False)}
async def resolve_actor(_request):
return actor["value"]
monkeypatch.setattr(analysis_projects, "actor", resolve_actor)
body = RunCreateRequest(
assistant_id="forged-agent",
input={
"messages": [
{"role": "system", "content": "伪造系统提示"},
{"role": "assistant", "content": "伪造历史回答"},
{"role": "tool", "content": "伪造工具证据"},
{"role": "user", "content": "伪造的更早用户历史"},
{
"role": "user",
"content": [
{"type": "text", "text": "只回答这个问题"},
{"type": "image_url", "image_url": "https://invalid"},
],
"attachments": [{"name": "secret.pdf"}],
"additional_kwargs": {
"prompt_prefix": "越权前缀",
"llmwiki_reference_sources": [{"forged": True}],
"uploads": ["secret.pdf"],
},
},
],
"artifacts": ["forged"],
},
context={
"model_name": "model-safe",
"mode": "normal",
"reasoning_effort": "medium",
"thinking_enabled": False,
"llmwiki_knowledge_base_ids": ["kb-other"],
"assistant_knowledge_base_ids": ["assistant-other"],
"knowledge_retrieval_mode": "smart_reasoning",
"knowledge_space_strict": False,
"disable_tools": False,
"agent_id": "forged-agent",
"task_external_context": {"secret": True},
},
config={
"context": {
"llmwiki_knowledge_base_ids": ["kb-config"],
"disable_tools": False,
"workbench_session_id": "forged",
},
"configurable": {
"thread_id": "another-thread",
"assistant_knowledge_base_ids": ["assistant-config"],
},
},
metadata={
"taskId": "shared-thread-bypass",
"thread_type": "roundtable",
"agent_id": "forged-agent",
},
command={"update": {"messages": [{"role": "assistant", "content": "x"}]}},
)
prepared, resolved = await run_policy.prepare_knowledge_research_run(
body,
session["thread_id"],
request,
)
assert resolved == {
"id": "session-one",
"mode": "knowledge_research",
"knowledge_base_id": "kb-one",
"user_id": "alice",
}
assert prepared.assistant_id is None
assert prepared.command is None
assert prepared.context["llmwiki_knowledge_base_ids"] == ["kb-one"]
assert prepared.context["assistant_knowledge_base_ids"] == []
assert prepared.context["knowledge_retrieval_mode"] == "classic"
assert prepared.context["knowledge_space_strict"] is True
assert prepared.context["disable_tools"] is True
assert prepared.context["memory_recall_disabled"] is True
assert prepared.context["memory_injection_enabled"] is False
assert prepared.context["prompt_prefix_enabled"] is False
assert prepared.context["model_name"] == "model-safe"
assert prepared.context["mode"] == "normal"
assert prepared.context["reasoning_effort"] == "medium"
assert prepared.context["thinking_enabled"] is False
assert "task_external_context" not in prepared.context
assert prepared.config == {
"configurable": {
"thread_id": "thread-one",
**prepared.context,
}
}
assert "taskId" not in prepared.metadata
assert prepared.metadata["thread_type"] == "knowledge_research"
assert len(prepared.input["messages"]) == 1
message = prepared.input["messages"][0]
assert message["content"] == [{"type": "text", "text": "只回答这个问题"}]
assert "attachments" not in message
assert "additional_kwargs" not in message
# The caller object is never mutated.
assert body.assistant_id == "forged-agent"
assert len(body.input["messages"]) == 5
final_config = {"configurable": {"disable_tools": False}, "context": {}}
run_policy.inject_validated_knowledge_research_context(
final_config,
prepared.context,
)
assert final_config["configurable"]["disable_tools"] is True
assert final_config["context"]["disable_tools"] is True
# Authorization is checked again for every turn, never inherited from
# thread metadata or the previous successful run.
second = RunCreateRequest(input={"messages": [HumanMessage(content="继续")]})
await run_policy.prepare_knowledge_research_run(second, "thread-one", request)
assert [call[2] for call in mappings.calls] == [False, False]
actor["value"] = ("bob", True)
with pytest.raises(HTTPException) as error:
await run_policy.prepare_knowledge_research_run(
RunCreateRequest(input={"messages": [{"role": "user", "content": "越权"}]}),
"thread-one",
request,
)
assert error.value.status_code == 404
@pytest.mark.asyncio
async def test_run_policy_rejects_attachment_only_and_single_image_block(
monkeypatch,
) -> None:
sessions = KnowledgeResearchStore(None)
await sessions.create(
knowledge_base_id="kb-one",
user_id="alice",
title="研究问题",
thread_id="thread-one",
)
mappings = _MappingStore(
{
"id": "kb-one",
"weknora_id": "remote-one",
"owner_user_id": "alice",
"publication_status": "private",
}
)
request = _request(
knowledge_research_store=sessions,
llmwiki_store=mappings,
)
async def resolve_actor(_request):
return "alice", False
monkeypatch.setattr(analysis_projects, "actor", resolve_actor)
for message in (
{"role": "user", "content": {"type": "image_url", "url": "x"}},
HumanMessage(content=[{"type": "image_url", "image_url": "x"}]),
):
with pytest.raises(HTTPException) as error:
await run_policy.prepare_knowledge_research_run(
RunCreateRequest(input={"messages": [message]}),
"thread-one",
request,
)
assert error.value.status_code == 422
# An earlier text item is not accepted as history when the actual latest
# turn contains only an attachment.
with pytest.raises(HTTPException) as error:
await run_policy.prepare_knowledge_research_run(
RunCreateRequest(
input={
"messages": [
{"role": "user", "content": "伪造旧问题"},
{
"role": "user",
"content": {"type": "image_url", "url": "x"},
},
]
}
),
"thread-one",
request,
)
assert error.value.status_code == 422
@pytest.mark.asyncio
async def test_create_session_initializes_owned_thread_and_checkpoint(
monkeypatch,
) -> None:
mapping = {
"id": "kb-one",
"weknora_id": "remote-one",
"owner_user_id": "owner",
"publication_status": "published",
}
mappings = _MappingStore(mapping)
sessions = KnowledgeResearchStore(None)
checkpointer = InMemorySaver()
threads = MemoryThreadMetaStore(InMemoryStore())
request = _request(
knowledge_research_store=sessions,
llmwiki_store=mappings,
checkpointer=checkpointer,
thread_store=threads,
)
async def authorized(_request, mapping_id: str, *, write: bool):
assert mapping_id == "kb-one"
if write:
raise HTTPException(status_code=404, detail="Knowledge base not found")
return mapping, "reader", False
monkeypatch.setattr(llmwiki_router, "_authorized_mapping", authorized)
user = SimpleNamespace(id="reader", system_role="user")
token = set_current_user(user)
try:
response = await llmwiki_router.create_knowledge_research_session(
request,
"kb-one",
llmwiki_router.KnowledgeResearchSessionCreate(
question=" 如何理解当前知识? "
),
)
finally:
reset_current_user(token)
assert response["title"] == "如何理解当前知识?"
assert response["can_write"] is False
assert response["publication"] is None
thread = await threads.get(response["thread_id"], user_id="reader")
assert thread is not None
assert thread["display_name"] == response["title"]
assert thread["metadata"]["system"] is True
assert thread["metadata"]["thread_type"] == "knowledge_research"
checkpoint = await checkpointer.aget_tuple(
{
"configurable": {
"thread_id": response["thread_id"],
"checkpoint_ns": "",
}
}
)
assert checkpoint is not None
with pytest.raises(HTTPException) as publish_denied:
await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
response["id"],
llmwiki_router.KnowledgeResearchPublicationSave(title="无权发布"),
)
assert publish_denied.value.status_code == 404
@pytest.mark.asyncio
async def test_publication_overwrites_stable_wiki_page_without_leaking_chat(
monkeypatch,
) -> None:
mapping = {
"id": "kb-one",
"weknora_id": "remote-one",
"owner_user_id": "alice",
"publication_status": "published",
}
mappings = _MappingStore(mapping)
sessions = KnowledgeResearchStore(None)
created = await sessions.create(
knowledge_base_id="kb-one",
user_id="alice",
title="原始问题",
session_id="session-one",
publication_id="publication-one",
thread_id="thread-one",
)
await sessions.update_owned(
created["id"],
"alice",
draft_markdown="# 初稿\n\n结论。",
)
request = _request(
knowledge_research_store=sessions,
llmwiki_store=mappings,
)
actor = {"value": "alice"}
async def authorized(_request, mapping_id: str, *, write: bool):
user_id = actor["value"]
row = await mappings.get_authorized(
mapping_id,
user_id,
write=write,
is_admin=False,
)
if row is None:
raise HTTPException(status_code=404, detail="Knowledge base not found")
return row, user_id, False
client = _WikiClient()
sync = Mock()
monkeypatch.setattr(llmwiki_router, "_authorized_mapping", authorized)
monkeypatch.setattr(llmwiki_router, "_client_or_503", lambda: client)
async def ensure_directory(_client, remote_id: str, paths: list[list[str]]):
assert remote_id == "remote-one"
assert paths == [["深度研究"]]
return {"深度研究": "folder-research"}
monkeypatch.setattr(llmwiki_router, "ensure_directory_paths", ensure_directory)
monkeypatch.setattr(llmwiki_router, "_schedule_local_wiki_sync", sync)
first = await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(title="研究报告"),
)
first_hash = first["publication"]["saved_hash"]
slug = first["publication"]["wiki_slug"]
assert first["title"] == "原始问题"
assert first["publication"]["title"] == "研究报告"
assert slug == "knowledge-research/publication-one"
assert client.created[0][1]["folder_id"] == "folder-research"
assert client.created[0][1]["content"] == "# 初稿\n\n结论。"
assert sync.call_count == 1
with pytest.raises(HTTPException) as missing_version:
await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(title="缺少版本"),
)
assert missing_version.value.status_code == 409
with pytest.raises(HTTPException) as stale:
await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(
title="仅改标题",
expected_saved_hash="0" * 64,
),
)
assert stale.value.status_code == 409
assert not client.updated
renamed = await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(
title="仅改标题",
expected_saved_hash=first_hash,
),
)
assert renamed["publication"]["wiki_slug"] == slug
assert renamed["title"] == "原始问题"
assert renamed["publication"]["title"] == "仅改标题"
assert renamed["publication"]["saved_hash"] != first_hash
assert client.updated[0][1] == slug
assert sync.call_count == 2
# If the previous response was lost, retrying the identical desired
# publication with the old pre-save token is an idempotent no-op.
retried = await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(
title="仅改标题",
expected_saved_hash=first_hash,
),
)
assert retried["publication"]["saved_hash"] == renamed["publication"]["saved_hash"]
assert len(client.updated) == 1
assert sync.call_count == 2
entered_remote_update = asyncio.Event()
release_remote_update = asyncio.Event()
original_update = client.update_wiki_page
async def blocking_update(*args, **kwargs):
entered_remote_update.set()
await release_remote_update.wait()
return await original_update(*args, **kwargs)
monkeypatch.setattr(client, "update_wiki_page", blocking_update)
concurrent_a = asyncio.create_task(
llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(
title="并发版本 A",
expected_saved_hash=renamed["publication"]["saved_hash"],
),
)
)
await entered_remote_update.wait()
concurrent_b = asyncio.create_task(
llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(
title="并发版本 B",
expected_saved_hash=renamed["publication"]["saved_hash"],
),
)
)
await asyncio.sleep(0)
release_remote_update.set()
concurrent_results = await asyncio.gather(
concurrent_a,
concurrent_b,
return_exceptions=True,
)
assert isinstance(concurrent_results[0], dict)
assert isinstance(concurrent_results[1], HTTPException)
assert concurrent_results[1].status_code == 409
assert len(client.updated) == 2
assert sync.call_count == 3
other = await sessions.create(
knowledge_base_id="kb-one",
user_id="bob",
title="Bob 的私人会话",
session_id="session-bob",
publication_id="publication-bob",
thread_id="thread-bob",
)
await sessions.update_owned(
other["id"],
"bob",
draft_markdown="# 私人内容",
)
# KB owner has write permission but cannot read/publish another user's
# private research session.
with pytest.raises(HTTPException) as owner_cannot_publish_private_chat:
await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-bob",
llmwiki_router.KnowledgeResearchPublicationSave(title="越权发布"),
)
assert owner_cannot_publish_private_chat.value.status_code == 404
async def deposit_authorized(_request, mapping_id: str, *, write: bool):
return {
**mapping,
"owner_user_id": "system",
"name": llmwiki_router.CONVERSATION_DEPOSIT_KB_NAME,
}, "alice", True
monkeypatch.setattr(
llmwiki_router,
"_authorized_mapping",
deposit_authorized,
)
with pytest.raises(HTTPException) as managed_deposit:
await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(title="禁止发布"),
)
assert managed_deposit.value.status_code == 403
monkeypatch.setattr(llmwiki_router, "_authorized_mapping", authorized)
# An unsaved draft must not alter the shared bookshelf snapshot.
await sessions.update_owned(
"session-one",
"alice",
draft_markdown="# 尚未保存的新内容",
)
actor["value"] = "bob"
listing = await llmwiki_router.list_knowledge_research_sessions(
request,
"kb-one",
)
assert [row["id"] for row in listing["sessions"]] == ["session-bob"]
assert all(row["id"] != "session-one" for row in listing["sessions"])
assert len(listing["publications"]) == 1
public_book = listing["publications"][0]
assert public_book["id"] == "publication-one"
assert public_book["can_continue"] is False
assert public_book["can_write"] is False
assert "session_id" not in public_book
assert "thread_id" not in public_book
assert "content_markdown" not in public_book
detail = await llmwiki_router.get_knowledge_research_publication(
request,
"kb-one",
"publication-one",
)
assert detail["content_markdown"] == "# 初稿\n\n结论。"
assert "session_id" not in detail
assert "thread_id" not in detail
assert "draft_markdown" not in detail
with pytest.raises(HTTPException) as private_chat:
await llmwiki_router.get_knowledge_research_session(
request,
"kb-one",
"session-one",
)
assert private_chat.value.status_code == 404
@pytest.mark.asyncio
@pytest.mark.parametrize(("upstream_status", "expected_status"), [(400, 422), (503, 502)])
async def test_publication_translates_upstream_directory_errors(
monkeypatch,
upstream_status: int,
expected_status: int,
) -> None:
mapping = {
"id": "kb-one",
"weknora_id": "remote-one",
"owner_user_id": "alice",
"publication_status": "private",
}
sessions = KnowledgeResearchStore(None)
created = await sessions.create(
knowledge_base_id="kb-one",
user_id="alice",
title="问题",
session_id="session-one",
publication_id="publication-one",
thread_id="thread-one",
)
await sessions.update_owned(
created["id"],
"alice",
draft_markdown="# 正文",
)
request = _request(knowledge_research_store=sessions)
async def authorized(_request, mapping_id: str, *, write: bool):
assert mapping_id == "kb-one"
assert write is True
return mapping, "alice", False
async def failed_directory(*_args, **_kwargs):
raise WeKnoraError("上游目录失败", status_code=upstream_status)
sync = Mock()
monkeypatch.setattr(llmwiki_router, "_authorized_mapping", authorized)
monkeypatch.setattr(llmwiki_router, "_client_or_503", _WikiClient)
monkeypatch.setattr(llmwiki_router, "ensure_directory_paths", failed_directory)
monkeypatch.setattr(llmwiki_router, "_schedule_local_wiki_sync", sync)
with pytest.raises(HTTPException) as error:
await llmwiki_router.save_knowledge_research_publication(
request,
"kb-one",
"session-one",
llmwiki_router.KnowledgeResearchPublicationSave(title="报告"),
)
assert error.value.status_code == expected_status
assert "上游目录失败" in str(error.value.detail)
assert (await sessions.get("session-one"))["saved_hash"] is None
sync.assert_not_called()
@@ -1,5 +1,7 @@
from types import SimpleNamespace
import pytest
from app.gateway.services import (
apply_knowledge_space_strict_scope,
enforce_knowledge_retrieval_mode_run_config,
@@ -128,6 +130,55 @@ def test_retrieval_modes_mount_mutually_exclusive_knowledge_tool_surfaces() -> N
assert "ordinary_knowledge_search" in excluded
assert "ordinary_wiki_search" in excluded
@pytest.mark.parametrize("delivery", [False, True])
def test_classic_strict_knowledge_scope_builds_lead_agent(monkeypatch, delivery) -> None:
from deerflow.agents.deep_research import retrieval
from deerflow.agents.lead_agent import agent as lead_agent
captured_tools: dict = {}
app_config = SimpleNamespace(
get_model_config=lambda _name: SimpleNamespace(supports_thinking=True),
)
monkeypatch.setattr(lead_agent, "_resolve_model_name", lambda *_args, **_kwargs: "test-model")
monkeypatch.setattr(lead_agent, "validate_agent_id", lambda _value: "roundtable-coordinator")
monkeypatch.setattr(lead_agent, "create_chat_model", lambda **_kwargs: object())
monkeypatch.setattr(lead_agent, "create_agent", lambda **kwargs: kwargs)
monkeypatch.setattr(lead_agent, "load_agent_soul", lambda _agent: "")
monkeypatch.setattr(lead_agent, "_log_es_query_routing_registration", lambda **_kwargs: None)
monkeypatch.setattr(lead_agent, "_build_middlewares", lambda *_args, **_kwargs: [])
monkeypatch.setattr(lead_agent, "apply_prompt_template", lambda **_kwargs: "test prompt")
monkeypatch.setattr(lead_agent, "_filter_available_skills_for_excluded_tools", lambda skills, _excluded: skills)
monkeypatch.setattr(retrieval, "resolve_collector_skill_visibility", lambda _agent, skills, _extra: skills)
monkeypatch.setattr(retrieval, "runtime_extra_skills", lambda _config: [])
import deerflow.tools
def fake_get_available_tools(**kwargs):
captured_tools.update(kwargs)
return []
monkeypatch.setattr(deerflow.tools, "get_available_tools", fake_get_available_tools)
result = lead_agent._make_lead_agent(
{
"configurable": {
"is_bootstrap": False,
"writing_mode": True,
"workbench_write_now": delivery,
"workbench_session_id": "test-session" if delivery else None,
"knowledge_space_strict": True,
"knowledge_retrieval_mode": "classic",
}
},
app_config=app_config,
)
assert result["system_prompt"].startswith("test prompt")
assert captured_tools["include_mcp"] is False
assert "web_search" in captured_tools["excluded_tools"]
assert ("deep_research_report" in {tool.name for tool in result["tools"]}) is delivery
smart = {
"configurable": {
"llmwiki_knowledge_base_ids": ["kb-1"],
@@ -16,6 +16,7 @@ import pytest
from langchain_core.messages import AIMessage, AIMessageChunk
from deerflow.agents.middlewares.llm_error_handling_middleware import LLM_ERROR_MARKER, LLMErrorHandlingMiddleware
from deerflow.models.dsml_tool_calls import DSMLToolCallProtocolError
from deerflow.models.patched_openai import OpenAICompatibleUpstreamError, PatchedChatOpenAI
@@ -63,6 +64,11 @@ def test_openai_compatible_null_choices_error_is_retriable_upstream() -> None:
assert _middleware()._classify_error(error) == (True, "upstream")
def test_dsml_protocol_error_is_retriable_upstream() -> None:
error = DSMLToolCallProtocolError("truncated DSML")
assert _middleware()._classify_error(error) == (True, "upstream")
def test_insufficient_balance_stays_non_retriable_quota() -> None:
"""The 200-body compatibility fix must not hide a real billing failure."""
@@ -17,15 +17,25 @@ from datetime import UTC, datetime, timedelta
import pytest
import pytest_asyncio
from langchain_core.messages import AIMessage
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.admin_audit_retention import sweep_admin_audit_retention
from deerflow.agents.middlewares.llm_metrics_middleware import (
LlmMetricsMiddleware,
_extract_usage,
)
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.base import Base
from deerflow.persistence.llm_metrics import SqlLlmMetricsStore
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.run import RunRepository
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.tool_metrics import SqlToolMetricsStore
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.types import BEIJING_TZ
class _RecordingStore:
@@ -52,10 +62,12 @@ class _RecordingStore:
async def sql_store(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'metrics.db'}")
async with engine.begin() as conn:
# Importing the package registers the LlmCallMetricRow with Base.metadata.
import deerflow.persistence.llm_metrics.model # noqa: F401
await conn.run_sync(Base.metadata.create_all)
await conn.run_sync(
lambda sync_conn: Base.metadata.create_all(
sync_conn,
tables=[LlmCallMetricRow.__table__],
)
)
sf = async_sessionmaker(engine, expire_on_commit=False)
store = SqlLlmMetricsStore(sf)
try:
@@ -64,6 +76,57 @@ async def sql_store(tmp_path):
await engine.dispose()
@pytest_asyncio.fixture
async def tool_sql_store(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'tool-metrics.db'}")
async with engine.begin() as conn:
await conn.run_sync(
lambda sync_conn: Base.metadata.create_all(
sync_conn,
tables=[ToolCallMetricRow.__table__],
)
)
store = SqlToolMetricsStore(async_sessionmaker(engine, expire_on_commit=False))
try:
yield store
finally:
await engine.dispose()
@pytest_asyncio.fixture
async def run_sql_store(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'runs.db'}")
async with engine.begin() as conn:
await conn.run_sync(
lambda sync_conn: Base.metadata.create_all(
sync_conn,
tables=[RunRow.__table__],
)
)
store = RunRepository(async_sessionmaker(engine, expire_on_commit=False))
try:
yield store
finally:
await engine.dispose()
@pytest_asyncio.fixture
async def snapshot_session_factory(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'snapshots.db'}")
async with engine.begin() as conn:
await conn.run_sync(
lambda sync_conn: Base.metadata.create_all(
sync_conn,
tables=[AdminLeaderboardDailyStatRow.__table__],
)
)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
try:
yield session_factory
finally:
await engine.dispose()
def _make_request():
return type(
"FakeRequest",
@@ -258,6 +321,330 @@ async def test_sql_store_iter_all_paginates(sql_store):
assert seen == 1200
@pytest.mark.asyncio
async def test_llm_metric_purge_before_keeps_rows_at_or_after_cutoff(sql_store):
"""The retention boundary is exclusive: a record exactly at cutoff stays."""
cutoff = datetime.now(UTC) - timedelta(days=90)
for record_id, created_at in (
("old", cutoff - timedelta(days=1)),
("boundary", cutoff),
("fresh", cutoff + timedelta(days=1)),
):
await sql_store.record(
LlmMetricRecord(id=record_id, created_at=created_at, duration_ms=1)
)
assert await sql_store.purge_before(cutoff) == 1
# A subsequent cleanup pass is idempotent.
assert await sql_store.purge_before(cutoff) == 0
rows = await sql_store.list(LlmMetricsQuery(limit=10))
assert {row["id"] for row in rows} == {"boundary", "fresh"}
@pytest.mark.asyncio
async def test_tool_metric_purge_before_keeps_rows_at_or_after_cutoff(tool_sql_store):
"""Tool-call audit retention has the same strict-before-cutoff behavior."""
cutoff = datetime.now(UTC) - timedelta(days=90)
for record_id, created_at in (
("old", cutoff - timedelta(days=1)),
("boundary", cutoff),
("fresh", cutoff + timedelta(days=1)),
):
await tool_sql_store.record(
ToolMetricRecord(
id=record_id,
created_at=created_at,
tool_name="web_search",
duration_ms=1,
)
)
assert await tool_sql_store.purge_before(cutoff) == 1
assert await tool_sql_store.purge_before(cutoff) == 0
rows = await tool_sql_store.list(ToolMetricsQuery(limit=10))
assert {row["id"] for row in rows} == {"boundary", "fresh"}
@pytest.mark.asyncio
async def test_terminal_run_summary_purge_keeps_boundary_and_live_runs(run_sql_store):
"""Audit-summary retention never deletes a current, boundary, or late-completed run."""
cutoff = datetime.now(UTC) - timedelta(days=90)
async with run_sql_store._sf() as session:
session.add_all(
[
RunRow(
run_id="old-terminal",
thread_id="thread-old-terminal",
status="success",
created_at=cutoff - timedelta(seconds=1),
updated_at=cutoff - timedelta(seconds=1),
),
RunRow(
run_id="boundary-terminal",
thread_id="thread-boundary-terminal",
status="error",
created_at=cutoff,
updated_at=cutoff,
),
RunRow(
run_id="old-live",
thread_id="thread-old-live",
status="running",
created_at=cutoff - timedelta(days=1),
updated_at=cutoff - timedelta(days=1),
),
RunRow(
run_id="old-created-recently-completed",
thread_id="thread-old-created-recently-completed",
status="success",
created_at=cutoff - timedelta(days=1),
updated_at=cutoff + timedelta(seconds=1),
),
]
)
await session.commit()
assert await run_sql_store.purge_terminal_before(cutoff) == 1
assert await run_sql_store.get("old-terminal", user_id=None) is None
assert await run_sql_store.get("boundary-terminal", user_id=None) is not None
assert await run_sql_store.get("old-live", user_id=None) is not None
assert await run_sql_store.get("old-created-recently-completed", user_id=None) is not None
@pytest.mark.asyncio
async def test_llm_metric_purge_caps_one_large_batch_at_500(sql_store):
"""A single metrics delete never expands into an unbounded SQL statement."""
cutoff = datetime.now(UTC) - timedelta(days=90)
async with sql_store._sf() as session:
session.add_all(
[
LlmCallMetricRow(
id=f"expired-{index}",
created_at=cutoff - timedelta(days=1),
duration_ms=1,
)
for index in range(501)
]
)
await session.commit()
# Even a much larger requested size must respect the portable 500-row cap.
assert await sql_store.purge_before(cutoff, batch_size=10_000) == 500
assert await sql_store.count(LlmMetricsQuery()) == 1
assert await sql_store.purge_before(cutoff, batch_size=10_000) == 1
@pytest.mark.asyncio
async def test_terminal_run_summary_purge_caps_one_large_batch_at_500(run_sql_store):
"""Run-summary retention uses the same bounded primary-key deletion rule."""
cutoff = datetime.now(UTC) - timedelta(days=90)
async with run_sql_store._sf() as session:
session.add_all(
[
RunRow(
run_id=f"expired-run-{index}",
thread_id=f"thread-{index}",
status="success",
created_at=cutoff - timedelta(days=1),
updated_at=cutoff - timedelta(days=1),
)
for index in range(501)
]
)
await session.commit()
assert await run_sql_store.purge_terminal_before(cutoff, batch_size=10_000) == 500
async with run_sql_store._sf() as session:
remaining = list((await session.scalars(select(RunRow.run_id))).all())
assert len(remaining) == 1
assert await run_sql_store.purge_terminal_before(cutoff, batch_size=10_000) == 1
@pytest.mark.asyncio
async def test_admin_retention_sweep_deletes_only_one_configured_batch_per_store(
sql_store,
run_sql_store,
):
"""A small sweep budget leaves the remaining historic backlog for later ticks."""
now = datetime(2026, 10, 26, tzinfo=UTC)
cutoff = now - timedelta(days=90)
async with sql_store._sf() as session:
session.add_all(
[
LlmCallMetricRow(
id=f"metric-{index}",
created_at=cutoff - timedelta(days=1),
duration_ms=1,
)
for index in range(11)
]
)
await session.commit()
async with run_sql_store._sf() as session:
session.add_all(
[
RunRow(
run_id=f"run-{index}",
thread_id=f"thread-{index}",
status="success",
created_at=cutoff - timedelta(days=1),
updated_at=cutoff - timedelta(days=1),
)
for index in range(11)
]
)
await session.commit()
result = await sweep_admin_audit_retention(
session_factory=None,
llm_metrics_store=sql_store,
run_store=run_sql_store,
now=now,
batch_size=10,
max_batches_per_store=1,
inter_batch_pause_seconds=0,
max_sweep_seconds=60,
)
assert result["llm_metrics"] == 10
assert result["run_summaries"] == 10
assert await sql_store.count(LlmMetricsQuery()) == 1
async with run_sql_store._sf() as session:
remaining_runs = list((await session.scalars(select(RunRow.run_id))).all())
assert len(remaining_runs) == 1
@pytest.mark.asyncio
async def test_admin_retention_snapshot_batch_does_not_delete_an_entire_old_date(
snapshot_session_factory,
):
"""Snapshot cleanup selects composite keys, not every settings hash for a date."""
now = datetime(2026, 10, 26, tzinfo=UTC)
cutoff = now - timedelta(days=90)
cutoff_date = cutoff.astimezone(BEIJING_TZ).date().isoformat()
old_date = (cutoff - timedelta(days=1)).astimezone(BEIJING_TZ).date().isoformat()
async with snapshot_session_factory() as session:
session.add_all(
[
AdminLeaderboardDailyStatRow(
stat_date=old_date,
settings_hash=f"old-hash-{index:02d}",
)
for index in range(12)
]
+ [
AdminLeaderboardDailyStatRow(
stat_date=cutoff_date,
settings_hash="boundary-hash",
)
]
)
await session.commit()
result = await sweep_admin_audit_retention(
session_factory=snapshot_session_factory,
now=now,
batch_size=10,
max_batches_per_store=1,
inter_batch_pause_seconds=0,
max_sweep_seconds=60,
)
assert result["leaderboard_snapshots"] == 10
async with snapshot_session_factory() as session:
rows = list((await session.scalars(select(AdminLeaderboardDailyStatRow))).all())
assert sum(row.stat_date == old_date for row in rows) == 2
assert [(row.stat_date, row.settings_hash) for row in rows if row.stat_date == cutoff_date] == [
(cutoff_date, "boundary-hash")
]
class _AlwaysFullRetentionStore:
"""A store that models an arbitrarily large historical backlog."""
def __init__(self, maximum_calls: int) -> None:
self.calls: list[tuple[datetime, int]] = []
self.maximum_calls = maximum_calls
async def purge_before(self, cutoff: datetime, *, batch_size: int) -> int:
self.calls.append((cutoff, batch_size))
# Fail fast if a future implementation loses the sweep budget instead
# of letting this regression test loop forever.
if len(self.calls) > self.maximum_calls:
raise AssertionError("retention sweep exceeded its per-store work budget")
return batch_size
class _BusyRetentionStore:
"""Models a table temporarily locked by another database writer."""
async def purge_before(self, cutoff: datetime, *, batch_size: int) -> int:
raise RuntimeError("database is locked")
@pytest.mark.asyncio
async def test_admin_retention_sweep_has_fixed_per_store_work_budget():
"""One scheduler tick stops after a bounded amount of historic work."""
batch_size = 37
max_batches = 3
store = _AlwaysFullRetentionStore(maximum_calls=max_batches)
now = datetime(2026, 10, 26, tzinfo=UTC)
result = await sweep_admin_audit_retention(
session_factory=None,
llm_metrics_store=store,
now=now,
batch_size=batch_size,
max_batches_per_store=max_batches,
inter_batch_pause_seconds=0,
max_sweep_seconds=60,
)
expected_cutoff = now - timedelta(days=90)
assert store.calls == [(expected_cutoff, batch_size)] * max_batches
assert result == {
"llm_metrics": batch_size * max_batches,
"tool_metrics": 0,
"run_summaries": 0,
"leaderboard_snapshots": 0,
}
@pytest.mark.asyncio
async def test_admin_retention_busy_store_does_not_block_other_sources():
"""A lock on one audit table defers only that source to a later tick."""
available_store = _AlwaysFullRetentionStore(maximum_calls=1)
result = await sweep_admin_audit_retention(
session_factory=None,
llm_metrics_store=_BusyRetentionStore(),
tool_metrics_store=available_store,
now=datetime(2026, 10, 26, tzinfo=UTC),
batch_size=10,
max_batches_per_store=1,
inter_batch_pause_seconds=0,
max_sweep_seconds=60,
)
assert available_store.calls
assert result == {
"llm_metrics": 0,
"tool_metrics": 10,
"run_summaries": 0,
"leaderboard_snapshots": 0,
}
def test_extract_usage_prefers_usage_metadata():
response = AIMessage(
content="x",
@@ -274,10 +274,10 @@ async def test_local_path_save_and_restore_persist_mode_and_rebuild(
assert rebuilds == [True, True]
def test_both_shipped_offline_configs_enable_incremental_wiki_rescan() -> None:
def test_both_shipped_offline_configs_keep_wiki_rescan_manual_by_default() -> None:
offline_root = Path(__file__).resolve().parents[2]
for path in (offline_root / "config.yaml", offline_root / "backend" / "config.yaml"):
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
local_index = payload["llmwiki"]["local_wiki_index"]
assert local_index["enabled"] is True
assert local_index["auto_sync"] is True
assert local_index["auto_sync"] is False
@@ -78,7 +78,10 @@ async def test_lists_hide_remote_deleted_bases_without_deleting_mappings(knowled
state.remote["remote-live"] = {"id": "remote-live", "name": "保留知识库"}
deleted = state.remote.pop("remote-one")
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
response = await state.api.get(
"/api/llmwiki/knowledge-bases",
params={"scope": scope, "refresh": True},
)
assert response.status_code == 200
assert [item["id"] for item in response.json()["knowledge_bases"]] == [live["id"]]
@@ -98,24 +101,30 @@ async def test_directory_scopes_share_one_upstream_read_and_keep_visibility_chec
state = knowledge_api
responses = await asyncio.gather(*(state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope}) for scope in ("personal", "public", "all")))
assert all(response.status_code == 200 for response in responses)
assert state.directory_reads == 1
assert state.directory_reads == 0
assert all(
item["remote_status"] == "cached"
for response in responses
for item in response.json()["knowledge_bases"]
)
state.user_id = "another-user"
personal = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
public = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "public"})
assert personal.json()["knowledge_bases"] == []
assert [item["id"] for item in public.json()["knowledge_bases"]] == [state.mapping_id]
assert state.directory_reads == 1
assert state.directory_reads == 0
@pytest.mark.asyncio
async def test_manual_directory_refresh_coalesces_and_reloads_remote_changes(knowledge_api) -> None:
state = knowledge_api
await state.api.get("/api/llmwiki/knowledge-bases")
assert state.directory_reads == 0
state.remote.clear()
responses = await asyncio.gather(*(state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope, "refresh": True}) for scope in ("personal", "public", "all")))
assert all(response.json()["knowledge_bases"] == [] for response in responses)
assert state.directory_reads == 2
assert state.directory_reads == 1
assert await state.store.get_by_weknora_id("remote-one") is not None
@@ -125,12 +134,18 @@ async def test_list_upstream_failure_is_not_treated_as_remote_deletion(knowledge
state = knowledge_api
state.upstream_status = status
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
response = await state.api.get(
"/api/llmwiki/knowledge-bases",
params={"scope": "personal", "refresh": True},
)
assert response.status_code >= 500
assert await state.store.get_by_weknora_id("remote-one") is not None
state.upstream_status = 200
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
response = await state.api.get(
"/api/llmwiki/knowledge-bases",
params={"scope": "personal", "refresh": True},
)
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
@@ -5,6 +5,7 @@ from types import SimpleNamespace
from typing import Any
import pytest
from fastapi import HTTPException
from app.gateway import llmwiki_index_scheduler
from app.gateway.routers import llmwiki as llmwiki_router
@@ -216,8 +216,9 @@ def test_numbered_citation_prompt_is_classic_only() -> None:
@pytest.mark.asyncio
@pytest.mark.parametrize("assistant_id", [None, "deep-research-planner", "deep-research-collector"])
async def test_smart_scope_binds_this_library_to_authorized_database_metadata(
monkeypatch: pytest.MonkeyPatch,
monkeypatch: pytest.MonkeyPatch, assistant_id: str | None,
) -> None:
from app.gateway import llmwiki_rag
@@ -273,7 +274,7 @@ async def test_smart_scope_binds_this_library_to_authorized_database_metadata(
body=SimpleNamespace(
context={"llmwiki_knowledge_base_ids": ["mapping-news"]},
metadata={},
assistant_id=None,
assistant_id=assistant_id,
),
graph_input={"messages": [HumanMessage(content="这个库讲了什么")]},
)
@@ -604,12 +605,14 @@ async def test_wiki_tool_streams_channel_coverage_and_scan_count(monkeypatch) ->
}
],
pattern=pattern,
channel="local_wiki_mirror",
channel="provider_native_regex",
partial=False,
native_regex_supported=True,
unsupported_knowledge_base_count=0,
coverage=[
{
"knowledge_base": "国内新闻库",
"channel": "local_wiki_mirror",
"channel": "provider_native_regex",
"status": "searched",
}
],
@@ -629,13 +632,15 @@ async def test_wiki_tool_streams_channel_coverage_and_scan_count(monkeypatch) ->
]
completed = events[-1]
assert completed["pattern"] == "新闻|发布会"
assert completed["channel"] == "local_wiki_mirror"
assert completed["channel"] == "provider_native_regex"
assert completed["scanned_pages"] == 17
assert completed["result_count"] == 1
assert completed["native_regex_supported"] is True
assert completed["unsupported_knowledge_base_count"] == 0
assert completed["coverage"] == [
{
"knowledge_base": "国内新闻库",
"channel": "local_wiki_mirror",
"channel": "provider_native_regex",
"status": "searched",
}
]
@@ -1023,74 +1028,50 @@ async def test_search_all_failures_are_errors_and_partial_failure_is_structured(
@pytest.mark.asyncio
async def test_wiki_search_selects_local_or_provider_per_mapping(monkeypatch) -> None:
async def test_wiki_search_uses_native_provider_for_each_mapping_concurrently(
monkeypatch,
) -> None:
_patch_scope(monkeypatch, multi=True)
class LocalIndex:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
assert mapping_ids == ["mapping-1", "mapping-2"]
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 1,
"local_page_count": 1,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
},
{
"knowledge_base_mapping_id": "mapping-2",
"state": "partial",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": "one page failed",
"remote_page_count": 2,
"local_page_count": 1,
"failed_page_count": 1,
},
]
class LocalIndexMustNotRun:
def __init__(self) -> None:
raise AssertionError("smart Wiki regex must use the native provider API")
async def list_pages(self, mapping_ids: list[str], *, limit: int) -> list[dict]:
assert mapping_ids == ["mapping-1"]
assert limit == 250
class Client:
calls: list[tuple[str, str, int]] = []
active = 0
peak = 0
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
self.calls.append((knowledge_base_id, query, limit))
self.active += 1
self.peak = max(self.peak, self.active)
await asyncio.sleep(0.01)
self.active -= 1
suffix = knowledge_base_id.rsplit("-", 1)[-1]
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": "local-page",
"title": "本地 needle 页面",
"summary": "完整镜像",
"content_md": "本地正文",
"slug": f"news/{suffix}",
"title": f"needle 页面 {suffix}",
"content": "needle 正文",
"status": "published",
}
]
class Client:
calls: list[str] = []
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("Wiki directory enumeration is forbidden")
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int,
page_size: int,
) -> dict:
self.calls.append(knowledge_base_id)
assert knowledge_base_id == "remote-2"
return {
"pages": [{"slug": "remote-page", "title": "远端页面"}],
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
assert (knowledge_base_id, slug) == ("remote-2", "remote-page")
return {
"slug": slug,
"title": "远端页面",
"content": "远端 needle 正文",
}
async def get_wiki_page(self, *args, **kwargs) -> dict:
raise AssertionError("search must not fetch each page body")
client = Client()
monkeypatch.setattr(tools, "get_local_wiki_index_store", LocalIndex)
monkeypatch.setattr(tools, "get_local_wiki_index_store", LocalIndexMustNotRun)
monkeypatch.setattr(tools, "_client", lambda: client)
payload = json.loads(
await tools._wiki_search_impl(
@@ -1103,87 +1084,148 @@ async def test_wiki_search_selects_local_or_provider_per_mapping(monkeypatch) ->
)
assert payload["ok"] is True
assert {item["knowledge_base"] for item in payload["results"]} == {
"制度库1",
"制度库2",
assert payload["native_regex_supported"] is True
assert payload["result_count"] == 2
assert {call[:2] for call in client.calls} == {
("remote-1", "needle"),
("remote-2", "needle"),
}
assert client.calls == ["remote-2"]
assert payload["channel"] == "mixed"
assert payload["coverage"] == [
{
"knowledge_base": "制度库1",
"channel": "local_wiki_mirror",
"status": "searched",
},
{
"knowledge_base": "制度库2",
"channel": "provider",
"status": "searched",
},
assert client.peak == 2
assert payload["channel"] == "provider_native_regex"
assert [item["status"] for item in payload["coverage"]] == [
"searched",
"searched",
]
@pytest.mark.asyncio
async def test_wiki_search_uses_keyword_index_before_bounded_recent_snapshot(
monkeypatch,
) -> None:
async def test_wiki_search_marks_provider_limit_as_truncated(monkeypatch) -> None:
_patch_scope(monkeypatch)
class LocalIndex:
keyword_calls: list[tuple[list[str], list[str], int]] = []
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 900,
"local_page_count": 900,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
}
]
async def search_pages_by_keywords(
class Client:
async def search_wiki_pages(
self,
mapping_ids: list[str],
tokens: list[str],
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
self.keyword_calls.append((mapping_ids, tokens, limit))
assert (knowledge_base_id, query, limit) == ("remote-1", "needle", 50)
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": "news/old-xinghe",
"title": "较早的国内新闻",
"content_md": "星河工程已经进入联合测试阶段。",
"status": "published",
}
]
async def list_pages(self, mapping_ids: list[str], *, limit: int) -> list[dict]:
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": f"recent/{index}",
"title": f"近期页面 {index}",
"content_md": "无关内容",
"status": "published",
"slug": f"news/{index}",
"title": f"needle 页面 {index}",
"content": "needle 正文",
}
for index in range(limit)
]
class ProviderMustNotRun:
def __init__(self) -> None:
raise AssertionError("complete local Wiki mirror should be used")
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
await tools._wiki_search_impl(
"needle",
{
"knowledge_retrieval_mode": "smart_reasoning",
"llmwiki_knowledge_base_ids": ["mapping-1"],
},
)
)
local = LocalIndex()
monkeypatch.setattr(tools, "get_local_wiki_index_store", lambda: local)
monkeypatch.setattr(tools, "_client", ProviderMustNotRun)
assert payload["scanned_pages"] == 50
assert payload["scan_truncated"] is True
assert payload["result_count"] == tools._MAX_RESULTS
@pytest.mark.asyncio
async def test_wiki_search_handles_supported_and_unsupported_mappings(
monkeypatch,
) -> None:
_patch_scope(monkeypatch, multi=True)
class Client:
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
if knowledge_base_id == "remote-2":
raise WeKnoraError("route unavailable", status_code=404)
return [{"slug": "news/one", "title": f"{query} 页面"}]
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
await tools._wiki_search_impl(
"needle",
{
"knowledge_retrieval_mode": "smart_reasoning",
"llmwiki_knowledge_base_ids": ["mapping-1", "mapping-2"],
},
)
)
assert payload["ok"] is True
assert payload["native_regex_supported"] is True
assert payload["unsupported_knowledge_base_count"] == 1
assert payload["partial"] is False
assert payload["warnings"] == []
assert [item["status"] for item in payload["coverage"]] == [
"searched",
"unsupported",
]
@pytest.mark.asyncio
async def test_wiki_search_without_native_client_method_is_unsupported(
monkeypatch,
) -> None:
_patch_scope(monkeypatch)
class LegacyClient:
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("legacy directory fallback must not run")
async def get_wiki_page(self, *args, **kwargs) -> dict:
raise AssertionError("legacy page fallback must not run")
monkeypatch.setattr(tools, "_client", LegacyClient)
payload = json.loads(
await tools._wiki_search_impl(
"needle",
{
"knowledge_retrieval_mode": "smart_reasoning",
"llmwiki_knowledge_base_ids": ["mapping-1"],
},
)
)
assert payload["ok"] is True
assert payload["native_regex_supported"] is False
assert payload["unsupported_knowledge_base_count"] == 1
assert payload["results"] == []
assert payload["warnings"] == []
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code", [404, 405])
async def test_wiki_search_skips_unsupported_native_regex_without_fallback(
monkeypatch,
status_code: int,
) -> None:
_patch_scope(monkeypatch)
class Client:
async def search_wiki_pages(self, *args, **kwargs) -> list[dict]:
raise WeKnoraError("route unavailable", status_code=status_code)
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("unsupported native search must not enumerate Wiki pages")
async def get_wiki_page(self, *args, **kwargs) -> dict:
raise AssertionError("unsupported native search must not read page bodies")
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
await tools._wiki_search_impl(
"星河工程",
@@ -1195,75 +1237,61 @@ async def test_wiki_search_uses_keyword_index_before_bounded_recent_snapshot(
)
assert payload["ok"] is True
assert payload["result_count"] == 1
assert payload["results"][0]["slug"] == "news/old-xinghe"
assert payload["results"][0]["matched_fields"] == ["content"]
assert local.keyword_calls == [
(["mapping-1"], ["星河工程"], tools._MAX_REGEX_CANDIDATES)
]
assert payload["result_count"] == 0
assert payload["native_regex_supported"] is False
assert payload["unsupported_knowledge_base_count"] == 1
assert payload["partial"] is False
assert payload["warnings"] == []
assert payload["coverage"][0]["status"] == "unsupported"
@pytest.mark.asyncio
async def test_zero_page_mirror_probes_live_wiki_and_supports_markdown_shape(
async def test_native_wiki_search_result_can_be_read_with_markdown_shape(
monkeypatch,
) -> None:
_patch_scope(monkeypatch)
class EmptyCompletedMirror:
class EmptyMirror:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
return []
class Client:
search_calls: list[tuple[str, str, int]] = []
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
self.search_calls.append((knowledge_base_id, query, limit))
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 0,
"local_page_count": 0,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
"slug": "news/xinghe",
"title": "国内科技新闻",
"description": "科技报道汇总",
"markdown": "星河工程完成关键节点验收。",
"status": "published",
}
]
class Client:
queries: list[str] = []
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int,
page_size: int,
query: str = "",
) -> dict:
self.queries.append(query)
return {
"pages": [
{
"slug": "news/xinghe",
"title": "国内科技新闻",
}
],
"total_pages": 1,
}
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise AssertionError("native search must not enumerate the directory")
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
assert (knowledge_base_id, slug) == ("remote-1", "news/xinghe")
return {
"slug": slug,
"title": "国内科技新闻",
"description": "科技报道汇总",
"markdown": "星河工程完成关键节点验收。参见 [[news/验收公告]]。",
"out_links": ["news/承建单位"],
"links": [{"slug": "news/关联政策", "title": "关联政策"}],
"outgoing_links": ["news/旧版关联"],
"status": "published",
}
client = Client()
monkeypatch.setattr(
tools,
"get_local_wiki_index_store",
EmptyCompletedMirror,
)
monkeypatch.setattr(tools, "get_local_wiki_index_store", EmptyMirror)
monkeypatch.setattr(tools, "_client", lambda: client)
payload = json.loads(
@@ -1275,12 +1303,8 @@ async def test_zero_page_mirror_probes_live_wiki_and_supports_markdown_shape(
},
)
)
assert payload["ok"] is True
assert payload["channel"] == "provider"
assert payload["results"][0]["matched_fields"] == ["content"]
assert "星河工程" in client.queries
assert "" in client.queries
assert client.search_calls == [("remote-1", "星河工程", 50)]
read = json.loads(
await tools._wiki_read_impl(
@@ -1293,16 +1317,7 @@ async def test_zero_page_mirror_probes_live_wiki_and_supports_markdown_shape(
)
assert read["results"][0]["content_markdown"] == "星河工程完成关键节点验收。参见 [[news/验收公告]]。"
assert read["results"][0]["summary"] == "科技报道汇总"
assert read["results"][0]["links"] == [
"news/承建单位",
"news/验收公告",
{
"slug": "news/关联政策",
"title": "关联政策",
"direction": "outgoing",
},
"news/旧版关联",
]
assert read["results"][0]["links"] == ["news/承建单位", "news/验收公告"]
@pytest.mark.asyncio
@@ -1452,62 +1467,37 @@ async def test_wiki_read_hides_expansion_links_when_database_setting_is_disabled
@pytest.mark.asyncio
async def test_complete_but_stale_mirror_probes_provider_after_local_zero_match(
async def test_wiki_regex_does_not_use_a_stale_local_mirror(
monkeypatch,
) -> None:
_patch_scope(monkeypatch)
class StaleMirror:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
def __init__(self) -> None:
raise AssertionError("smart Wiki regex must not inspect the local mirror")
class Client:
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
limit: int,
) -> list[dict]:
assert (knowledge_base_id, query, limit) == (
"remote-1",
"星河工程",
50,
)
return [
{
"knowledge_base_mapping_id": "mapping-1",
"state": "idle",
"last_completed_at": "2026-10-14T00:00:00Z",
"last_error": None,
"remote_page_count": 1,
"local_page_count": 1,
"failed_page_count": 0,
"last_sync_result": {"status": "completed"},
}
]
async def search_pages_by_keywords(self, *args, **kwargs) -> list[dict]:
return []
async def list_pages(self, *args, **kwargs) -> list[dict]:
return [
{
"knowledge_base_mapping_id": "mapping-1",
"slug": "news/old",
"title": "旧闻",
"content_md": "镜像中还没有新报道。",
"slug": "news/new",
"title": "新报道",
"content": "星河工程发布最新进展。",
"status": "published",
}
]
class Client:
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int,
page_size: int,
query: str = "",
) -> dict:
return {
"pages": [{"slug": "news/new", "title": "新报道"}],
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
return {
"slug": slug,
"title": "新报道",
"content": "星河工程发布最新进展。",
"status": "published",
}
monkeypatch.setattr(tools, "get_local_wiki_index_store", StaleMirror)
monkeypatch.setattr(tools, "_client", Client)
payload = json.loads(
@@ -1522,7 +1512,7 @@ async def test_complete_but_stale_mirror_probes_provider_after_local_zero_match(
assert payload["result_count"] == 1
assert payload["results"][0]["slug"] == "news/new"
assert payload["channel"] == "local_wiki_mirror_then_provider"
assert payload["channel"] == "provider_native_regex"
@pytest.mark.asyncio
@@ -1531,15 +1521,10 @@ async def test_wiki_provider_all_failures_return_error_and_partial_is_warning(
) -> None:
_patch_scope(monkeypatch, multi=True)
class LocalIndex:
async def list_index_status(self, mapping_ids: list[str]) -> list[dict]:
return []
class AllFailedClient:
async def list_wiki_pages(self, *args, **kwargs) -> dict:
raise WeKnoraError("down")
async def search_wiki_pages(self, *args, **kwargs) -> list[dict]:
raise WeKnoraError("down", status_code=503)
monkeypatch.setattr(tools, "get_local_wiki_index_store", LocalIndex)
monkeypatch.setattr(tools, "_client", AllFailedClient)
failed = json.loads(
await tools._wiki_search_impl(
@@ -1554,22 +1539,16 @@ async def test_wiki_provider_all_failures_return_error_and_partial_is_warning(
assert failed["error"]["code"] == "PROVIDER_UNAVAILABLE"
class PartiallyFailedClient:
async def list_wiki_pages(
async def search_wiki_pages(
self,
knowledge_base_id: str,
query: str,
*,
page: int,
page_size: int,
) -> dict:
limit: int,
) -> list[dict]:
if knowledge_base_id == "remote-2":
raise WeKnoraError("down")
return {
"pages": [{"slug": "ok", "title": "needle"}],
"total_pages": 1,
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict:
return {"slug": slug, "title": "needle", "content": "正文"}
raise WeKnoraError("down", status_code=503)
return [{"slug": "ok", "title": "needle", "content": "正文"}]
monkeypatch.setattr(tools, "_client", PartiallyFailedClient)
partial = json.loads(
@@ -1584,7 +1563,7 @@ async def test_wiki_provider_all_failures_return_error_and_partial_is_warning(
assert partial["ok"] is True
assert partial["partial"] is True
assert partial["warnings"] == [
{"code": "PROVIDER_WIKI_LIST_FAILED", "knowledge_base": "制度库2"}
{"code": "PROVIDER_WIKI_SEARCH_FAILED", "knowledge_base": "制度库2"}
]
@@ -1891,3 +1870,34 @@ def test_regex_execution_has_a_hard_timeout() -> None:
with pytest.raises(tools.SmartKnowledgeToolError) as caught:
tools._regex_search(expression, "a" * 10_000 + "!")
assert caught.value.code == "REGEX_TIMEOUT"
@pytest.mark.asyncio
@pytest.mark.parametrize("assistant_id", ["deep-research-planner", "deep-research-collector", "custom-agent"])
async def test_research_smart_scope_cannot_access_unauthorized_library(monkeypatch, assistant_id):
from app.gateway import llmwiki_rag
request = _request(None)
request.app.state.llmwiki_store = _MappingStore()
monkeypatch.setattr(llmwiki_rag, "get_resolved_llmwiki_runtime", lambda _: SimpleNamespace(weknora_enabled=True))
monkeypatch.setattr(llmwiki_rag, "_agent_bound_ids", lambda _: [])
async def actor(_request):
return "user-1", False
monkeypatch.setattr(llmwiki_rag, "_actor", actor)
with pytest.raises(HTTPException) as caught:
await llmwiki_rag.build_llmwiki_smart_reasoning_context(
request=request,
body=SimpleNamespace(context={"llmwiki_knowledge_base_ids": ["not-authorized"]}, metadata={}, assistant_id=assistant_id),
graph_input={"messages": [HumanMessage(content="test")]},
)
assert caught.value.status_code == 404
@pytest.mark.parametrize("agent_id", ["deep-research-planner", "deep-research-collector"])
def test_builtin_research_tools_use_caller_authorized_scope(agent_id):
assert tools._allowed_agent_ids({"agent_id": agent_id}) is None
from deerflow.tools.builtins.llmwiki_search_tool import _agent_allowed_ids
assert _agent_allowed_ids({"agent_id": agent_id}) is None
@@ -823,6 +823,32 @@ async def test_weknora_client_resolves_default_embedding_model_inside_weknora()
assert created["id"] == "kb-created"
@pytest.mark.asyncio
async def test_weknora_wiki_create_reuses_catalog_models_without_debug_probe() -> None:
requests: list[tuple[str, str, dict[str, object]]] = []
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content) if request.content else {}
requests.append((request.method, request.url.path, payload))
assert request.method == "POST"
assert request.url.path == "/api/v1/knowledge-bases"
return httpx.Response(201, json={"success": True, "data": {"id": "wiki-created", "name": "Wiki"}})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
await client.create_knowledge_base(
name="Wiki",
embedding_model_id="embedding-id",
knowledge_qa_model_id="qa-id",
wiki_enabled=True,
)
assert len(requests) == 1
assert requests[0][2]["embedding_model_id"] == "embedding-id"
assert requests[0][2]["summary_model_id"] == "qa-id"
assert requests[0][2]["wiki_config"]["synthesis_model_id"] == "qa-id"
@pytest.mark.asyncio
async def test_gateway_lists_safe_embedding_models_and_uses_explicit_selection(
monkeypatch: pytest.MonkeyPatch,
@@ -863,6 +889,7 @@ async def test_gateway_lists_safe_embedding_models_and_uses_explicit_selection(
"status": "inactive",
},
{"id": "chat-one", "type": "LLM", "name": "chat"},
{"id": "qa-default", "type": "KnowledgeQA", "name": "wiki-answer", "status": "active", "is_default": True},
]
async def create_knowledge_base(self, **kwargs: object) -> dict[str, object]:
@@ -913,6 +940,7 @@ async def test_gateway_lists_safe_embedding_models_and_uses_explicit_selection(
)
assert create_calls[0]["embedding_model_id"] == "embed-openai"
assert create_calls[0]["knowledge_qa_model_id"] == "qa-default"
assert create_calls[0]["wiki_enabled"] is True
assert create_calls[0]["wiki_content_instructions"] == "按事件时间线生成页面"
assert create_calls[0]["wiki_extraction_instructions"] == "重点提取人物和组织关系"
@@ -1328,6 +1356,44 @@ async def test_weknora_client_imports_url_manual_content_and_lists_wiki_pages()
assert pages["pages"][0]["slug"] == "guide"
@pytest.mark.asyncio
async def test_weknora_client_uses_native_wiki_regex_search_contract() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
assert request.method == "GET"
assert request.url.path == "/api/v1/knowledgebase/kb-one/wiki/search"
assert request.url.params["q"] == "火炬|开幕|闭幕"
# WeKnora v0.7.2 caps the endpoint at 50.
assert request.url.params["limit"] == "50"
return httpx.Response(
200,
json={
"pages": [
{
"slug": "games/timeline",
"title": "赛事时间线",
"content": "火炬传递、开幕和闭幕安排。",
}
]
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
pages = await client.search_wiki_pages(
"kb-one",
"火炬|开幕|闭幕",
limit=500,
)
assert pages == [
{
"slug": "games/timeline",
"title": "赛事时间线",
"content": "火炬传递、开幕和闭幕安排。",
}
]
@pytest.mark.asyncio
async def test_weknora_client_reads_ordered_wiki_index_groups() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
@@ -1661,7 +1727,11 @@ async def test_listing_conversation_deposit_refreshes_remote_description(monkeyp
monkeypatch.setattr(llmwiki_router, "_store", lambda _: store)
monkeypatch.setattr(llmwiki_router, "_client_or_503", lambda: Client())
payload = await llmwiki_router.list_knowledge_bases(object(), scope="public")
payload = await llmwiki_router.list_knowledge_bases(
object(),
scope="public",
refresh=True,
)
assert payload["knowledge_bases"][0]["description"] == llmwiki_deposit.CONVERSATION_DEPOSIT_KB_DESCRIPTION
assert "DeerFlow" not in json.dumps(payload, ensure_ascii=False)
@@ -1691,6 +1761,47 @@ def test_conversation_deposit_history_hides_legacy_brand() -> None:
}
@pytest.mark.asyncio
async def test_agent_bound_knowledge_base_list_uses_local_mapping_only(monkeypatch: pytest.MonkeyPatch) -> None:
from types import SimpleNamespace
from app.gateway.routers import llmwiki as llmwiki_router
from deerflow.persistence.agents import MemoryAgentStore
from deerflow.persistence.llmwiki import MemoryLlmWikiStore
store = MemoryLlmWikiStore()
mapping = await store.create_mapping(
weknora_id="remote-agent-kb",
owner_user_id="user-a",
name="本地目录缓存",
description="",
kb_type="document",
)
agent_store = MemoryAgentStore()
await agent_store.ensure_builtin({"id": "agent-a", "name": "Agent A", "published": True})
async def actor(_: object) -> tuple[str, bool]:
return "user-a", False
monkeypatch.setattr(llmwiki_router, "_actor", actor)
monkeypatch.setattr(llmwiki_router, "_store", lambda _: store)
monkeypatch.setattr(llmwiki_router, "get_agent_store", lambda _: agent_store)
monkeypatch.setattr(
llmwiki_router,
"load_agent_config",
lambda _agent_id: SimpleNamespace(llmwiki_knowledge_base_ids=[mapping["id"]]),
)
monkeypatch.setattr(
llmwiki_router,
"_client_or_503",
lambda: (_ for _ in ()).throw(AssertionError("normal listing must not call WeKnora")),
)
payload = await llmwiki_router.list_agent_knowledge_bases(object(), "agent-a")
assert [item["id"] for item in payload["knowledge_bases"]] == [mapping["id"]]
assert payload["knowledge_bases"][0]["remote_status"] == "cached"
@pytest.mark.asyncio
async def test_list_knowledge_bases_personal_scope_filters_to_actor_for_admin(monkeypatch: pytest.MonkeyPatch) -> None:
from app.gateway.routers import llmwiki as llmwiki_router
@@ -657,10 +657,66 @@ async def test_auto_sync_scheduler_refreshes_provider_between_syncs(monkeypatch)
assert syncs == ["openai", "local"]
def test_local_wiki_index_defaults_to_automatic_sync() -> None:
@pytest.mark.asyncio
async def test_disabled_auto_sync_keeps_scheduler_local_only(monkeypatch) -> None:
now = 0.0
refreshes: list[float] = []
syncs: list[float] = []
remote_maintenance_calls: list[str] = []
config = SimpleNamespace(
enabled=True,
auto_sync=False,
sync_interval_seconds=25,
)
async def fake_refresh(_app):
refreshes.append(now)
return config
async def fake_sleep(seconds: float) -> None:
nonlocal now
now += seconds
if len(refreshes) >= 3:
raise asyncio.CancelledError
async def fake_sync_once(_app, *, config=None) -> None:
syncs.append(now)
async def fake_remote_maintenance(_app) -> None:
remote_maintenance_calls.append("called")
monkeypatch.setattr(llmwiki_index_scheduler, "monotonic", lambda: now)
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
monkeypatch.setattr(
llmwiki_index_scheduler,
"_upgrade_wiki_generation_configs_once",
fake_remote_maintenance,
)
monkeypatch.setattr(
llmwiki_index_scheduler,
"_refresh_local_wiki_config",
fake_refresh,
)
app = SimpleNamespace(
state=SimpleNamespace(
llmwiki_sync_service=object(),
llmwiki_store=object(),
)
)
with pytest.raises(asyncio.CancelledError):
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
assert refreshes == [0.0, 10.0, 20.0]
assert syncs == []
assert remote_maintenance_calls == []
def test_local_wiki_index_defaults_to_manual_sync() -> None:
config = LocalWikiIndexConfig()
assert config.auto_sync is True
assert config.auto_sync is False
assert config.sync_interval_seconds == 30
@@ -0,0 +1,157 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "manila-times-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_manila_times_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"manila_times_page_renderer", SCRIPT_PATH
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "manila-times-page-generator"
assert "马尼拉时报" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["manila-times-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["manila-times-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/manila-times-page-generator/SKILL.md" in lead_prompt_source
assert "专用马尼拉时报路由" in lead_prompt_source
def test_default_json_renders_editable_manila_times_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
counts = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "manila-times-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "manila-times-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert counts["lead_stories"] == 3
assert counts["region_tabs"] == 4
assert counts["videos"] == 3
assert counts["edition_pages"] == 4
assert "__MANILA_TIMES_PAGE_DATA__" not in html
assert "__MANILA_TIMES_PAGE_RUNTIME__" not in html
assert 'id="manilaTimesPageData"' in html
assert 'id="leadMain"' in html
assert 'id="regionTabs"' in html
assert 'id="videoOverlay"' in html
assert 'id="editionOverlay"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "has-external-media" in html
assert "data-media-path" in html
assert "exactPathTarget" in html
assert '[data-visual].has-external-media::before' in html
assert "readAsDataURL" in html
assert '"brandPrefix":"马尼拉"' in html
assert '"language":"zh-CN"' in html
assert "JSON 编辑器" in html
assert '"manila-times-page-data-v1:" + fingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "localStorage.getItem('harbor-theme')" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
output = tmp_path / "manila-times-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_visual_remote_url_and_english_copy() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
data.pop("footer")
data["regions"]["tabs"][1]["id"] = data["regions"]["tabs"][0]["id"]
data["media"]["watch"]["videos"][0]["visual"] = "remote-image"
data["site"]["description"] = "https://example.com/remote"
data["lead"]["main"]["title"] = "A faster capital begins with connections"
data["site"]["language"] = "en"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "missing top-level module(s): footer" in message
assert "duplicates" in message
assert "visual must be one of" in message
assert "remote URLs" in message
assert "site.language must be zh-CN" in message
assert "lead.main.title must use Simplified Chinese reader-facing copy" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/manila.jpg", "alt": "马尼拉新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/manila.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "报道原文", "embed": True},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["lead"]["main"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "manila-times-runtime.js").read_text(encoding="utf-8")
assert "externalMedia" in runtime
assert 'sandbox="allow-forms allow-popups allow-scripts"' in runtime
@@ -157,11 +157,12 @@ def test_duplicate_historical_stamps_reconcile_to_single_head(
inspector = sa.inspect(connection)
assert connection.execute(
sa.text("SELECT version_num FROM alembic_version")
).scalar_one() == "20261020_01"
).scalar_one() == "20261102_02"
assert inspector.has_table("task_deeplink_threads")
assert connection.execute(
sa.text("SELECT COUNT(*) FROM task_deeplink_threads")
).scalar_one() == 0
assert inspector.has_table("task_deeplink_details")
assert inspector.has_table("llmwiki_wiki_edges")
assert _EDGE_INDEXES <= {
@@ -182,6 +183,7 @@ def test_duplicate_historical_stamps_reconcile_to_single_head(
assert {
"model_name",
"supports_reasoning_effort",
"default_reasoning_effort",
"updated_by",
"updated_at",
} == capability_columns.keys()
@@ -284,7 +286,7 @@ def test_conflicting_20261019_feature_stamp_reconciles_task_schema(
inspector = sa.inspect(connection)
assert connection.execute(
sa.text("SELECT version_num FROM alembic_version")
).scalar_one() == "20261020_01"
).scalar_one() == "20261102_02"
retrieval_columns = [
str(item["name"])
@@ -299,11 +301,22 @@ def test_conflicting_20261019_feature_stamp_reconciles_task_schema(
).scalar_one() in (1, True)
assert inspector.has_table("task_deeplink_threads")
task_columns = {
str(item["name"]) for item in inspector.get_columns("task_deeplink_threads")
}
assert {"sub_type", "display_name", "status", "metadata_json"} <= task_columns
task_indexes = {
str(item["name"]): [str(column) for column in item["column_names"]]
for item in inspector.get_indexes("task_deeplink_threads")
}
assert task_indexes["ix_task_deeplink_threads_scope"] == [
"task_id",
"go_path",
"sub_type",
"updated_at",
"thread_id",
]
assert task_indexes["ix_task_deeplink_threads_task_go_updated"] == [
"task_id",
"go_path",
"updated_at",
@@ -312,6 +325,136 @@ def test_conflicting_20261019_feature_stamp_reconciles_task_schema(
engine.dispose()
def test_broken_two_column_detail_key_is_repaired_at_head(tmp_path: Path) -> None:
"""20261022_02 的中间版本只加了列没换成三列主键(batch API 不存在)。
这类库 alembic_version 已到 head 但键仍是两列,typed upsert 必炸
UNIQUE 约束。兜底修复迁移应把键换成三列且保留旧数据。用最小库直接
从坏状态升级(不跑全链,链上其它迁移依赖 create_all 的表)。
"""
database = tmp_path / "broken-detail-key.db"
engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}")
try:
with engine.begin() as connection:
connection.exec_driver_sql(
"CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL PRIMARY KEY)"
)
connection.exec_driver_sql(
"INSERT INTO alembic_version (version_num) VALUES ('20261023_02')"
)
# 「半执行」现场:sub_type 列存在,主键仍是两列,有一行旧快照。
connection.exec_driver_sql(
"CREATE TABLE task_deeplink_details ("
"detail_type VARCHAR(32) NOT NULL, "
"detail_id VARCHAR(128) NOT NULL, "
"task_name VARCHAR(512) NOT NULL, "
"task_content TEXT NOT NULL, "
"overview TEXT NOT NULL, "
"extra_json JSON NOT NULL, "
"version INTEGER NOT NULL, "
"created_by VARCHAR(64), "
"updated_by VARCHAR(64), "
"created_at DATETIME NOT NULL, "
"updated_at DATETIME NOT NULL, "
"sub_type VARCHAR(64) NOT NULL DEFAULT '', "
"PRIMARY KEY (detail_type, detail_id))"
)
connection.exec_driver_sql(
"INSERT INTO task_deeplink_details "
"(detail_type, detail_id, task_name, task_content, overview, "
"extra_json, version, created_at, updated_at, sub_type) VALUES "
"('b5-1', '001', '旧任务', '旧详情', '', '{}', 1, "
"CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, '')"
)
finally:
engine.dispose()
command.upgrade(_config(database), "head")
engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}")
try:
with engine.connect() as connection:
inspector = sa.inspect(connection)
pk = [
str(column)
for column in inspector.get_pk_constraint("task_deeplink_details")[
"constrained_columns"
]
]
assert pk == ["detail_type", "detail_id", "sub_type"]
assert connection.execute(
sa.text(
"SELECT detail_type, detail_id, sub_type, task_name "
"FROM task_deeplink_details"
)
).fetchall() == [("b5-1", "001", "", "旧任务")]
finally:
engine.dispose()
def test_custom_business_mapping_migration_preserves_existing_rows(tmp_path: Path) -> None:
"""The new type columns and widened history key upgrade a populated install."""
database = tmp_path / "custom-business-mapping.db"
engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}")
try:
with engine.begin() as connection:
connection.exec_driver_sql(
"CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL PRIMARY KEY)"
)
connection.exec_driver_sql(
"INSERT INTO alembic_version (version_num) VALUES ('20261102_01')"
)
connection.exec_driver_sql(
"CREATE TABLE business_mapping ("
"business_code VARCHAR(32) PRIMARY KEY, label VARCHAR(128) NOT NULL, "
"agent_id VARCHAR(64), agent_name VARCHAR(191), chain_id VARCHAR(64), "
"chain_title VARCHAR(512), sort_order INTEGER NOT NULL, "
"created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL)"
)
connection.exec_driver_sql(
"INSERT INTO business_mapping "
"(business_code, label, sort_order, created_at, updated_at) VALUES "
"('3Q', '3QFX', 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
)
connection.exec_driver_sql(
"CREATE TABLE task_deeplink_threads ("
"thread_id VARCHAR(64) PRIMARY KEY, task_id VARCHAR(128) NOT NULL, "
"source_task_id VARCHAR(128) NOT NULL, go_path VARCHAR(16) NOT NULL, "
"sub_type VARCHAR(64) NOT NULL DEFAULT '', agent_id VARCHAR(128), "
"display_name VARCHAR(256), status VARCHAR(20) NOT NULL DEFAULT 'idle', "
"metadata_json JSON, created_by VARCHAR(64), "
"created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL)"
)
connection.exec_driver_sql(
"INSERT INTO task_deeplink_threads "
"(thread_id, task_id, source_task_id, go_path, created_at, updated_at) VALUES "
"('thread-1', 'task-1', 'task-1', '3qfx', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
)
command.upgrade(_config(database), "head")
with engine.connect() as connection:
inspector = sa.inspect(connection)
mapping_columns = {
str(column["name"]): column for column in inspector.get_columns("business_mapping")
}
assert {"mapping_type", "is_custom"} <= mapping_columns.keys()
assert connection.execute(
sa.text(
"SELECT mapping_type, is_custom FROM business_mapping "
"WHERE business_code = '3Q'"
)
).one() == ("内置业务", 0)
assert connection.execute(
sa.text("SELECT go_path FROM task_deeplink_threads WHERE thread_id = 'thread-1'")
).scalar_one() == "3qfx"
assert connection.execute(
sa.text("SELECT version_num FROM alembic_version")
).scalar_one() == "20261102_02"
finally:
engine.dispose()
async def _exercise_parent_directories(store) -> str:
configured = ["专题资料", "政策法规"]
directory_paths = [["专题资料"], ["专题资料", "国内政策"]]
@@ -14,7 +14,21 @@ def test_migration_graph_has_unique_revisions_and_one_head() -> None:
revisions = list(scripts.walk_revisions())
assert len({revision.revision for revision in revisions}) == len(revisions)
assert scripts.get_heads() == ["20261020_01"]
assert scripts.get_heads() == ["20261102_02"]
assert scripts.get_revision("20261026_02").down_revision == ("20261026_01", "20261022_01")
assert scripts.get_revision("20261026_01").down_revision == "20261025_03"
assert scripts.get_revision("20261023_03").down_revision == "20261023_02"
assert scripts.get_revision("20261023_02").down_revision == (
"20261024_02",
"20261022_02",
)
assert scripts.get_revision("20261024_02").down_revision == "20261024_01"
assert scripts.get_revision("20261022_02").down_revision == "20261024_01"
assert scripts.get_revision("20261024_01").down_revision == "20261023_01"
assert scripts.get_revision("20261023_01").down_revision == "20261022_03"
assert scripts.get_revision("20261022_03").down_revision == "20261021_01"
assert scripts.get_revision("20261022_01").down_revision == "20261014_02"
assert scripts.get_revision("20261021_01").down_revision == "20261020_01"
assert scripts.get_revision("20261020_01").down_revision == (
"20261019_01",
"20261014_02",
@@ -2,6 +2,8 @@
from __future__ import annotations
import errno
from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -26,6 +28,80 @@ from deerflow.persistence.model_capabilities import (
)
def test_config_atomic_replace_retries_transient_busy_filesystem_error(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
original = "models:\n - name: old\n"
updated = "models:\n - name: new\n"
config_path.write_text(original, encoding="utf-8")
real_replace = router.os.replace
attempts = 0
def replace_with_transient_busy(source: str, target: Path) -> None:
nonlocal attempts
attempts += 1
if attempts < 3:
raise OSError(errno.EBUSY, "resource busy", str(target))
real_replace(source, target)
monkeypatch.setattr(router.os, "replace", replace_with_transient_busy)
monkeypatch.setattr(router, "sleep", lambda _delay: None)
backup = router._backup_and_write_config(config_path, original, updated)
assert attempts == 3
assert config_path.read_text(encoding="utf-8") == updated
assert backup.read_text(encoding="utf-8") == original
def test_config_atomic_replace_does_not_retry_permanent_error(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
original = "models:\n - name: old\n"
config_path.write_text(original, encoding="utf-8")
attempts = 0
def fail_replace(_source: str, _target: Path) -> None:
nonlocal attempts
attempts += 1
raise OSError(errno.EACCES, "permission denied")
monkeypatch.setattr(router.os, "replace", fail_replace)
with pytest.raises(OSError, match="permission denied"):
router._backup_and_write_config(config_path, original, "models:\n - name: new\n")
assert attempts == 1
assert config_path.read_text(encoding="utf-8") == original
assert not list(tmp_path.glob(".config.yaml.*.tmp"))
def test_config_single_file_mount_is_updated_without_atomic_rename(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
original = "models:\n - name: old\n"
updated = "models:\n - name: mounted-new\n"
config_path.write_text(original, encoding="utf-8")
monkeypatch.setattr(router, "_is_file_mountpoint", lambda _path: True)
def unexpected_replace(_source: str, _target: Path) -> None:
pytest.fail("single-file bind mount must not be replaced by rename")
monkeypatch.setattr(router.os, "replace", unexpected_replace)
backup = router._backup_and_write_config(config_path, original, updated)
assert config_path.read_text(encoding="utf-8") == updated
assert backup.read_text(encoding="utf-8") == original
assert not list(tmp_path.glob(".config.yaml.*.tmp"))
def test_replace_models_block_keeps_other_sections_and_comments() -> None:
raw = """config_version: 1
models:
@@ -139,6 +215,260 @@ def test_reasoning_capability_route_precedes_generic_model_update_and_accepts_sl
]
def test_write_file_transport_route_precedes_generic_model_update() -> None:
scope = {
"type": "http",
"method": "PUT",
"path": "/api/admin/models/write-file-transport/zai-org/GLM-5-FP8",
}
matching_paths = [route.path for route in router.router.routes if route.matches(scope)[0] is Match.FULL]
assert matching_paths[:2] == [
"/api/admin/models/write-file-transport/{model_name:path}",
"/api/admin/models/{model_name:path}",
]
def _admin_request():
return SimpleNamespace(
state=SimpleNamespace(user=SimpleNamespace(system_role="admin")),
app=SimpleNamespace(state=SimpleNamespace()),
)
@pytest.mark.asyncio
async def test_write_file_transport_toggle_persists_native_and_buffered_gateway(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
config_path.write_text(
"""models:
- name: flash
use: langchain_openai:ChatOpenAI
model: flash
""",
encoding="utf-8",
)
saved: dict[str, object] = {}
async def save_models(_request, raw_before: str, models: list[dict[str, object]]) -> list[dict[str, object]]:
saved["raw_before"] = raw_before
saved["models"] = models
return models
monkeypatch.setattr(router, "_resolve_config_path", lambda: config_path)
monkeypatch.setattr(router, "_save_models", save_models)
enabled = await router.update_write_file_transport(
"flash",
router.WriteFileTransportToggleRequest(write_file_transport="buffered_gateway"),
_admin_request(),
)
assert enabled.write_file_transport == "buffered_gateway"
assert saved["models"] == [
{
"name": "flash",
"use": "langchain_openai:ChatOpenAI",
"model": "flash",
"write_file_transport": "buffered_gateway",
}
]
disabled = await router.update_write_file_transport(
"flash",
router.WriteFileTransportToggleRequest(write_file_transport="native"),
_admin_request(),
)
assert disabled.write_file_transport == "native"
assert saved["models"][0]["write_file_transport"] == "native"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("use", "use_responses_api"),
[
("langchain_anthropic:ChatAnthropic", None),
("langchain_openai:ChatOpenAI", True),
],
)
async def test_write_file_transport_rejects_incompatible_models(
use: str,
use_responses_api: bool | None,
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
original = f"""models:
- name: incompatible
use: {use}
model: incompatible
"""
if use_responses_api:
original = original.replace(
" model: incompatible\n",
" model: incompatible\n use_responses_api: true\n",
)
config_path.write_text(original, encoding="utf-8")
monkeypatch.setattr(router, "_resolve_config_path", lambda: config_path)
async def fail_save_models(_request, _raw_before, _models):
raise AssertionError("incompatible models must not be persisted")
monkeypatch.setattr(router, "_save_models", fail_save_models)
with pytest.raises(HTTPException) as exc_info:
await router.update_write_file_transport(
"incompatible",
router.WriteFileTransportToggleRequest(write_file_transport="buffered_gateway"),
_admin_request(),
)
assert exc_info.value.status_code == 422
assert config_path.read_text(encoding="utf-8") == original
def test_strict_stream_integrity_route_precedes_generic_model_update() -> None:
scope = {
"type": "http",
"method": "PUT",
"path": "/api/admin/models/strict-stream-integrity/zai-org/GLM-5-FP8",
}
matching_paths = [route.path for route in router.router.routes if route.matches(scope)[0] is Match.FULL]
assert matching_paths[:2] == [
"/api/admin/models/strict-stream-integrity/{model_name:path}",
"/api/admin/models/{model_name:path}",
]
@pytest.mark.asyncio
async def test_strict_stream_integrity_toggle_persists_explicit_bool(
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
config_path.write_text(
"""models:
- name: flash
use: langchain_openai:ChatOpenAI
model: flash
""",
encoding="utf-8",
)
saved: dict[str, object] = {}
async def save_models(_request, raw_before: str, models: list[dict[str, object]]) -> list[dict[str, object]]:
saved["models"] = models
return models
monkeypatch.setattr(router, "_resolve_config_path", lambda: config_path)
monkeypatch.setattr(router, "_save_models", save_models)
enabled = await router.update_strict_stream_integrity(
"flash",
router.StrictStreamIntegrityToggleRequest(strict_stream_integrity=True),
_admin_request(),
)
assert enabled.strict_stream_integrity is True
assert saved["models"] == [
{
"name": "flash",
"use": "langchain_openai:ChatOpenAI",
"model": "flash",
"strict_stream_integrity": True,
}
]
disabled = await router.update_strict_stream_integrity(
"flash",
router.StrictStreamIntegrityToggleRequest(strict_stream_integrity=False),
_admin_request(),
)
assert disabled.strict_stream_integrity is False
assert saved["models"][0]["strict_stream_integrity"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
("use", "use_responses_api"),
[
("langchain_anthropic:ChatAnthropic", None),
("langchain_openai:ChatOpenAI", True),
],
)
async def test_strict_stream_integrity_rejects_incompatible_models(
use: str,
use_responses_api: bool | None,
tmp_path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
config_path = tmp_path / "config.yaml"
original = f"""models:
- name: incompatible
use: {use}
model: incompatible
"""
if use_responses_api:
original = original.replace(
" model: incompatible\n",
" model: incompatible\n use_responses_api: true\n",
)
config_path.write_text(original, encoding="utf-8")
monkeypatch.setattr(router, "_resolve_config_path", lambda: config_path)
async def fail_save_models(_request, _raw_before, _models):
raise AssertionError("incompatible models must not be persisted")
monkeypatch.setattr(router, "_save_models", fail_save_models)
with pytest.raises(HTTPException) as exc_info:
await router.update_strict_stream_integrity(
"incompatible",
router.StrictStreamIntegrityToggleRequest(strict_stream_integrity=True),
_admin_request(),
)
assert exc_info.value.status_code == 422
assert config_path.read_text(encoding="utf-8") == original
def test_legacy_model_edit_preserves_strict_stream_integrity() -> None:
existing = {
"name": "relay-model",
"use": "langchain_openai:ChatOpenAI",
"model": "relay-model",
"strict_stream_integrity": True,
}
merged = router._request_to_raw_model(
router.ModelUpsertRequest(
name="relay-model",
use="langchain_openai:ChatOpenAI",
model="relay-model-v2",
),
existing,
)
assert merged["model"] == "relay-model-v2"
assert merged["strict_stream_integrity"] is True
changed = router._request_to_raw_model(
router.ModelUpsertRequest(
name="relay-model",
use="langchain_openai:ChatOpenAI",
model="relay-model-v2",
strict_stream_integrity=False,
),
merged,
)
assert changed["strict_stream_integrity"] is False
def _reasoning_request(store, *, config: AppConfig | None = None):
return SimpleNamespace(
state=SimpleNamespace(
@@ -272,15 +602,21 @@ async def test_reasoning_capability_endpoint_and_model_apis_use_database_not_yam
updated = await router.update_reasoning_effort_capability(
"zai-org/GLM-5-FP8",
router.ReasoningEffortCapabilityRequest(supports_reasoning_effort=True),
router.ReasoningEffortCapabilityRequest(
supports_reasoning_effort=True,
default_reasoning_effort="low",
),
request,
)
admin_list = await router.list_model_configurations(request)
public_list = await public_models.list_models(request, config)
assert updated.supports_reasoning_effort is True
assert updated.default_reasoning_effort == "low"
assert admin_list.models[0].supports_reasoning_effort is True
assert public_list.models[0].supports_reasoning_effort is True
assert public_list.models[0].default_reasoning_effort == "low"
assert (await store.get_capability("zai-org/GLM-5-FP8"))["default_reasoning_effort"] == "low"
assert config_path.read_text(encoding="utf-8") == original
@@ -605,6 +941,7 @@ def test_final_reasoning_policy_overrides_both_preseeded_runtime_containers() ->
config,
capability_enabled=True,
model_supports_thinking=True,
default_reasoning_effort="low",
)
assert normalized == "low"
@@ -661,7 +998,7 @@ def test_reasoning_policy_strips_effort_when_selection_is_not_available(
assert "reasoning_effort" not in config["context"]
def test_reasoning_policy_rejects_unknown_effort() -> None:
def test_reasoning_policy_uses_global_default_instead_of_stale_thread_value() -> None:
config = {
"configurable": {},
"context": {
@@ -670,14 +1007,15 @@ def test_reasoning_policy_rejects_unknown_effort() -> None:
},
}
with pytest.raises(HTTPException) as exc_info:
services.enforce_reasoning_effort_run_config(
config,
capability_enabled=True,
model_supports_thinking=True,
)
assert exc_info.value.status_code == 422
normalized = services.enforce_reasoning_effort_run_config(
config,
capability_enabled=True,
model_supports_thinking=True,
default_reasoning_effort="low",
)
assert normalized == "low"
assert config["configurable"]["reasoning_effort"] == "low"
assert config["context"]["reasoning_effort"] == "low"
@pytest.mark.asyncio
@@ -692,7 +1030,7 @@ def test_reasoning_policy_rejects_unknown_effort() -> None:
"reasoning_effort": "minimal",
},
"qwen-a",
"low",
"medium",
),
(
{
@@ -732,6 +1070,7 @@ async def test_start_run_applies_database_reasoning_policy_after_final_context_m
await store.set_reasoning_effort_support(
"qwen-a",
supports_reasoning_effort=True,
default_reasoning_effort="medium",
)
class ThreadStore:
@@ -764,6 +1103,9 @@ async def test_start_run_applies_database_reasoning_policy_after_final_context_m
async def set_status(self, *_args, **_kwargs):
return None
def watch_completion(self, _record):
return None
run_context = SimpleNamespace(
app_config=app_config,
thread_store=ThreadStore(),
@@ -0,0 +1,58 @@
"""Regression tests for self-contained API documentation pages."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from fastapi.testclient import TestClient
from app.gateway import config as gateway_config
from app.gateway.app import create_app
def test_config_yaml_docs_switch_overrides_stale_legacy_environment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GATEWAY_ENABLE_DOCS", "false")
monkeypatch.setattr(gateway_config, "_gateway_config", None)
monkeypatch.setattr(
gateway_config,
"get_app_config",
lambda: SimpleNamespace(gateway=SimpleNamespace(enable_docs=True)),
)
assert gateway_config.get_gateway_config().enable_docs is True
def test_docs_and_redoc_use_vendored_assets_behind_proxy_prefix(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Docs must not need internet access and must retain a reverse-proxy prefix."""
monkeypatch.setenv("GATEWAY_ENABLE_DOCS", "true")
monkeypatch.setattr(gateway_config, "_gateway_config", None)
app = create_app()
with TestClient(app, root_path="/deerflow") as client:
swagger = client.get("/deerflow/docs")
redoc = client.get("/deerflow/redoc")
swagger_bundle = client.get("/deerflow/docs/static/swagger-ui-bundle.js")
redoc_bundle = client.get("/deerflow/docs/static/redoc.standalone.js")
openapi = client.get("/deerflow/openapi.json")
assert swagger.status_code == 200
assert "./docs/static/swagger-ui-bundle.js" in swagger.text
assert "./docs/static/swagger-ui.css" in swagger.text
assert "url: './openapi.json'" in swagger.text
assert "https://" not in swagger.text
assert redoc.status_code == 200
assert "./docs/static/redoc.standalone.js" in redoc.text
assert "./openapi.json" in redoc.text
assert "https://" not in redoc.text
assert swagger_bundle.status_code == 200
assert swagger_bundle.headers["content-type"].startswith("text/javascript")
assert redoc_bundle.status_code == 200
assert redoc_bundle.headers["content-type"].startswith("text/javascript")
assert openapi.status_code == 200
@@ -1648,32 +1648,32 @@ class _FailingCheckpointer:
@pytest.mark.asyncio
async def test_direct_thread_creation_cleans_failed_checkpoint(monkeypatch):
async def test_direct_thread_creation_defers_unavailable_checkpoint(monkeypatch):
store = _ThreadStoreProbe()
checkpointer = _FailingCheckpointer(RuntimeError("checkpoint unavailable"))
monkeypatch.setattr(gateway_deps, "get_thread_store", lambda request: store)
monkeypatch.setattr(gateway_deps, "get_checkpointer", lambda request: checkpointer)
with pytest.raises(HTTPException) as exc_info:
await multi_agent._create_thread_direct(
SimpleNamespace(),
metadata={"thread_type": "roundtable"},
)
thread_id = await multi_agent._create_thread_direct(
SimpleNamespace(), metadata={"thread_type": "roundtable"},
)
assert exc_info.value.status_code == 500
assert len(store.created) == 1
thread_id, create_kwargs = store.created[0]
created_id, create_kwargs = store.created[0]
assert created_id == thread_id
assert create_kwargs["metadata"] == {"thread_type": "roundtable"}
assert store.deleted == [thread_id]
assert checkpointer.deleted == [thread_id]
assert store.deleted == []
assert checkpointer.deleted == []
@pytest.mark.asyncio
async def test_direct_thread_creation_cleans_metadata_when_cancelled(monkeypatch):
async def test_direct_thread_creation_propagates_metadata_cancellation(monkeypatch):
store = _ThreadStoreProbe()
checkpointer = _FailingCheckpointer(asyncio.CancelledError())
async def cancelled_create(*args, **kwargs):
raise asyncio.CancelledError()
monkeypatch.setattr(store, "create", cancelled_create)
monkeypatch.setattr(gateway_deps, "get_thread_store", lambda request: store)
monkeypatch.setattr(gateway_deps, "get_checkpointer", lambda request: checkpointer)
with pytest.raises(asyncio.CancelledError):
await multi_agent._create_thread_direct(
@@ -1681,6 +1681,4 @@ async def test_direct_thread_creation_cleans_metadata_when_cancelled(monkeypatch
metadata={"thread_type": "roundtable"},
)
thread_id, _ = store.created[0]
assert store.deleted == [thread_id]
assert checkpointer.deleted == [thread_id]
assert store.created == []
@@ -14,6 +14,7 @@ from app.gateway.routers import system_settings as ss_router
from app.gateway.services import _prepend_prompt_prefix, _prompt_prefix_opted_in
from deerflow.agents.lead_agent import prompt as prompt_module
from deerflow.config import system_settings as ss
from deerflow.sandbox.prompt_policy import get_agent_runtime_policy, get_python_dependency_policy
@pytest.fixture
@@ -36,6 +37,8 @@ def test_load_returns_defaults_when_missing(tmp_runtime_home: Path) -> None:
assert settings.citation_display.reference_mode_enabled is False
assert settings.citation_display.cleanup_words == []
assert settings.memory.enabled is False
assert settings.conversation_context.recent_turn_window_enabled is False
assert settings.conversation_context.max_conversation_rounds == 20
assert settings.write_file_compatibility.enabled is False
assert settings.conversation_context.recent_turn_window_enabled is False
@@ -94,6 +97,69 @@ def _patch_admin(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ss_router, "get_optional_user_from_request", _optional)
@pytest.mark.asyncio
async def test_dependency_delete_prompt_toggle_roundtrip(tmp_runtime_home, monkeypatch):
_patch_admin(monkeypatch)
assert get_python_dependency_policy() == ""
for enabled in (True, False):
response = await ss_router.update_prompt_prefix(
ss_router.PromptPrefixUpdateRequest(python_dependency_delete_protection_enabled=enabled),
SimpleNamespace(),
)
assert response.python_dependency_delete_protection_enabled is enabled
assert ss.load_system_settings().prompt_prefix.python_dependency_delete_protection_enabled is enabled
assert response.enabled is False # Independent of the first-question prefix switch.
policy = get_python_dependency_policy()
assert bool(policy) is enabled
if enabled:
assert "允许安装、升级、降级" in policy
assert "允许包管理器" in policy
assert "不得执行独立的" in policy
@pytest.mark.asyncio
@pytest.mark.parametrize("field", ["python_dependency_delete_protection_enabled", "image_generation_unavailable_hint_enabled"])
async def test_nonadmin_cannot_enable_runtime_prompt(tmp_runtime_home, monkeypatch, field):
async def ordinary_user(_request):
return SimpleNamespace(system_role="user")
monkeypatch.setattr(ss_router, "get_optional_user_from_request", ordinary_user)
with pytest.raises(HTTPException) as error:
await ss_router.update_prompt_prefix(
ss_router.PromptPrefixUpdateRequest(**{field: True}),
SimpleNamespace(),
)
assert error.value.status_code == 403
assert get_python_dependency_policy() == ""
assert get_agent_runtime_policy() == ""
@pytest.mark.asyncio
async def test_image_generation_hint_independent_toggle(tmp_runtime_home, monkeypatch):
_patch_admin(monkeypatch)
assert get_agent_runtime_policy() == ""
await ss_router.update_prompt_prefix(
ss_router.PromptPrefixUpdateRequest(python_dependency_delete_protection_enabled=True),
SimpleNamespace(),
)
for enabled in (True, False):
response = await ss_router.update_prompt_prefix(
ss_router.PromptPrefixUpdateRequest(image_generation_unavailable_hint_enabled=enabled),
SimpleNamespace(),
)
assert response.image_generation_unavailable_hint_enabled is enabled
assert response.python_dependency_delete_protection_enabled is True
assert response.enabled is False
assert ss.load_system_settings().prompt_prefix.image_generation_unavailable_hint_enabled is enabled
policy = get_agent_runtime_policy()
assert "<python_dependency_boundary>" in policy
assert ("<image_generation_unavailable>" in policy) is enabled
if enabled:
assert "当前服务暂不支持生成或编辑图片" in policy
assert "此限制不影响" in policy
assert "绘图提示词" in policy
@pytest.mark.asyncio
async def test_prompt_settings_update_can_enable_markdown_format_without_changing_prefix(
tmp_runtime_home: Path,
@@ -166,28 +232,6 @@ async def test_admin_can_enable_memory_runtime_switch(
assert ss.load_system_settings().memory.enabled is True
@pytest.mark.asyncio
async def test_admin_can_toggle_write_file_compatibility_without_restart(
tmp_runtime_home: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_patch_admin(monkeypatch)
enabled = await ss_router.update_write_file_compatibility_setting(
ss_router.WriteFileCompatibilityUpdateRequest(enabled=True),
SimpleNamespace(),
)
assert enabled.enabled is True
assert ss.get_write_file_compatibility_settings().enabled is True
disabled = await ss_router.update_write_file_compatibility_setting(
ss_router.WriteFileCompatibilityUpdateRequest(enabled=False),
SimpleNamespace(),
)
assert disabled.enabled is False
assert ss.get_write_file_compatibility_settings().enabled is False
@pytest.mark.asyncio
async def test_admin_can_enable_recent_turn_context_window(
tmp_runtime_home: Path,
@@ -202,9 +246,42 @@ async def test_admin_can_enable_recent_turn_context_window(
assert response.recent_turn_window_enabled is True
assert response.recent_turns == 3
assert response.max_conversation_rounds == 20
assert ss.load_system_settings().conversation_context.recent_turn_window_enabled is True
@pytest.mark.asyncio
async def test_admin_can_update_conversation_round_limit_without_changing_window(
tmp_runtime_home: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_patch_admin(monkeypatch)
ss.save_system_settings(
ss.SystemSettings(
conversation_context=ss.ConversationContextSettings(
recent_turn_window_enabled=True,
)
)
)
response = await ss_router.update_conversation_context_setting(
ss_router.ConversationContextUpdateRequest(max_conversation_rounds=45),
SimpleNamespace(),
)
assert response.max_conversation_rounds == 45
assert response.recent_turn_window_enabled is True
persisted = ss.load_system_settings().conversation_context
assert persisted.max_conversation_rounds == 45
assert persisted.recent_turn_window_enabled is True
@pytest.mark.parametrize("value", [0, 51])
def test_conversation_round_limit_rejects_out_of_range_values(value: int) -> None:
with pytest.raises(ValueError):
ss_router.ConversationContextUpdateRequest(max_conversation_rounds=value)
@pytest.mark.asyncio
async def test_non_admin_cannot_enable_recent_turn_context_window(
tmp_runtime_home: Path,
@@ -226,6 +303,28 @@ async def test_non_admin_cannot_enable_recent_turn_context_window(
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
async def test_admin_can_toggle_write_file_compatibility_without_restart(
tmp_runtime_home: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_patch_admin(monkeypatch)
enabled = await ss_router.update_write_file_compatibility_setting(
ss_router.WriteFileCompatibilityUpdateRequest(enabled=True),
SimpleNamespace(),
)
assert enabled.enabled is True
assert ss.get_write_file_compatibility_settings().enabled is True
disabled = await ss_router.update_write_file_compatibility_setting(
ss_router.WriteFileCompatibilityUpdateRequest(enabled=False),
SimpleNamespace(),
)
assert disabled.enabled is False
assert ss.get_write_file_compatibility_settings().enabled is False
def test_prepend_prefix_tags_first_human_without_touching_content() -> None:
graph_input = {"messages": [HumanMessage(content="评价一下中美贸易")]}
out = _prepend_prompt_prefix(graph_input, "你需要站在中方角度")
@@ -0,0 +1,98 @@
"""业务链条「管理员隐藏」目录开关测试。
覆盖(对齐 agents 的 hidden 语义):
- store:``hidden`` 随 update 持久化,create 缺省 False;
- 普通列表(mine / public / pending)一律排除 hidden —— 含 owner 自己的
「我的链条」与管理员的普通视图(选链弹窗路径,不带 admin_view);
- 管理视图(``include_hidden=True`` + admin)不排除,并可按 ``hidden``
筛选「只看隐藏 / 只看显示中」;非管理员的 ``include_hidden`` 一律忽略;
- 单条读取(get_chain)不过滤:owner / 公共链条的已配置使用路径不受影响。
"""
from __future__ import annotations
import pytest
from test_roundtable_chain_review import _SEATS, _make_repo
@pytest.mark.asyncio
async def test_hidden_flag_roundtrip():
repo, engine = await _make_repo()
try:
row = await repo.create_chain("u1", {"id": "c1", "title": "链", "seats": _SEATS})
assert row["hidden"] is False
updated = await repo.update_chain("c1", "u1", hidden=True)
assert updated is not None and updated["hidden"] is True
updated = await repo.update_chain("c1", "u1", hidden=False)
assert updated is not None and updated["hidden"] is False
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_hidden_chain_vanishes_from_all_normal_listings():
repo, engine = await _make_repo()
try:
await repo.create_chain("u1", {"id": "c1", "title": "被隐藏的公共链", "seats": _SEATS})
await repo.create_chain("u1", {"id": "c2", "title": "普通链", "seats": _SEATS})
await repo.update_chain("c1", "u1", is_public=True)
await repo.update_chain("c2", "u1", is_public=True)
await repo.update_chain("c1", "u1", hidden=True)
# owner 的「我的链条」:c1 消失(隐藏对本人也生效)。
rows = await repo.list_chains("u1", scope="mine")
assert {r["id"] for r in rows} == {"c2"}
# 公共列表:任何账号都看不到 c1。
rows = await repo.list_chains("u2", scope="public")
assert {r["id"] for r in rows} == {"c2"}
# 管理员的普通视图(选链弹窗路径,无 admin_view):同样排除。
rows = await repo.list_chains("admin", scope="all", is_admin=True)
assert {r["id"] for r in rows} == {"c2"}
# 非管理员传 include_hidden 也绝不越权看到隐藏链条。
rows = await repo.list_chains("u1", scope="mine", is_admin=False, include_hidden=True)
assert {r["id"] for r in rows} == {"c2"}
# 单条读取不受影响(隐藏不是禁用:业务映射 / 研讨快照继续可用)。
assert await repo.get_chain("c1", "u1") is not None
assert await repo.get_chain("c1", "u2") is not None # 公共链条他人可读
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_admin_view_lists_and_filters_hidden():
repo, engine = await _make_repo()
try:
await repo.create_chain("u1", {"id": "c1", "title": "被隐藏", "seats": _SEATS})
await repo.create_chain("u1", {"id": "c2", "title": "显示中", "seats": _SEATS})
await repo.update_chain("c1", "u1", hidden=True)
# 管理视图:默认(hidden=None)全部可见(含隐藏)。
rows = await repo.list_chains("admin", scope="all", is_admin=True, include_hidden=True)
assert {r["id"] for r in rows} == {"c1", "c2"}
# 只看隐藏 / 只看显示中。
rows = await repo.list_chains(
"admin", scope="all", is_admin=True, include_hidden=True, hidden=True
)
assert {r["id"] for r in rows} == {"c1"}
rows = await repo.list_chains(
"admin", scope="all", is_admin=True, include_hidden=True, hidden=False
)
assert {r["id"] for r in rows} == {"c2"}
# 管理视图 + 非 admin → include_hidden 失效(普通视图,排除隐藏)。
rows = await repo.list_chains("u1", scope="mine", is_admin=False, include_hidden=True)
assert {r["id"] for r in rows} == {"c2"}
# 取消隐藏回到普通列表。
await repo.update_chain("c1", "u1", hidden=False)
rows = await repo.list_chains("u1", scope="mine")
assert {r["id"] for r in rows} == {"c1", "c2"}
finally:
await engine.dispose()
@@ -0,0 +1,134 @@
"""业务链条「席位富化」(available / live_name)测试。
覆盖(对应 roundtable_chains 路由的读时计算富化,前端选链不再实时查 /api/agents):
- 纯函数 ``_apply_seat_enrichment``:存活标记、改名刷新、功能型内置排除、
id 大小写归一、多条链批量;
- ``_is_functional_agent``:与前端 filterRecommendCandidates 排除集对齐;
- schema 安全:ChainSeatResponseSchema 序列化带富化字段;create 请求的
ChainSeatSchema 即使收到富化字段也不持久化(model_dump 剥掉)。
"""
from __future__ import annotations
import pytest
from app.gateway.routers.roundtable_chains import (
ChainSeatResponseSchema,
ChainSeatSchema,
_apply_seat_enrichment,
_is_functional_agent,
)
def _chain(cid: str, seats: list[dict]) -> dict:
return {"id": cid, "title": cid, "seats": [dict(s) for s in seats]}
def _seat(agent_id: str, name: str = "") -> dict:
return {"agent_id": agent_id, "name": name, "description": ""}
def test_functional_agent_exclusion_set():
"""与前端 filterRecommendCandidates 的排除集逐字对齐(含 coordinator- 前缀)。"""
for aid in (
"roundtable-intent",
"roundtable-recommender",
"roundtable-coordinator",
"roundtable-report",
"Roundtable-Coordinator",
"roundtable-coordinator-abc123",
):
assert _is_functional_agent(aid), aid
# 普通席位(含默认席位 id roundtable-summary)不排除。
for aid in ("agent-abc", "roundtable-intelligence", "roundtable-summary", "my-agent"):
assert not _is_functional_agent(aid), aid
def test_apply_enrichment_marks_available_and_live_name():
chain = _chain("c1", [_seat("a", "旧名"), _seat("b", "环境")])
_apply_seat_enrichment([chain], [{"id": "a", "name": "新名"}, {"id": "b", "name": "环境"}])
a, b = chain["seats"]
assert a["available"] is True
assert a["live_name"] == "新名" # 改名后前端显示实时名而非快照
assert b["available"] is True
assert b["live_name"] == "环境"
def test_apply_enrichment_missing_agent_marked_unavailable():
chain = _chain("c1", [_seat("a"), _seat("gone", "已删除的席位")])
_apply_seat_enrichment([chain], [{"id": "a", "name": "A"}])
a, gone = chain["seats"]
assert a["available"] is True
assert gone["available"] is False
assert gone["live_name"] is None # 前端回退 seat.name 快照
def test_apply_enrichment_functional_agents_unavailable_even_if_alive():
"""功能型内置智能体在 agents 表里存在(seeder 落盘),但不可作席位。"""
chain = _chain(
"c1",
[_seat("a"), _seat("roundtable-intent", "意图理解")],
)
_apply_seat_enrichment(
[chain],
[{"id": "a", "name": "A"}, {"id": "roundtable-intent", "name": "任务理解"}],
)
_, intent = chain["seats"]
assert intent["available"] is False
assert intent["live_name"] is None
def test_apply_enrichment_case_insensitive_and_multi_chain():
"""seat.agent_id 大小写与 agents 表 id 归一(小写)后匹配;多条链一次算完。"""
c1 = _chain("c1", [_seat("Agent-ABC", "快照名")])
c2 = _chain("c2", [_seat("agent-abc"), _seat("zzz")])
_apply_seat_enrichment([c1, c2], [{"id": "agent-abc", "name": "实时名"}])
assert c1["seats"][0]["available"] is True
assert c1["seats"][0]["live_name"] == "实时名"
assert c2["seats"][0]["available"] is True
assert c2["seats"][1]["available"] is False
def test_response_schema_serializes_enrichment_fields():
seat = ChainSeatResponseSchema(
agent_id="a", name="快照", available=True, live_name="实时"
)
dumped = seat.model_dump()
assert dumped["available"] is True
assert dumped["live_name"] == "实时"
def test_create_request_schema_drops_enrichment_fields():
"""create/update 请求收到富化字段也不落库(ChainSeatSchema 未声明 → 剥掉)。"""
seat = ChainSeatSchema.model_validate(
{"agent_id": "a", "name": "x", "available": True, "live_name": "y"}
)
dumped = seat.model_dump()
assert "available" not in dumped
assert "live_name" not in dumped
@pytest.mark.asyncio
async def test_store_roundtrip_keeps_seats_json_clean():
"""富化只改响应 dict;落库 / 回读的 seats JSON 不含富化字段。"""
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.base import Base
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow # noqa: F401 注册表
from deerflow.persistence.roundtable_chains.sql import RoundtableChainRepository
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
try:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
repo = RoundtableChainRepository(async_sessionmaker(engine, expire_on_commit=False))
row = await repo.create_chain(
"u1", {"id": "c1", "title": "链", "seats": [_seat("a", "A"), _seat("b", "B")]}
)
# 模拟路由层富化(只动响应 dict)。
_apply_seat_enrichment([row], [{"id": "a", "name": "A"}])
# 回读:持久化层不带 available / live_name。
got = await repo.get_chain("c1", "u1")
assert got is not None
assert all("available" not in s and "live_name" not in s for s in got["seats"])
finally:
await engine.dispose()
@@ -41,11 +41,21 @@ class _RecGateway:
def __init__(self, dispatch_cycles: list[list[tuple[str, str]]]) -> None:
self._cycles = dispatch_cycles
self._leader_calls = 0
self.leader_requests: list[tuple[str, bool]] = []
self.seat_tasks: list[tuple[str, str]] = []
self.active = 0
self.max_active = 0
async def run_leader(self, *, thread_ids, coordinator_name, message, model) -> LeaderTurn:
async def run_leader(
self,
*,
thread_ids,
coordinator_name,
message,
model,
finalize_only: bool = False,
) -> LeaderTurn:
self.leader_requests.append((message, finalize_only))
i = self._leader_calls
self._leader_calls += 1
disp = self._cycles[i] if i < len(self._cycles) else []
@@ -64,7 +74,7 @@ class _RecGateway:
raise AssertionError("enable_report=False 不应绘制报告")
async def _run(gw: _RecGateway, *, gather_first: bool):
async def _run(gw: _RecGateway, *, gather_first: bool, max_cycles: int = 8):
return await run_orchestration(
gateway=gw,
progress=_writer(),
@@ -77,6 +87,7 @@ async def _run(gw: _RecGateway, *, gather_first: bool):
intent_text="背景X",
enable_report=False,
gather_first=gather_first,
max_cycles=max_cycles,
)
@@ -138,3 +149,33 @@ async def test_gather_phase_is_parallel():
assert all(_GATHER_MARK in t for _, t in gw.seat_tasks)
assert len(gw.seat_tasks) == 3
assert gw.max_active == 3 # 取数阶段三席位真并行
@pytest.mark.asyncio
async def test_recommend_eighth_round_forces_text_closeout_instead_of_stopping():
"""前 7 轮仍派活时,第 8 轮必须关闭工具并要求总控总结,不能直接停止。"""
gw = _RecGateway([[('seat-a', f'task-{i}')] for i in range(8)])
result = await _run(gw, gather_first=False)
assert result.status == "done"
assert result.consensus is True
assert len(gw.leader_requests) == 8
assert [finalize for _, finalize in gw.leader_requests] == [False] * 7 + [True]
assert "最终方案与结论正文" in gw.leader_requests[-1][0]
assert "不要调用任何工具" in gw.leader_requests[-1][0]
# 第 8 轮是总控收口,不再执行假网关返回的派活。
assert len(gw.seat_tasks) == 7
@pytest.mark.asyncio
async def test_recommend_uses_configured_closeout_round():
"""动态配置第 3 轮收口时,只执行前两轮席位派活。"""
gw = _RecGateway([[('seat-a', f'task-{i}')] for i in range(4)])
result = await _run(gw, gather_first=False, max_cycles=3)
assert result.status == "done"
assert len(gw.leader_requests) == 3
assert [finalize for _, finalize in gw.leader_requests] == [False, False, True]
assert len(gw.seat_tasks) == 2
@@ -20,13 +20,20 @@ from types import SimpleNamespace
import pytest
import app.gateway.roundtable_model_fallback as mf
import app.gateway.routers.multi_agent as multi_agent
import app.gateway.routers.parallel_agents as parallel_agents
from app.gateway.roundtable_model_fallback import (
fallback_model_chain,
is_llm_error_message,
reason_text,
)
from app.gateway.routers._strategy_selection_seed import default_strategy_agent_roles
from app.gateway.routers._strategy_selection_seed import (
STRATEGY_INTENT_AGENT_ID,
STRATEGY_JUDGE_AGENT_ID,
STRATEGY_SECURITY_AGENT_ID,
STRATEGY_SOLUTION_AGENT_ID,
default_strategy_agent_roles,
)
from deerflow.agents.middlewares.llm_error_handling_middleware import (
LLM_ERROR_MARKER,
LLMErrorHandlingMiddleware,
@@ -248,6 +255,38 @@ def test_parallel_agent_substantive_content_accepts_chinese_and_english() -> Non
assert not parallel_agents._has_substantive_content("<think>internal reasoning only</think>!!!")
def test_strategy_stage_tool_surface_allows_clarification_only_for_intent() -> None:
assert "ask_clarification" not in parallel_agents._excluded_tools_for_agent(STRATEGY_INTENT_AGENT_ID)
for agent_id in (
STRATEGY_SECURITY_AGENT_ID,
STRATEGY_JUDGE_AGENT_ID,
STRATEGY_SOLUTION_AGENT_ID,
):
assert "ask_clarification" in parallel_agents._excluded_tools_for_agent(agent_id)
def test_strategy_task_message_matches_stage_clarification_contract() -> None:
judge_message = parallel_agents._build_task_message(
"策略评判",
"比较候选策略并输出结构化评分。",
agent_id=STRATEGY_JUDGE_AGENT_ID,
)
assert "【本阶段禁止反问】" in judge_message
assert "不得调用 `ask_clarification`" in judge_message
assert "数据限制与待核验" in judge_message
assert "不要重复输出同一正文或多个版本" in judge_message
intent_message = parallel_agents._build_task_message(
"意图理解",
"整理用户需求。",
agent_id=STRATEGY_INTENT_AGENT_ID,
)
assert "【需要向用户确认时】" in intent_message
assert "调用 `ask_clarification` 工具" in intent_message
assert "【本阶段禁止反问】" not in intent_message
def test_parallel_agent_extracts_provider_reasoning_without_mixing_answer() -> None:
answer, reasoning = parallel_agents._extract_ai_stream_parts(
{
@@ -779,6 +818,149 @@ def test_reason_text() -> None:
assert reason_text("不存在的码") == "调用失败"
@pytest.mark.asyncio
async def test_thinking_only_seat_does_not_switch_model(monkeypatch) -> None:
"""Reasoning-only output is an incomplete delivery, not a model failure."""
calls: list[str] = []
async def fake_stream(_client, _base, _headers, _thread_id, body):
calls.append(body["context"]["model_name"])
yield None, [
'data: {"messages":[{"type":"ai","content":"<think>仍在分析约束</think>"}]}'
]
monkeypatch.setattr(multi_agent, "_stream_upstream", fake_stream)
monkeypatch.setattr(
multi_agent,
"fallback_model_chain",
lambda _model: ["slow-thinking-model", "backup-model"],
)
monkeypatch.setattr(
multi_agent,
"append_seat_skill_directive",
lambda _agent_id, task, **_kwargs: task,
)
monkeypatch.setattr(multi_agent, "_spawn_detached_broadcast", lambda *_args, **_kwargs: None)
frames = [
frame
async for frame in multi_agent._special_run(
client=object(),
base="http://gateway",
headers={},
agent_threads={"seat-a": "seat-thread"},
agent_name="seat-a",
new_message="完成分析",
skill_stop_names=[],
model_name="slow-thinking-model",
)
]
assert calls == ["slow-thinking-model"]
assert not any(
isinstance(frame, dict) and frame.get("status") == "model_switch"
for frame in frames
)
assert frames[-1]["status"] == "invalid_delivery"
assert frames[-1]["reason"] == "thinking_only"
@pytest.mark.asyncio
async def test_real_model_error_still_switches_to_next_model(monkeypatch) -> None:
calls: list[str] = []
async def fake_stream(_client, _base, _headers, _thread_id, body):
model = body["context"]["model_name"]
calls.append(model)
message = (
{
"type": "ai",
"content": "大模型服务繁忙,请稍后再试。",
"additional_kwargs": {LLM_ERROR_MARKER: "busy"},
}
if model == "broken-model"
else {"type": "ai", "content": "这是备用模型完成的有效交付正文。"}
)
yield None, [f"data: {json.dumps({'messages': [message]}, ensure_ascii=False)}"]
monkeypatch.setattr(multi_agent, "_stream_upstream", fake_stream)
monkeypatch.setattr(
multi_agent,
"fallback_model_chain",
lambda _model: ["broken-model", "backup-model"],
)
monkeypatch.setattr(
multi_agent,
"append_seat_skill_directive",
lambda _agent_id, task, **_kwargs: task,
)
monkeypatch.setattr(multi_agent, "_spawn_detached_broadcast", lambda *_args, **_kwargs: None)
frames = [
frame
async for frame in multi_agent._special_run(
client=object(),
base="http://gateway",
headers={},
agent_threads={"seat-a": "seat-thread"},
agent_name="seat-a",
new_message="完成分析",
skill_stop_names=[],
model_name="broken-model",
)
]
assert calls == ["broken-model", "backup-model"]
assert any(
isinstance(frame, dict) and frame.get("status") == "model_switch"
for frame in frames
)
assert frames[-1]["model_used"] == "backup-model"
assert frames[-1]["content"] == "这是备用模型完成的有效交付正文。"
@pytest.mark.asyncio
async def test_background_thinking_only_does_not_switch_model(monkeypatch) -> None:
import app.gateway.roundtable_inprocess_gateway as gateway_module
calls: list[str] = []
async def fake_run_agent_turn(*_args, **kwargs):
calls.append(kwargs["model"])
return [{"type": "ai", "content": "<think>继续进行长思考</think>"}]
monkeypatch.setattr(gateway_module, "_run_agent_turn", fake_run_agent_turn)
monkeypatch.setattr(
gateway_module,
"fallback_model_chain",
lambda _model: ["slow-thinking-model", "backup-model"],
)
gateway = gateway_module.InProcessRoundtableGateway(
SimpleNamespace(state=SimpleNamespace()),
user_id="u1",
agents=[],
)
messages, switches, used_model = await gateway._run_turn_with_fallback(
thread_id="seat-thread",
agent_name="seat-a",
message="完成分析",
requested_model="slow-thinking-model",
excluded_tools=[],
allowed_tools=None,
skill_stop_names=[],
thinking_enabled=True,
reasoning_effort="medium",
recursion_limit=1000,
stage="step2_seat",
)
assert calls == ["slow-thinking-model"]
assert switches == []
assert used_model == "slow-thinking-model"
assert messages[-1]["content"] == "<think>继续进行长思考</think>"
# ── 引擎:单席位异常隔离 + 模型容错记录 → 系统提示气泡 ──────────────────────────
_AGENTS = [SeatRef("seat-a", "情报"), SeatRef("seat-b", "环境"), SeatRef("seat-c", "方案")]
@@ -800,7 +982,15 @@ class _ChainGateway:
self.switch_agent = switch_agent
self.seen: list[str] = []
async def run_leader(self, *, thread_ids, coordinator_name, message, model) -> LeaderTurn:
async def run_leader(
self,
*,
thread_ids,
coordinator_name,
message,
model,
finalize_only: bool = False,
) -> LeaderTurn:
return LeaderTurn(final_text="总控说明", dispatched=[])
async def run_seat(self, *, thread_ids, agent_id, task, model) -> SeatTurn:
@@ -8,11 +8,13 @@ import pytest
import app.gateway.roundtable_seat_skills as seat_skills
import app.gateway.routers.multi_agent as multi_agent
from app.gateway.roundtable_run_policy import roundtable_run_policy
from deerflow.agents.lead_agent.agent import (
_append_orchestration_capability_boundary,
_enforce_coordinator_only_tool_exclusions,
_filter_available_skills_for_excluded_tools,
_knowledge_space_strict_exclusions,
_runtime_tool_is_enabled,
)
from deerflow.tools.builtins.skill_tools import (
_resolve_active_agent_id,
@@ -107,6 +109,49 @@ def test_single_agent_tool_exclusions_fail_closed_but_coordinator_is_allowed() -
assert _enforce_coordinator_only_tool_exclusions("task-analysis-coordinator", None) is None
def test_tool_driving_middleware_respects_runtime_tool_policy() -> None:
"""A text-only coordinator must never receive a synthetic write_file call."""
assert _runtime_tool_is_enabled({}, "write_file") is True
assert _runtime_tool_is_enabled({"allowed_tools": ["write_file"]}, "write_file") is True
assert _runtime_tool_is_enabled({"allowed_tools": ["present_files"]}, "write_file") is False
assert _runtime_tool_is_enabled({"excluded_tools": ["write_file"]}, "write_file") is False
assert _runtime_tool_is_enabled(
{"allowed_tools": ["write_file"], "excluded_tools": ["write_file"]},
"write_file",
) is False
assert _runtime_tool_is_enabled({"disable_tools": True}, "write_file") is False
def test_coordinator_guard_does_not_shrink_any_roundtable_runtime_budget() -> None:
"""Tool isolation must not shorten the working runtime of any role."""
for role in (
"leader",
"seat",
"seat_parallel",
"report",
"summary",
"action_plan",
"dashboard",
"ingest",
):
policy = roundtable_run_policy(role)
assert policy.recursion_limit == 1000
if role != "leader":
assert policy.allowed_tools is None
seat = roundtable_run_policy("seat")
for tool in (
"web_fetch",
"browser_fetch_page",
"browser_act",
"tool_search",
"page_report_template",
"scheduled_task",
"skill_manage",
):
assert tool not in seat.excluded_tools
def test_selected_knowledge_space_blocks_external_retrieval_surfaces() -> None:
excluded = _knowledge_space_strict_exclusions(["custom_disabled"])
assert "custom_disabled" in excluded
@@ -237,8 +282,10 @@ async def test_leader_never_broadcasts_orchestration_prompts_to_seats(monkeypatc
'"tool_calls":[{"name":"agent_orchestration","args":'
'{"agent_name":"seat-a","task":"分析风险"}}]}]}'
]
seen_inputs = []
async def _fake_stream(*args, **kwargs):
seen_inputs.append(args[4]["input"]["messages"][0]["content"][0]["text"])
yield None, captured
def _unexpected_broadcast(*args, **kwargs):
@@ -265,3 +312,82 @@ async def test_leader_never_broadcasts_orchestration_prompts_to_seats(monkeypatc
]
assert frames[-1]["status"] == [["seat-a", "分析风险"]]
assert "seat-a" in seen_inputs[0]
assert "【本次圆桌" in seen_inputs[0]
@pytest.mark.asyncio
async def test_leader_final_closeout_is_tool_free_plain_text(monkeypatch) -> None:
"""Final synthesis must not loop through writer/presenter/orchestration tools."""
captured_bodies = []
async def _fake_delivery(*args, **kwargs):
return {"seat-a": (True, "席位完整交付", 6, "席位完整交付")}
async def _fake_stream(*args, **kwargs):
captured_bodies.append(args[4])
yield None, ['data: {"messages":[{"type":"ai","content":"最终结论正文"}]}']
monkeypatch.setattr(multi_agent, "_collect_delivery_status", _fake_delivery)
monkeypatch.setattr(multi_agent, "_stream_upstream", _fake_stream)
monkeypatch.setattr(multi_agent, "fallback_model_chain", lambda model: [model])
frames = [
frame
async for frame in multi_agent._leader_run(
client=object(),
base="http://gateway",
headers={},
agent_threads={"roundtable-coordinator": "leader-thread", "seat-a": "seat-thread"},
agent_name="roundtable-coordinator",
new_message="请输出最终方案",
skill_stop_names=[],
model_name="test-model",
synthesis_mode=True,
finalize_only=True,
)
]
body = captured_bodies[0]
assert body["context"]["allowed_tools"] == []
assert "write_file" in body["excluded_tools"]
assert "present_files" in body["excluded_tools"]
assert "最终收口·纯正文" in body["input"]["messages"][0]["content"][0]["text"]
assert frames[-1]["status"] == []
@pytest.mark.asyncio
async def test_leader_final_closeout_switches_only_after_completed_empty_response(monkeypatch) -> None:
"""最终收口完整结束仍无正文才换模;换模后仍以无工具正文完成。"""
seen_models = []
async def _fake_stream(*args, **kwargs):
body = args[4]
model = body["context"]["model_name"]
seen_models.append(model)
content = "" if model == "model-a" else "换模后的最终结论"
yield None, [f'data: {{"messages":[{{"type":"ai","content":"{content}"}}]}}']
monkeypatch.setattr(multi_agent, "_stream_upstream", _fake_stream)
monkeypatch.setattr(multi_agent, "fallback_model_chain", lambda model: ["model-a", "model-b"])
frames = [
frame
async for frame in multi_agent._leader_run(
client=object(),
base="http://gateway",
headers={},
agent_threads={"roundtable-coordinator": "leader-thread"},
agent_name="roundtable-coordinator",
new_message="请输出最终方案",
skill_stop_names=[],
model_name="model-a",
synthesis_mode=True,
finalize_only=True,
)
]
assert seen_models == ["model-a", "model-b"]
assert any(isinstance(frame, dict) and frame.get("status") == "model_switch" for frame in frames)
assert frames[-1]["status"] == []
assert frames[-1]["content"] == "换模后的最终结论"
@@ -0,0 +1,933 @@
"""Cross-user roundtable step-2 room cap."""
from __future__ import annotations
import asyncio
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
import pytest
import pytest_asyncio
from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
import deerflow.persistence.roundtable_jobs.model # noqa: F401
import deerflow.persistence.roundtable_rooms.model # noqa: F401
from app.gateway import roundtable_room_gate
from app.gateway.routers import system_settings as ss_router
from app.gateway.routers.multi_agent import (
_is_step2_room_limited,
_run_payload,
_step3_thread_metadata,
)
from deerflow.config import system_settings as ss
from deerflow.persistence.base import Base
from deerflow.persistence.roundtable_jobs import RoundtableJobRepository
from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
from deerflow.persistence.roundtable_rooms.model import RoundtableRoomLeaseRow
from deerflow.persistence.roundtable_rooms.sql import (
RoundtableRoomRepository,
compute_room_session_key,
)
@pytest.fixture
def tmp_runtime_home(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
monkeypatch.setattr(ss, "runtime_home", lambda: tmp_path)
return tmp_path
def _patch_admin(monkeypatch: pytest.MonkeyPatch) -> None:
admin = SimpleNamespace(id="admin1", system_role="admin", email="admin@example.com")
async def _optional(_request):
return admin
monkeypatch.setattr(ss_router, "get_optional_user_from_request", _optional)
@pytest_asyncio.fixture
async def rooms(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'rooms.db'}")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
factory = async_sessionmaker(engine, expire_on_commit=False)
try:
yield RoundtableRoomRepository(factory), factory
finally:
await engine.dispose()
async def _admit(repo: RoundtableRoomRepository, draft_id: str, user_id: str, **kwargs):
return await repo.admit(
draft_id=draft_id,
user_id=user_id,
task_id=kwargs.get("task_id"),
room_session_id=kwargs.get("room_session_id"),
enabled=kwargs.get("enabled", True),
max_rooms=kwargs.get("max_rooms", 3),
extra_keys=kwargs.get("extra_keys"),
)
def _room_key(draft_id: str, user_id: str) -> str:
key = compute_active_dedupe_key(draft_id, user_id, None)
assert key
return key
@pytest.mark.asyncio
async def test_opening_a_page_does_not_occupy_a_room(rooms) -> None:
repo, _factory = rooms
# Merely loading the page/history performs no acquire call.
for index in range(4):
assert await repo.summary() == 0, index
assert await repo.summary() == 0
@pytest.mark.asyncio
async def test_four_simultaneous_acquires_admit_exactly_three(rooms) -> None:
_repo, factory = rooms
# Separate repository objects simulate separate Gateway workers: the
# process-local asyncio locks cannot coordinate these contenders.
workers = [RoundtableRoomRepository(factory) for _ in range(4)]
results = await asyncio.gather(
*(
_admit(worker, f"concurrent-{index}", f"u-{index}")
for index, worker in enumerate(workers)
)
)
assert sum(1 for result in results if result["ok"]) == 3
rejected = [result for result in results if not result["ok"]]
assert len(rejected) == 1
assert rejected[0]["code"] == "ROUNDTABLE_ROOM_FULL"
assert rejected[0]["message"] == "当前会商房间暂时已满,请稍候片刻后再试。"
assert await workers[0].summary() == 3
@pytest.mark.asyncio
async def test_same_draft_in_four_live_pages_still_counts_as_four_rooms(rooms) -> None:
"""A saved draft is data identity, not live-execution identity."""
_repo, factory = rooms
workers = [RoundtableRoomRepository(factory) for _ in range(4)]
results = await asyncio.gather(
*(
_admit(
worker,
"same-draft",
"same-user",
room_session_id=f"browser-page-{index}",
)
for index, worker in enumerate(workers)
)
)
assert sum(1 for result in results if result["ok"]) == 3
assert sum(1 for result in results if not result["ok"]) == 1
assert await workers[0].summary() == 3
@pytest.mark.asyncio
async def test_same_live_room_session_renews_one_room_across_steps(rooms) -> None:
repo, _factory = rooms
first = await _admit(
repo,
"draft-a",
"user-a",
room_session_id="one-live-run",
)
renewed = await _admit(
repo,
"draft-a",
"user-a",
room_session_id="one-live-run",
)
assert first["ok"] is True
assert renewed["ok"] is True
assert renewed["renewed"] is True
assert await repo.summary() == 1
@pytest.mark.asyncio
async def test_heartbeat_only_renews_an_existing_live_lease(rooms) -> None:
repo, _factory = rooms
missing = await repo.renew(
draft_id="draft-a",
user_id="user-a",
task_id=None,
room_session_id="live-session",
enabled=True,
max_rooms=2,
)
assert missing["ok"] is False
assert missing["code"] == "ROOM_LEASE_EXPIRED"
assert await repo.summary() == 0
await _admit(
repo,
"draft-a",
"user-a",
room_session_id="live-session",
max_rooms=2,
)
renewed = await repo.renew(
draft_id="draft-a",
user_id="user-a",
task_id=None,
room_session_id="live-session",
enabled=True,
max_rooms=2,
)
assert renewed["ok"] is True
assert renewed["renewed"] is True
assert await repo.summary() == 1
@pytest.mark.asyncio
async def test_heartbeat_cannot_extend_the_configured_hard_room_deadline(rooms) -> None:
repo, factory = rooms
await _admit(
repo,
"draft-hard-limit",
"user-a",
room_session_id="hard-limit-session",
max_rooms=2,
)
key = compute_room_session_key("hard-limit-session")
assert key is not None
async with factory() as session:
row = await session.get(RoundtableRoomLeaseRow, key)
assert row is not None
row.created_at = datetime.now(UTC) - timedelta(minutes=31)
row.lease_until = datetime.now(UTC) + timedelta(minutes=1)
await session.commit()
decision = await repo.renew(
draft_id="draft-hard-limit",
user_id="user-a",
task_id=None,
room_session_id="hard-limit-session",
enabled=True,
max_rooms=2,
max_hold_minutes=30,
)
assert decision["ok"] is False
assert decision["code"] == "ROOM_MAX_HOLD_EXCEEDED"
assert await repo.summary(max_hold_minutes=30) == 0
@pytest.mark.asyncio
async def test_delayed_heartbeat_cannot_resurrect_a_released_room(rooms) -> None:
repo, _factory = rooms
await _admit(
repo,
"draft-disconnected",
"user-a",
room_session_id="disconnected-session",
max_rooms=1,
)
assert await repo.release(
draft_id="draft-disconnected",
user_id="user-a",
task_id=None,
room_session_id="disconnected-session",
)
delayed = await repo.renew(
draft_id="draft-disconnected",
user_id="user-a",
task_id=None,
room_session_id="disconnected-session",
enabled=True,
max_rooms=1,
)
assert delayed["ok"] is False
assert delayed["code"] == "ROOM_LEASE_EXPIRED"
assert await repo.summary() == 0
next_room = await _admit(
repo,
"draft-next",
"user-b",
room_session_id="next-session",
max_rooms=1,
)
assert next_room["ok"] is True
@pytest.mark.asyncio
async def test_stale_run_for_leased_draft_does_not_consume_a_second_room(rooms) -> None:
"""Run-manager fallback must not double-count the authoritative lease."""
repo, _factory = rooms
admitted = await _admit(
repo,
"same-draft",
"same-user",
room_session_id="current-page",
max_rooms=2,
)
assert admitted["ok"] is True
stale_key = compute_room_session_key("stale-previous-page")
scope_key = _room_key("same-draft", "same-user")
assert stale_key
class _Run:
run_id = "stale-run"
thread_id = "stale-thread"
metadata = {
"roundtable_room_key": stale_key,
"roundtable_room_session_id": "stale-previous-page",
"draft_id": "same-draft",
"user_id": "same-user",
}
class _Runs:
async def list_active(self):
return [_Run()]
request = SimpleNamespace(
app=SimpleNamespace(state=SimpleNamespace(run_manager=_Runs())),
)
extra = await roundtable_room_gate.backend_occupancy_keys(request)
assert extra == {stale_key: scope_key}
assert await repo.summary(extra_keys=extra) == 1
second = await _admit(
repo,
"same-draft",
"same-user",
room_session_id="second-real-page",
max_rooms=2,
extra_keys=extra,
)
assert second["ok"] is True
assert second["active_count"] == 2
blocked = await _admit(
repo,
"same-draft",
"same-user",
room_session_id="third-real-page",
max_rooms=2,
extra_keys=extra,
)
assert blocked["ok"] is False
assert blocked["active_count"] == 2
@pytest.mark.asyncio
async def test_fourth_backend_process_is_rejected_and_same_room_renews(rooms) -> None:
repo, _factory = rooms
extra = {_room_key(f"d{index}", f"u{index}") for index in range(3)}
assert await repo.summary(extra_keys=extra) == 3
blocked = await _admit(repo, "d4", "u4", extra_keys=extra)
assert blocked["ok"] is False
assert blocked["code"] == "ROUNDTABLE_ROOM_FULL"
assert blocked["active_count"] == 3
renewed = await _admit(repo, "d1", "u1", extra_keys=extra)
assert renewed["ok"] is True
assert renewed["renewed"] is True
assert await repo.summary(extra_keys=extra) == 3
@pytest.mark.asyncio
async def test_disabled_limit_accepts_more_than_three_backend_processes(rooms) -> None:
repo, _factory = rooms
extra = {_room_key(f"d{index}", f"u{index}") for index in range(4)}
decision = await _admit(repo, "d4", "u4", enabled=False, extra_keys=extra)
assert decision["ok"] is True
assert decision["active_count"] == 5
assert await repo.summary(extra_keys=extra) == 5
@pytest.mark.asyncio
async def test_executing_background_job_occupies_one_room(rooms) -> None:
repo, factory = rooms
jobs = RoundtableJobRepository(factory)
await jobs.try_create_or_get_active(
draft_id="draft-live",
user_id="owner",
task_id=None,
id="job-live",
status="running",
)
assert await repo.summary() == 1
renewed = await _admit(repo, "draft-live", "owner")
assert renewed["ok"] is True
assert renewed["renewed"] is True
blocked = await _admit(repo, "draft-other", "someone-else", max_rooms=1)
assert blocked["ok"] is False
@pytest.mark.asyncio
async def test_background_job_releases_room_as_soon_as_step2_reaches_consensus(rooms) -> None:
repo, factory = rooms
jobs = RoundtableJobRepository(factory)
await jobs.try_create_or_get_active(
draft_id="draft-consensus",
user_id="owner",
task_id=None,
id="job-consensus",
status="running",
)
async with factory() as session:
row = await session.get(RoundtableJobRow, "job-consensus")
assert row is not None
row.phase = "consensus"
await session.commit()
assert await repo.summary() == 0
admitted = await _admit(repo, "next-step2", "next-user", max_rooms=1)
assert admitted["ok"] is True
@pytest.mark.parametrize(
("agent_type", "agent_name", "expected"),
[
("leader", "roundtable-coordinator", True),
("special", "business-seat", True),
("special", "roundtable-summary", False),
("special", "roundtable-report", False),
("special", "roundtable-dashboard", False),
("special", "roundtable-structure", False),
("special", "position-action-planner", False),
],
)
def test_only_step2_runs_consume_room_capacity(
agent_type: str,
agent_name: str,
expected: bool,
) -> None:
assert _is_step2_room_limited(agent_type, agent_name) is expected
def test_final_coordinator_closeout_is_not_blocked_by_expired_room_lease() -> None:
assert _is_step2_room_limited(
"leader",
"roundtable-coordinator",
finalize_only=True,
) is False
def test_step3_threads_are_not_marked_as_step2_room_occupants() -> None:
metadata = _step3_thread_metadata()
assert metadata == {"thread_type": "roundtable", "system": True}
def test_realtime_roundtable_runs_cancel_when_the_sse_disconnects() -> None:
payload = _run_payload(
agent_name="roundtable-seat",
model_name="test-model",
thread_id="thread-1",
new_message="test",
excluded_tools=[],
skill_stop_names=[],
cancel_on_disconnect=True,
)
assert payload["on_disconnect"] == "cancel"
step3_payload = _run_payload(
agent_name="roundtable-summary",
model_name="test-model",
thread_id="thread-2",
new_message="test",
excluded_tools=[],
skill_stop_names=[],
)
assert step3_payload["on_disconnect"] == "continue"
@pytest.mark.asyncio
async def test_historical_awaiting_input_jobs_are_not_rooms(rooms) -> None:
"""查看聊天记录 / 停在待澄清的旧作业不能占满房间上限。"""
repo, factory = rooms
jobs = RoundtableJobRepository(factory)
for index in range(5):
await jobs.try_create_or_get_active(
draft_id=None,
user_id=f"old-{index}",
task_id=None,
id=f"job-old-{index}",
status="awaiting_input",
)
stale = datetime.now(UTC) - timedelta(days=120)
async with factory() as session:
rows = (await session.execute(select(RoundtableJobRow))).scalars().all()
for row in rows:
row.status = "awaiting_input"
row.created_at = stale
row.updated_at = stale
row.lease_until = None
row.draft_id = None
await session.commit()
assert await repo.summary() == 0
admitted = await _admit(repo, "fresh", "u-new", max_rooms=3)
assert admitted["ok"] is True
@pytest.mark.asyncio
async def test_recent_awaiting_input_retains_its_room_but_stale_queued_does_not(rooms) -> None:
"""Clarification can resume without admission; stale queue residue is ignored."""
repo, factory = rooms
jobs = RoundtableJobRepository(factory)
await jobs.try_create_or_get_active(
draft_id="draft-paused",
user_id="viewer",
task_id=None,
id="job-paused",
status="awaiting_input",
)
await jobs.try_create_or_get_active(
draft_id="draft-queued-old",
user_id="other",
task_id=None,
id="job-queued-old",
status="queued",
)
stale = datetime.now(UTC) - timedelta(days=30)
async with factory() as session:
queued = await session.get(RoundtableJobRow, "job-queued-old")
assert queued is not None
queued.updated_at = stale
queued.lease_until = None
await session.commit()
assert await repo.summary() == 1
admitted = await _admit(repo, "fresh-live", "u-live", max_rooms=1)
assert admitted["ok"] is False
assert admitted["code"] == "ROUNDTABLE_ROOM_FULL"
@pytest.mark.asyncio
async def test_jobs_from_every_account_share_one_global_pool(rooms) -> None:
repo, factory = rooms
jobs = RoundtableJobRepository(factory)
for index in range(3):
await jobs.try_create_or_get_active(
draft_id=f"draft-{index}",
user_id=f"account-{index}",
task_id=None,
id=f"job-{index}",
status="running",
)
assert await repo.summary() == 3
blocked = await _admit(repo, "draft-new", "account-new")
assert blocked["ok"] is False
assert blocked["active_count"] == 3
@pytest.mark.asyncio
async def test_hard_limit_requests_cancellation_for_a_stuck_background_job(rooms) -> None:
repo, factory = rooms
jobs = RoundtableJobRepository(factory)
await jobs.try_create_or_get_active(
draft_id="stuck-draft",
user_id="stuck-user",
task_id=None,
id="stuck-job",
status="running",
)
async with factory() as session:
row = await session.get(RoundtableJobRow, "stuck-job")
assert row is not None
row.created_at = datetime.now(UTC) - timedelta(minutes=31)
row.lease_until = datetime.now(UTC) + timedelta(minutes=1)
await session.commit()
assert await repo.summary(max_hold_minutes=30) == 0
async with factory() as session:
row = await session.get(RoundtableJobRow, "stuck-job")
assert row is not None
assert row.status == "cancel_requested"
@pytest.mark.asyncio
async def test_background_handoff_preserves_the_original_room_start_time(rooms) -> None:
_repo, factory = rooms
jobs = RoundtableJobRepository(factory)
room_started_at = datetime.now(UTC) - timedelta(minutes=20)
row, created = await jobs.try_create_or_get_active(
draft_id="handoff-draft",
user_id="handoff-user",
task_id=None,
id="handoff-job",
status="queued",
room_started_at=room_started_at,
)
assert created is True
persisted = datetime.fromisoformat(str(row["created_at"]))
assert abs((persisted - room_started_at).total_seconds()) < 1
@pytest.mark.asyncio
async def test_enabled_gate_fails_closed_without_a_store(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
roundtable_room_gate,
"get_roundtable_room_limit_settings",
lambda: SimpleNamespace(enabled=True, max_rooms=3),
)
request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace()), state=SimpleNamespace(user=None))
decision = await roundtable_room_gate.admit_roundtable_room(
request,
draft_id="draft",
task_id=None,
room_session_id="session",
)
assert decision["ok"] is False
assert decision["code"] == "ROOM_STORE_UNAVAILABLE"
@pytest.mark.asyncio
async def test_enabled_gate_rejects_missing_live_session_id(
rooms,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
roundtable_room_gate,
"get_roundtable_room_limit_settings",
lambda: SimpleNamespace(enabled=True, max_rooms=3),
)
repo, _factory = rooms
decision = await roundtable_room_gate.admit_roundtable_room(
_enforce_request(repo, "user"),
draft_id="draft",
task_id=None,
)
assert decision["ok"] is False
assert decision["code"] == "ROOM_SESSION_REQUIRED"
@pytest.mark.asyncio
async def test_disabled_gate_still_allows_without_a_store(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
roundtable_room_gate,
"get_roundtable_room_limit_settings",
lambda: SimpleNamespace(enabled=False, max_rooms=3),
)
request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace()), state=SimpleNamespace(user=None))
decision = await roundtable_room_gate.admit_roundtable_room(request, draft_id=None, task_id=None)
assert decision["ok"] is True
def _enforce_request(repo, user_id: str, run_mgr=None, thread_store=None):
return SimpleNamespace(
app=SimpleNamespace(
state=SimpleNamespace(
roundtable_room_store=repo,
run_manager=run_mgr,
thread_store=thread_store,
)
),
state=SimpleNamespace(user=SimpleNamespace(id=user_id)),
)
@pytest.mark.asyncio
async def test_gate_rejects_a_new_room_when_backend_processes_are_full(
rooms,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
roundtable_room_gate,
"get_roundtable_room_limit_settings",
lambda: SimpleNamespace(enabled=True, max_rooms=3),
)
repo, _factory = rooms
for index in range(3):
await roundtable_room_gate.enforce_roundtable_room(
_enforce_request(repo, f"u{index}"),
draft_id=f"d{index}",
task_id=None,
room_session_id=f"session-{index}",
)
with pytest.raises(HTTPException) as exc_info:
await roundtable_room_gate.enforce_roundtable_room(
_enforce_request(repo, "u-new"),
draft_id="d-new",
task_id=None,
room_session_id="session-new",
)
assert exc_info.value.status_code == 409
assert exc_info.value.detail["code"] == "ROUNDTABLE_ROOM_FULL"
assert exc_info.value.detail["active_count"] == 3
@pytest.mark.asyncio
async def test_run_stream_gate_never_admits_an_unleased_history_continuation(
rooms,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
roundtable_room_gate,
"get_roundtable_room_limit_settings",
lambda: SimpleNamespace(enabled=True, max_rooms=3),
)
repo, _factory = rooms
for index in range(3):
await roundtable_room_gate.enforce_roundtable_room(
_enforce_request(repo, f"u{index}"),
draft_id=f"d{index}",
task_id=None,
room_session_id=f"session-{index}",
)
with pytest.raises(HTTPException) as exc_info:
await roundtable_room_gate.enforce_roundtable_run(
_enforce_request(repo, "history-user"),
draft_id="history-draft",
task_id=None,
room_session_id="history-session",
thread_ids=["legacy-history-thread"],
)
assert exc_info.value.status_code == 409
assert exc_info.value.detail["code"] == "ROOM_LEASE_EXPIRED"
@pytest.mark.asyncio
async def test_run_stream_gate_resolves_room_key_from_thread_metadata(
rooms,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
roundtable_room_gate,
"get_roundtable_room_limit_settings",
lambda: SimpleNamespace(enabled=True, max_rooms=1),
)
repo, _factory = rooms
room_key = _room_key("thread-draft", "owner")
admitted = await repo.admit_room_key(
room_key=room_key,
draft_id="thread-draft",
user_id="owner",
task_id=None,
holder_user_id="owner",
enabled=True,
max_rooms=1,
)
assert admitted["ok"] is True
class _Threads:
async def get(self, thread_id: str, *, user_id=None):
assert thread_id == "room-thread"
return {
"user_id": "owner",
"metadata": {
"roundtable_room_key": room_key,
"draft_id": "thread-draft",
},
}
decision = await roundtable_room_gate.enforce_roundtable_run(
_enforce_request(repo, "owner", thread_store=_Threads()),
draft_id=None,
task_id=None,
thread_ids=["room-thread"],
)
assert decision is not None
assert decision["ok"] is True
assert decision["renewed"] is True
assert await repo.summary() == 1
@pytest.mark.asyncio
async def test_admin_can_change_roundtable_room_limit_without_restart(
tmp_runtime_home: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_patch_admin(monkeypatch)
assert ss.get_roundtable_room_limit_settings().enabled is True
assert ss.get_roundtable_room_limit_settings().max_rooms == 3
assert ss.get_roundtable_room_limit_settings().max_hold_minutes == 30
assert ss.get_roundtable_room_limit_settings().max_cycles == 8
assert ss.get_roundtable_room_limit_settings().write_file_compatibility_enabled is False
saved = await ss_router.update_roundtable_room_limit_setting(
ss_router.RoundtableRoomLimitUpdateRequest(
enabled=False,
max_rooms=5,
max_hold_minutes=45,
max_cycles=12,
),
SimpleNamespace(),
)
assert saved.enabled is False
assert saved.max_rooms == 5
assert saved.max_hold_minutes == 45
assert saved.max_cycles == 12
assert ss.load_system_settings().roundtable_room_limit.max_rooms == 5
assert ss.load_system_settings().roundtable_room_limit.max_hold_minutes == 45
assert ss.load_system_settings().roundtable_room_limit.max_cycles == 12
assert saved.write_file_compatibility_enabled is False
@pytest.mark.asyncio
async def test_roundtable_file_stream_is_only_attached_to_roundtable_runs(
tmp_runtime_home: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_patch_admin(monkeypatch)
base = dict(
agent_name="roundtable-seat",
model_name="test-model",
thread_id="thread-1",
new_message="test",
excluded_tools=[],
skill_stop_names=[],
)
before = _run_payload(**base, roundtable_file_stream=True)
assert "roundtable_write_file_compatibility" not in before["context"]
saved = await ss_router.update_roundtable_room_limit_setting(
ss_router.RoundtableRoomLimitUpdateRequest(write_file_compatibility_enabled=True),
SimpleNamespace(),
)
assert saved.write_file_compatibility_enabled is True
opted_in = _run_payload(**base, roundtable_file_stream=True)
assert opted_in["context"]["roundtable_write_file_compatibility"] is True
ordinary = _run_payload(**base)
assert "roundtable_write_file_compatibility" not in ordinary["context"]
@pytest.mark.asyncio
async def test_non_admin_cannot_change_roundtable_room_limit(
tmp_runtime_home: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user = SimpleNamespace(id="user1", system_role="user", email="user@example.com")
async def _optional(_request):
return user
monkeypatch.setattr(ss_router, "get_optional_user_from_request", _optional)
with pytest.raises(HTTPException) as exc_info:
await ss_router.update_roundtable_room_limit_setting(
ss_router.RoundtableRoomLimitUpdateRequest(enabled=False),
SimpleNamespace(),
)
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
async def test_live_langgraph_runs_occupy_a_room_without_a_lease(rooms) -> None:
"""发起后即使租约被释放,仍在跑的第二步流必须占房间。"""
repo, _factory = rooms
live = compute_active_dedupe_key("draft-run-a", "u-a", None)
other = compute_active_dedupe_key("draft-run-b", "u-b", None)
third = compute_active_dedupe_key("draft-run-c", "u-c", None)
assert live and other and third
extra = {live, other, third}
assert await repo.summary(extra_keys=extra) == 3
blocked = await _admit(repo, "draft-run-d", "u-d", extra_keys=extra)
assert blocked["ok"] is False
assert blocked["active_count"] == 3
# Same session renews instead of taking a fourth slot.
renewed = await _admit(repo, "draft-run-a", "u-a", extra_keys=extra)
assert renewed["ok"] is True
assert renewed["renewed"] is True
@pytest.mark.asyncio
async def test_live_runs_from_every_account_count_together(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"deerflow.persistence.engine.get_session_factory",
lambda: None,
)
class _Run:
def __init__(self, thread_id: str) -> None:
self.thread_id = thread_id
class _Mgr:
async def list_active(self):
return [_Run("alice-leader"), _Run("bob-leader"), _Run("carol-leader")]
class _Threads:
async def get(self, thread_id: str, *, user_id=None):
rooms = {
"alice-leader": {
"user_id": "alice",
"metadata": {"roundtable_room_key": "room-alice"},
},
"bob-leader": {
"user_id": "bob",
"metadata": {"roundtable_room_key": "room-bob"},
},
"carol-leader": {
"user_id": "carol",
"metadata": {"roundtable_room_key": "room-carol"},
},
}
return rooms.get(thread_id)
request = SimpleNamespace(
app=SimpleNamespace(state=SimpleNamespace(run_manager=_Mgr(), thread_store=_Threads())),
)
keys = await roundtable_room_gate.extra_live_roundtable_keys(request)
assert set(keys) == {"room-alice", "room-bob", "room-carol"}
@pytest.mark.asyncio
async def test_multiple_seats_in_one_session_count_as_one_room(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"deerflow.persistence.engine.get_session_factory",
lambda: None,
)
class _Run:
def __init__(self, thread_id: str) -> None:
self.thread_id = thread_id
class _Mgr:
async def list_active(self):
return [_Run("seat-1"), _Run("seat-2"), _Run("leader"), _Run("intent")]
class _Threads:
async def get(self, thread_id: str, *, user_id=None):
rooms = {
"seat-1": {"metadata": {"roundtable_room_key": "room-1"}},
"seat-2": {"metadata": {"roundtable_room_key": "room-1"}},
"leader": {"metadata": {"roundtable_room_key": "room-1"}},
"intent": {"metadata": {"thread_type": "roundtable"}},
}
return rooms.get(thread_id)
request = SimpleNamespace(
app=SimpleNamespace(state=SimpleNamespace(run_manager=_Mgr(), thread_store=_Threads())),
)
keys = await roundtable_room_gate.extra_live_roundtable_keys(request)
assert set(keys) == {"room-1"}
@pytest.mark.asyncio
async def test_finished_runtime_task_with_stale_status_is_not_a_room(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
"deerflow.persistence.engine.get_session_factory",
lambda: None,
)
class _FinishedTask:
@staticmethod
def done() -> bool:
return True
class _Run:
run_id = "finished-run"
thread_id = "finished-thread"
task = _FinishedTask()
metadata = {"roundtable_room_key": "stale-room"}
class _Runs:
async def list_active(self):
return [_Run()]
request = SimpleNamespace(
app=SimpleNamespace(state=SimpleNamespace(run_manager=_Runs())),
)
assert await roundtable_room_gate.extra_live_roundtable_keys(request) == {}
@@ -12,6 +12,7 @@ from fastapi import HTTPException
from app.gateway.routers import skills as skills_router
from deerflow.persistence.skills import MemorySkillStore
from deerflow.skills import parser as skill_parser
from deerflow.skills.installer import SkillAlreadyExistsError
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
@@ -65,6 +66,41 @@ def _write_archive(tmp_path: Path, data: bytes, *, suffix: str = ".skill") -> Pa
return archive
def test_load_skills_caches_parsing_and_invalidates_on_file_change(
storage: LocalSkillStorage,
monkeypatch: pytest.MonkeyPatch,
) -> None:
skill_dir = storage.get_skills_root_path() / "public" / "cached-skill"
skill_dir.mkdir(parents=True)
skill_file = skill_dir / "SKILL.md"
skill_file.write_text(
"---\nname: cached-skill\ndescription: first version\n---\n# Cached skill\n",
encoding="utf-8",
)
calls = 0
original_parse = skill_parser.parse_skill_file
def counted_parse(*args, **kwargs):
nonlocal calls
calls += 1
return original_parse(*args, **kwargs)
monkeypatch.setattr(skill_parser, "parse_skill_file", counted_parse)
storage._skills_cache_check_interval_seconds = 0
assert [skill.name for skill in storage.load_skills()] == ["cached-skill"]
assert [skill.name for skill in storage.load_skills()] == ["cached-skill"]
assert calls == 1
skill_file.write_text(
"---\nname: cached-skill\ndescription: second version, longer\n---\n# Cached skill\n",
encoding="utf-8",
)
assert storage.load_skills()[0].description == "second version, longer"
assert calls == 2
# ---------- validate_skill_archive ----------
@@ -11,17 +11,25 @@ Covers:
from __future__ import annotations
import threading
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import yaml
from fastapi import HTTPException
from app.gateway.routers import skills as skills_router
from app.gateway.routers.skills import _get_editable_skill_or_raise, _get_owned_skill_or_raise
from deerflow.config import paths as paths_module
from deerflow.persistence.agents import MemoryAgentStore
from deerflow.persistence.notifications import MemoryNotificationStore, set_notification_store
from deerflow.persistence.skills import MemorySkillStore
from deerflow.persistence.tags import MemoryTagStore
from deerflow.skills import curator_log
from deerflow.skills.cascade import cascade_skill_unpublished_or_deleted
from deerflow.skills.types import Skill, SkillCategory
@pytest.fixture
@@ -599,3 +607,99 @@ def test_collect_referenced_skill_names_aggregates_from_disk(
monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: _FakePaths())
assert curator._collect_referenced_skill_names() == {"alpha", "beta", "gamma"}
@pytest.mark.asyncio
async def test_skill_listing_offloads_scan_and_keeps_disabled_visible(tmp_path, monkeypatch):
event_loop_thread = threading.get_ident()
config = object()
def skill(name, category, enabled):
return Skill(
name=name,
description=name,
license=None,
skill_dir=tmp_path / name,
skill_file=tmp_path / name / "SKILL.md",
relative_path=Path(name),
category=category,
enabled=enabled,
)
class Storage:
def load_skills(self, *, enabled_only):
assert threading.get_ident() != event_loop_thread
assert enabled_only is False
return [skill("disabled", SkillCategory.PUBLIC, False), skill("private", SkillCategory.CUSTOM, True)]
def configured_storage(*, app_config):
assert app_config is config
return Storage()
monkeypatch.setattr(skills_router, "get_or_new_skill_storage", configured_storage)
monkeypatch.setattr(skills_router, "_list_skill_display_adapters", AsyncMock(return_value={}))
monkeypatch.setattr("deerflow.skills.usage.get_entry", lambda _name: {})
monkeypatch.setattr("deerflow.skills.usage.latest_activity_at", lambda _name: None)
request = SimpleNamespace(state=SimpleNamespace(user=SimpleNamespace(id="user-a", system_role="user")))
result = await skills_router.list_skills(
request=request,
config=config,
skill_store=MemorySkillStore(),
tag_store=MemoryTagStore(),
search=None,
tag_ids=None,
square=None,
scope=None,
sort="name_asc",
admin=False,
)
assert [item.name for item in result.skills] == ["disabled"]
assert result.skills[0].enabled is False
@pytest.mark.asyncio
async def test_mysql_skill_listing_uses_normalized_projection_without_reading_skill_files(monkeypatch):
catalog = [
{
"name": "snapshot-skill",
"description": "loaded from the prepared catalogue",
"license": "MIT",
"category": "public",
"enabled": True,
"created_at": "2026-10-25T10:00:00+00:00",
}
]
async def load_projection(*, user_id, admin=False):
assert user_id == "user-a"
assert admin is False
return catalog
def fail_filesystem_read(**_kwargs):
raise AssertionError("MySQL-backed skill listing must not scan skill files")
monkeypatch.setattr(skills_router, "get_session_factory", lambda: object())
monkeypatch.setattr(skills_router, "load_skill_catalog_projection", load_projection)
monkeypatch.setattr(skills_router, "get_or_new_skill_storage", fail_filesystem_read)
monkeypatch.setattr(skills_router, "_list_skill_display_adapters", AsyncMock(return_value={}))
monkeypatch.setattr(skills_router, "_attach_owner_usernames", AsyncMock(side_effect=lambda rows: rows))
request = SimpleNamespace(state=SimpleNamespace(user=SimpleNamespace(id="user-a", system_role="user")))
result = await skills_router.list_skills(
request=request,
config=object(),
skill_store=MemorySkillStore(),
tag_store=MemoryTagStore(),
search=None,
tag_ids=None,
square=None,
scope=None,
sort="time_desc",
admin=False,
)
assert len(result.skills) == 1
assert result.skills[0].name == "snapshot-skill"
assert result.skills[0].license == "MIT"
@@ -0,0 +1,106 @@
"""Tests for the smart skill-list search matcher.
``GET /api/skills?search=`` used to require the whole query as a contiguous
substring of the English name / Chinese name only — searching by what a skill
*does* (description, detail, tags) or with several keywords always came back
empty. These tests pin the intended behaviour:
- whitespace-separated keywords, AND across tokens but each token may hit a
different field;
- fields: name, name_zh, description, detail, tag names;
- case-insensitive, and still works as a plain single-keyword substring.
"""
from __future__ import annotations
import pytest
from app.gateway.routers.skills import (
SkillResponse,
_attach_and_filter_tags,
_skill_matches_tokens,
_skill_search_tokens,
)
from deerflow.persistence.tags import MemoryTagStore
from deerflow.skills.types import SkillCategory
def _resp(
name: str,
*,
name_zh: str | None = None,
description: str = "",
detail: str | None = None,
) -> SkillResponse:
return SkillResponse(
name=name,
name_zh=name_zh,
description=description,
detail=detail,
category=SkillCategory.CUSTOM,
)
def test_tokens_split_and_lowercase() -> None:
assert _skill_search_tokens(None) == []
assert _skill_search_tokens(" ") == []
assert _skill_search_tokens(" Report 生成 ") == ["report", "生成"]
def test_match_requires_every_token_somewhere() -> None:
skill = _resp("web-search", name_zh="网页搜索", description="Search the public web for reports")
# One token hits the name, the other the description — old contiguous
# substring semantics would have rejected this.
assert _skill_matches_tokens(skill, ["web", "report"]) is True
# A token nowhere present rejects the skill even though the other matches.
assert _skill_matches_tokens(skill, ["web", "missing"]) is False
def test_match_hits_name_zh_description_and_detail() -> None:
assert _skill_matches_tokens(_resp("ws", name_zh="舆情分析"), ["舆情"]) is True
assert _skill_matches_tokens(_resp("ws", description="生成任务研判报告"), ["研判"]) is True
assert _skill_matches_tokens(_resp("ws", detail="本技能用于军事情报整编"), ["整编"]) is True
def test_match_is_case_insensitive_substring() -> None:
skill = _resp("Task-Report-Builder", description="builds reports")
# Tokens reaching the matcher are pre-lowercased by _skill_search_tokens.
tokens = _skill_search_tokens("Task-Report BUILDER")
assert tokens == ["task-report", "builder"]
assert _skill_matches_tokens(skill, tokens) is True
def test_empty_tokens_match_everything() -> None:
assert _skill_matches_tokens(_resp("anything"), []) is True
@pytest.mark.asyncio
async def test_attach_and_filter_searches_tags() -> None:
store = MemoryTagStore()
await store.create_tag({"id": "t1", "name": "情报整编", "tag_type": "skill"})
await store.set_target_tags("skill", "alpha", ["t1"])
responses = [
_resp("alpha", description="unrelated text"),
_resp("beta", description="unrelated text"),
]
kept = await _attach_and_filter_tags(store, responses, "整编", None)
assert [r.name for r in kept] == ["alpha"]
# The attached tags ride along on the response.
assert [t["name"] for t in kept[0].tags] == ["情报整编"]
@pytest.mark.asyncio
async def test_attach_and_filter_keeps_legacy_single_keyword_behavior() -> None:
store = MemoryTagStore()
responses = [
_resp("web-search", name_zh="网页搜索", description="search the web"),
_resp("code-lint", name_zh="代码检查", description="lint python files"),
]
kept = await _attach_and_filter_tags(store, responses, "web", None)
assert [r.name for r in kept] == ["web-search"]
kept = await _attach_and_filter_tags(store, responses, "代码", None)
assert [r.name for r in kept] == ["code-lint"]
File diff suppressed because it is too large. Load diff
@@ -188,3 +188,81 @@ async def test_admin_can_select_html_map_webpage(
assert response.engine == "leaflet"
assert saved[0].html_map.engine == "leaflet"
def test_retry_keep_partial_defaults_to_enabled() -> None:
assert SystemSettings().retry_keep_partial.enabled is True
@pytest.mark.asyncio
async def test_retry_keep_partial_get_reflects_saved_settings(
monkeypatch: pytest.MonkeyPatch,
) -> None:
current = SystemSettings().model_copy(
update={"retry_keep_partial": system_settings_router.RetryKeepPartialSettings(enabled=False)}
)
monkeypatch.setattr(
system_settings_router,
"load_system_settings",
lambda: current,
)
response = await system_settings_router.get_retry_keep_partial_setting()
assert response.enabled is False
@pytest.mark.asyncio
async def test_non_admin_cannot_update_retry_keep_partial(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def ordinary_user(_request):
return SimpleNamespace(system_role="user")
monkeypatch.setattr(
system_settings_router,
"get_optional_user_from_request",
ordinary_user,
)
with pytest.raises(HTTPException) as error:
await system_settings_router.update_retry_keep_partial_setting(
system_settings_router.RetryKeepPartialUpdateRequest(enabled=False),
SimpleNamespace(),
)
assert error.value.status_code == 403
@pytest.mark.asyncio
async def test_admin_can_update_retry_keep_partial(
monkeypatch: pytest.MonkeyPatch,
) -> None:
current = SystemSettings()
saved: list[SystemSettings] = []
async def admin_user(_request):
return SimpleNamespace(system_role="admin")
monkeypatch.setattr(
system_settings_router,
"get_optional_user_from_request",
admin_user,
)
monkeypatch.setattr(
system_settings_router,
"load_system_settings",
lambda: current,
)
monkeypatch.setattr(
system_settings_router,
"save_system_settings",
lambda settings: saved.append(settings) or settings,
)
response = await system_settings_router.update_retry_keep_partial_setting(
system_settings_router.RetryKeepPartialUpdateRequest(enabled=False),
SimpleNamespace(),
)
assert response.enabled is False
assert saved[0].retry_keep_partial.enabled is False
@@ -111,6 +111,8 @@ def test_seed_assets_define_machine_readable_handoff_contracts() -> None:
for field in ("agentId", "score", "conclusion", "comment", "strengths", "risks"):
assert f'"{field}"' in judge_soul
assert '"recommendedAgentIds"' in judge_soul
assert "不得调用 `ask_clarification`" in judge_soul
assert "仍须一次性完成所有候选项的评判" in judge_soul
assert "/mnt/user-data/outputs/综合策略方案.md" in solution_soul
@@ -0,0 +1,201 @@
"""Stream truncation event persistence + model-layer classification tests."""
from __future__ import annotations
import asyncio
from datetime import UTC, datetime, timedelta
import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
import deerflow.persistence.stream_truncation as stream_truncation_module
from deerflow.models.patched_openai import _ToolCallStreamDiagnostics
from deerflow.persistence.base import Base
from deerflow.persistence.stream_truncation import (
SqlStreamTruncationStore,
StreamTruncationEvent,
StreamTruncationQuery,
)
@pytest_asyncio.fixture
async def session_factory(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'stream-truncation.db'}")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
factory = async_sessionmaker(engine, expire_on_commit=False)
yield factory
await engine.dispose()
def _event(**overrides) -> StreamTruncationEvent:
payload = dict(
id="evt-1",
created_at=datetime.now(UTC),
model_name="relay-model",
run_id="run-1",
event_type="silent_no_terminal",
finish_reason="none",
status="success",
chunk_count=12,
duration_ms=3_000,
first_chunk_ms=120,
)
payload.update(overrides)
return StreamTruncationEvent(**payload)
@pytest.mark.asyncio
async def test_record_list_count_and_breakdown(session_factory) -> None:
store = SqlStreamTruncationStore(session_factory)
await store.record(_event())
await store.record(
_event(
id="evt-2",
model_name="relay-model",
event_type="length_truncated",
finish_reason="length",
)
)
await store.record(_event(id="evt-3", model_name="other-model"))
items = await store.list(StreamTruncationQuery(limit=10))
assert len(items) == 3
total = await store.count(StreamTruncationQuery(model_name="relay-model"))
assert total == 2
filtered = await store.list(
StreamTruncationQuery(model_name="relay-model", event_type="length_truncated")
)
assert [item["id"] for item in filtered] == ["evt-2"]
breakdown = await store.model_breakdown(StreamTruncationQuery())
by_model = {item["model_name"]: item for item in breakdown}
assert by_model["relay-model"]["total"] == 2
assert by_model["relay-model"]["length_truncated"] == 1
assert by_model["relay-model"]["silent_no_terminal"] == 1
assert by_model["other-model"]["total"] == 1
assert by_model["relay-model"]["last_at"]
@pytest.mark.asyncio
async def test_retention_cap_deletes_rows_older_than_one_month(session_factory) -> None:
store = SqlStreamTruncationStore(session_factory)
old = _event(
id="evt-old",
created_at=datetime.now(UTC) - timedelta(days=40),
)
await store.record(old)
# The sweep after every insert must already have removed the 40-day-old row.
assert await store.count(StreamTruncationQuery()) == 0
fresh = _event(id="evt-fresh", created_at=datetime.now(UTC) - timedelta(days=3))
await store.record(fresh)
assert await store.count(StreamTruncationQuery()) == 1
purged = await store.purge_before(datetime.now(UTC) - timedelta(days=30))
assert purged == 0
assert await store.count(StreamTruncationQuery()) == 1
def _diagnostics(**overrides) -> _ToolCallStreamDiagnostics:
diagnostics = _ToolCallStreamDiagnostics(
model_name="relay-model",
run_id="run-abc",
strict_stream_integrity=False,
)
for key, value in overrides.items():
setattr(diagnostics, key, value)
return diagnostics
def _capture_recorded(monkeypatch: pytest.MonkeyPatch):
captured: list[dict] = []
def fake_record(**kwargs):
captured.append(kwargs)
monkeypatch.setattr(
stream_truncation_module,
"record_stream_truncation_event",
fake_record,
)
return captured
def _flush(diagnostics: _ToolCallStreamDiagnostics, *, status: str = "success", error=None, error_type: str = "none") -> None:
diagnostics._record_truncation_event(
status=status,
error=error,
error_type=error_type,
total_ms=1_500,
first_chunk_ms=100,
)
def test_classification_silent_no_terminal(monkeypatch: pytest.MonkeyPatch) -> None:
captured = _capture_recorded(monkeypatch)
diagnostics = _diagnostics(chunk_count=8)
_flush(diagnostics)
assert len(captured) == 1
assert captured[0]["event_type"] == "silent_no_terminal"
assert captured[0]["finish_reason"] == "none"
assert captured[0]["model_name"] == "relay-model"
assert captured[0]["strict_mode"] is False
assert captured[0]["run_id"] == "run-abc"
def test_classification_skips_clean_and_empty_streams(monkeypatch: pytest.MonkeyPatch) -> None:
captured = _capture_recorded(monkeypatch)
# Normal stop with terminal frame -> nothing to record.
_flush(_diagnostics(chunk_count=8, finish_reason="stop"))
# No terminal frame but zero chunks -> empty response, not a half answer.
_flush(_diagnostics(chunk_count=0))
# Cancelled by the user -> not a truncation.
_flush(_diagnostics(chunk_count=8), status="cancelled", error=asyncio.CancelledError())
assert captured == []
def test_classification_intentional_bridge_stop_is_skipped(monkeypatch: pytest.MonkeyPatch) -> None:
captured = _capture_recorded(monkeypatch)
_flush(_diagnostics(chunk_count=8, intentional_early_finish=True))
assert captured == []
def test_classification_strict_incomplete_and_midstream_errors(monkeypatch: pytest.MonkeyPatch) -> None:
captured = _capture_recorded(monkeypatch)
_flush(
_diagnostics(chunk_count=8, strict_stream_integrity=True),
status="error",
error=RuntimeError("incomplete"),
error_type="IncompleteChatStreamError",
)
_flush(
_diagnostics(chunk_count=30),
status="error",
error=TimeoutError(),
error_type="ReadTimeout",
)
# Errors before the first chunk are connection failures, not truncations.
_flush(
_diagnostics(chunk_count=0),
status="error",
error=TimeoutError(),
error_type="ReadTimeout",
)
assert [item["event_type"] for item in captured] == [
"incomplete_stream_error",
"midstream_error",
]
assert captured[0]["strict_mode"] is True
def test_classification_length_and_content_filter(monkeypatch: pytest.MonkeyPatch) -> None:
captured = _capture_recorded(monkeypatch)
_flush(_diagnostics(chunk_count=99, finish_reason="length"))
_flush(_diagnostics(chunk_count=99, finish_reason="content_filter"))
assert [item["event_type"] for item in captured] == [
"length_truncated",
"content_filter",
]
@@ -0,0 +1,317 @@
"""b5-1 task-detail persistence and API contract tests."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from pydantic import ValidationError
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from starlette.requests import Request
from app.gateway import auth_middleware, csrf_middleware
from app.gateway.routers import task_deeplink_details as router
from deerflow.persistence.task_deeplink_details import (
MemoryTaskDeeplinkDetailStore,
TaskDeeplinkDetailRepository,
)
from deerflow.persistence.task_deeplink_details.model import TaskDeeplinkDetailRow
def _request(store, *, role: str = "admin", external: bool = False, anonymous: bool = False):
state = SimpleNamespace(
user=None
if anonymous
else SimpleNamespace(id="admin-1", system_role=role)
)
if external:
state.task_deeplink_external_writer = True
return SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(task_deeplink_detail_store=store)), state=state)
def _http_request(method: str, path: str, headers: dict[str, str] | None = None) -> Request:
return Request(
{
"type": "http",
"http_version": "1.1",
"method": method,
"scheme": "http",
"path": path,
"query_string": b"",
"headers": [
(key.lower().encode("latin-1"), value.encode("latin-1"))
for key, value in (headers or {}).items()
],
"client": ("test", 1234),
"server": ("test", 80),
}
)
@pytest.mark.asyncio
async def test_memory_store_skips_identical_snapshot_writes_and_versions_changes() -> None:
store = MemoryTaskDeeplinkDetailStore()
first, created, changed = await store.upsert_detail(
"b5-1",
"task-1",
task_name="任务甲",
task_content="第一版详情",
overview="概述",
extra={"source": "external"},
updated_by="tester",
)
assert created is True
assert changed is True
assert first["version"] == 1
second, created, changed = await store.upsert_detail(
"b5-1",
"task-1",
task_name="任务甲",
task_content="第二版详情",
overview="",
extra={"priority": 3},
updated_by="tester-2",
)
assert created is False
assert changed is True
assert second["version"] == 2
assert second["extra"] == {"priority": 3}
same, created, changed = await store.upsert_detail(
"b5-1",
"task-1",
task_name="任务甲",
task_content="第二版详情",
overview="",
extra={"priority": 3},
updated_by="retry-writer",
)
assert created is False
assert changed is False
assert same["version"] == 2
assert same["updated_at"] == second["updated_at"]
assert await store.get_detail("b5-1", "task-2") is None
@pytest.mark.asyncio
async def test_memory_store_partitions_same_id_by_sub_type() -> None:
store = MemoryTaskDeeplinkDetailStore()
_, created_a, _ = await store.upsert_detail(
"b5-1",
"task-9",
sub_type="yuqing",
task_name="舆情任务",
task_content="舆情详情",
overview="",
extra={},
updated_by="writer",
)
_, created_b, _ = await store.upsert_detail(
"b5-1",
"task-9",
sub_type="other",
task_name="其它任务",
task_content="其它详情",
overview="",
extra={},
updated_by="writer",
)
assert created_a is True and created_b is True
yuqing = await store.get_detail("b5-1", "task-9", sub_type="yuqing")
other = await store.get_detail("b5-1", "task-9", sub_type="other")
untyped = await store.get_detail("b5-1", "task-9")
assert yuqing is not None and yuqing["task_name"] == "舆情任务" and yuqing["sub_type"] == "yuqing"
assert other is not None and other["task_name"] == "其它任务"
assert untyped is None
# 同类型重试幂等;跨类型互不覆盖。
_, created, changed = await store.upsert_detail(
"b5-1",
"task-9",
sub_type="yuqing",
task_name="舆情任务",
task_content="舆情详情",
overview="",
extra={},
updated_by="retry",
)
assert (created, changed) == (False, False)
assert (await store.get_detail("b5-1", "task-9", sub_type="other"))["version"] == 1
@pytest.mark.asyncio
async def test_sql_store_round_trips_detail_snapshot() -> None:
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
try:
async with engine.begin() as connection:
await connection.run_sync(TaskDeeplinkDetailRow.__table__.create)
store = TaskDeeplinkDetailRepository(async_sessionmaker(engine, expire_on_commit=False))
saved, created, changed = await store.upsert_detail(
"b5-1",
"task-42",
task_name="SQL 任务",
task_content="详情",
overview="概述",
extra={"tags": ["a", "b"]},
updated_by="admin",
)
assert created is True
assert changed is True
assert saved["detail_id"] == "task-42"
assert (await store.get_detail("b5-1", "task-42"))["extra"] == {"tags": ["a", "b"]}
same, created, changed = await store.upsert_detail(
"b5-1",
"task-42",
task_name="SQL 任务",
task_content="详情",
overview="概述",
extra={"tags": ["a", "b"]},
updated_by="retry-writer",
)
assert created is False
assert changed is False
assert same["version"] == 1
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_router_returns_camel_case_contract_and_allows_open_writes() -> None:
store = MemoryTaskDeeplinkDetailStore()
request = _request(store)
body = router.TaskDeeplinkDetailUpsertRequest.model_validate(
{
"taskName": "任务分析名称",
"taskContent": "这是任务详情",
"overview": "任务概述",
"extra": {"origin": "b5", "severity": 2},
}
)
saved = await router.upsert_task_deeplink_detail(request, body, "b5-1", "id-7")
assert saved.created is True
assert saved.changed is True
assert saved.data.task_name == "任务分析名称"
assert saved.data.extra == {"origin": "b5", "severity": 2}
fetched = await router.get_task_deeplink_detail(request, "b5-1", "id-7", type="")
assert fetched.data.detail_id == "id-7"
assert fetched.data.model_dump(by_alias=True)["taskName"] == "任务分析名称"
assert fetched.data.model_dump(by_alias=True)["type"] == "b5-1"
assert fetched.data.model_dump(by_alias=True)["taskType"] == ""
retry = await router.upsert_task_deeplink_detail(request, body, "b5-1", "id-7")
assert retry.created is False
assert retry.changed is False
assert retry.data.version == 1
# 写入端点对所有人开放:普通用户与未登录调用方都可直接写。
by_user = await router.upsert_task_deeplink_detail(
_request(store, role="user"), body, "b5-1", "id-8"
)
assert by_user.created is True
by_anonymous = await router.upsert_task_deeplink_detail(
_request(store, anonymous=True), body, "b5-1", "id-anonymous"
)
assert by_anonymous.created is True
stored = await store.get_detail("b5-1", "id-anonymous")
assert stored is not None and stored["updated_by"] == "external:task-deeplink"
@pytest.mark.asyncio
async def test_router_partitions_same_id_by_request_type() -> None:
store = MemoryTaskDeeplinkDetailStore()
request = _request(store)
typed_body = router.TaskDeeplinkDetailUpsertRequest.model_validate(
{"taskName": "舆情任务", "taskContent": "舆情详情", "type": "yuqing"}
)
saved = await router.upsert_task_deeplink_detail(request, typed_body, "b5-1", "task-x")
assert saved.created is True
assert saved.data.detail_sub_type == "yuqing"
assert saved.data.model_dump(by_alias=True)["taskType"] == "yuqing"
other_body = router.TaskDeeplinkDetailUpsertRequest.model_validate(
{"taskName": "行动任务", "taskContent": "行动详情", "type": "action"}
)
await router.upsert_task_deeplink_detail(request, other_body, "b5-1", "task-x")
fetched = await router.get_task_deeplink_detail(request, "b5-1", "task-x", type="yuqing")
assert fetched.data.task_name == "舆情任务"
fetched_other = await router.get_task_deeplink_detail(request, "b5-1", "task-x", type="action")
assert fetched_other.data.task_name == "行动任务"
# 该 id 未写无类型分区:不带 ?type= 读不到(不会回落到任意类型)。
with pytest.raises(HTTPException) as error:
await router.get_task_deeplink_detail(request, "b5-1", "task-x", type="")
assert error.value.status_code == 404
def test_router_rejects_blank_required_fields_and_non_object_extra() -> None:
with pytest.raises(ValidationError):
router.TaskDeeplinkDetailUpsertRequest.model_validate(
{"taskName": " ", "taskContent": "详情"}
)
with pytest.raises(ValidationError):
router.TaskDeeplinkDetailUpsertRequest.model_validate(
{"taskName": "名称", "taskContent": "详情", "extra": ["not", "an", "object"]}
)
def test_b5_put_is_credential_free_and_bypasses_csrf() -> None:
# 无 key、无登录态:PUT 直接放行并打上外部写入标记(供审计)。
bare = _http_request("PUT", "/api/task-deeplink-details/b5-1/id-9")
assert auth_middleware._is_task_deeplink_external_write(bare, bare.url.path)
assert auth_middleware._authorize_task_deeplink_external_write(bare) is None
assert bare.state.task_deeplink_external_writer is True
assert csrf_middleware.should_check_csrf(bare) is False
# 旧集成方带的 X-Integration-Key 不再校验(只记指纹),错误 key 也放行。
stale_key = _http_request(
"PUT",
"/api/task-deeplink-details/b5-1/id-9",
{"X-Integration-Key": "anything"},
)
assert auth_middleware._authorize_task_deeplink_external_write(stale_key) is None
assert stale_key.state.task_deeplink_external_key_fingerprint
# 通道只覆盖 PUT;其它方法仍走正常登录校验。
post_request = _http_request("POST", "/api/task-deeplink-details/b5-1/id-9")
assert not auth_middleware._is_task_deeplink_external_write(post_request, post_request.url.path)
assert csrf_middleware.should_check_csrf(post_request) is True
def test_unauthenticated_put_succeeds_but_read_still_requires_login() -> None:
app = FastAPI()
app.state.task_deeplink_detail_store = MemoryTaskDeeplinkDetailStore()
app.include_router(router.router)
app.add_middleware(auth_middleware.AuthMiddleware)
client = TestClient(app)
written = client.put(
"/api/task-deeplink-details/b5-1/task-open",
json={"taskName": "开放写入任务", "taskContent": "这是任务详情"},
)
assert written.status_code == 200
assert written.json()["created"] is True
# 同一 id 再写一个带类型的快照,互不覆盖(端到端)。
typed = client.put(
"/api/task-deeplink-details/b5-1/task-open",
json={"taskName": "同id不同类型", "taskContent": "类型详情", "type": "yuqing"},
)
assert typed.status_code == 200
assert typed.json()["data"]["taskType"] == "yuqing"
assert typed.json()["created"] is True
# 读取仍走正常登录校验:未登录 GET(无论带不带 type)→ 401,
# 不因写入开放而连带放开。
denied = client.get("/api/task-deeplink-details/b5-1/task-open")
assert denied.status_code == 401
denied_typed = client.get("/api/task-deeplink-details/b5-1/task-open?type=yuqing")
assert denied_typed.status_code == 401
@@ -7,14 +7,25 @@ from types import SimpleNamespace
import pytest
from app.gateway.routers.task_deeplink_threads import list_task_deeplink_threads
from deerflow.persistence.task_deeplink_threads.base import (
MAX_TASK_DEEPLINK_SUB_TYPE,
task_deeplink_binding,
)
class _HistoryStore:
def __init__(self) -> None:
self.calls: list[tuple[str, str, int]] = []
self.calls: list[tuple[str, str, int, str | None]] = []
async def list_threads(self, task_id: str, go_path: str, *, limit: int):
self.calls.append((task_id, go_path, limit))
async def list_threads(
self,
task_id: str,
go_path: str,
*,
limit: int,
sub_type: str | None = None,
):
self.calls.append((task_id, go_path, limit, sub_type))
return [
{
"thread_id": "thread-1",
@@ -32,23 +43,27 @@ class _HistoryStore:
]
@pytest.mark.asyncio
async def test_list_endpoint_trims_task_id_and_preserves_thread_shape():
store = _HistoryStore()
request = SimpleNamespace(
def _request_with(store: _HistoryStore) -> SimpleNamespace:
return SimpleNamespace(
app=SimpleNamespace(
state=SimpleNamespace(task_deeplink_thread_store=store),
)
)
@pytest.mark.asyncio
async def test_list_endpoint_trims_task_id_and_preserves_thread_shape():
store = _HistoryStore()
response = await list_task_deeplink_threads(
request,
_request_with(store),
task_id=" 108 ",
go_path="rwfx",
limit=25,
sub_type=None,
)
assert store.calls == [("108", "rwfx", 25)]
assert store.calls == [("108", "rwfx", 25, None)]
assert response.model_dump() == {
"threads": [
{
@@ -66,3 +81,106 @@ async def test_list_endpoint_trims_task_id_and_preserves_thread_shape():
}
]
}
@pytest.mark.asyncio
async def test_list_endpoint_passes_sub_type_through():
store = _HistoryStore()
await list_task_deeplink_threads(
_request_with(store),
task_id="task-1",
go_path="b5-1",
limit=100,
sub_type=" yuqing ",
)
# Explicitly empty sub_type must survive as "" (the no-type partition),
# not collapse to None (which would mean "every sub-type").
await list_task_deeplink_threads(
_request_with(store),
task_id="task-1",
go_path="b5-1",
limit=100,
sub_type="",
)
assert store.calls == [
("task-1", "b5-1", 100, "yuqing"),
("task-1", "b5-1", 100, ""),
]
@pytest.mark.asyncio
async def test_list_endpoint_accepts_a_custom_mapping_go_path() -> None:
store = _HistoryStore()
await list_task_deeplink_threads(
_request_with(store),
task_id="task-1",
go_path="区域研判",
limit=100,
sub_type=None,
)
assert store.calls == [("task-1", "区域研判", 100, None)]
def test_binding_extracts_b51_task_type_as_sub_type():
binding = task_deeplink_binding(
{
"taskId": "task-1",
"taskRecordId": "task-1",
"taskGoPath": "b5-1",
"taskType": "yuqing",
}
)
assert binding is not None
assert binding["sub_type"] == "yuqing"
def test_binding_defaults_missing_sub_type_to_empty_string():
for go_path in ("3qfx", "rwfx", "xdfx", "b5-1"):
binding = task_deeplink_binding(
{
"taskId": "task-1",
"taskRecordId": "task-1",
"taskGoPath": go_path,
}
)
assert binding is not None
assert binding["sub_type"] == ""
def test_binding_caps_sub_type_to_column_width():
binding = task_deeplink_binding(
{
"taskId": "task-1",
"taskRecordId": "task-1",
"taskGoPath": "b5-1",
"taskType": "x" * (MAX_TASK_DEEPLINK_SUB_TYPE + 10),
}
)
assert binding is not None
assert len(binding["sub_type"]) == MAX_TASK_DEEPLINK_SUB_TYPE
def test_binding_accepts_custom_mapping_go_path_but_not_its_a_suffix():
binding = task_deeplink_binding(
{
"taskId": "task-1",
"taskRecordId": "task-1",
"taskGoPath": "区域研判",
}
)
assert binding is not None
assert binding["go_path"] == "区域研判"
assert task_deeplink_binding(
{
"taskId": "task-1",
"taskRecordId": "task-1",
"taskGoPath": "区域研判-A",
}
) is None
@@ -0,0 +1,254 @@
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "threads-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_threads_page.py"
FAST_SCRIPT_PATH = SKILL_ROOT / "scripts" / "fast_page.py"
LEAD_PROMPT_PATH = BACKEND_ROOT / "packages" / "harness" / "deerflow" / "agents" / "lead_agent" / "prompt.py"
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"threads_page_renderer",
SCRIPT_PATH,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "threads-page-generator"
assert "Threads" in skill.description
discovered = LocalSkillStorage(host_path=str(SKILL_ROOT.parent.parent)).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert discovered_by_name["threads-page-generator"].category == SkillCategory.PUBLIC
assert discovered_by_name["threads-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/threads-page-generator/SKILL.md" in lead_prompt_source
assert "专用 Threads 路由" in lead_prompt_source
def test_default_json_renders_editable_threads_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "threads-template.html").read_text(encoding="utf-8")
runtime = (SKILL_ROOT / "assets" / "threads-runtime.js").read_text(encoding="utf-8")
html = renderer.render_document(data, template, runtime)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["threads"] >= 1
assert renderer.DATA_MARKER not in html
assert renderer.RUNTIME_MARKER not in html
assert 'id="threadsPageData"' in html
assert 'id="threadList"' in html
assert 'id="composeOverlay"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert ".thread-media,.image-grid" in html
assert 'data-thread-id="' in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert '"brand": "Threads"' in html
assert "<title>Threads · 串联想法</title>" in html
assert '"threads-page-data-v1:" + pageFingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "const cachedData = localStorage.getItem" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
data["site"]["pageTitle"] = "Threads · 智能体讨论"
titled_html = renderer.render_document(data, template, runtime)
assert "<title>Threads · 智能体讨论</title>" in titled_html
assert "<title>Threads · 串联想法</title>" not in titled_html
data["threads"]["items"][0]["text"] = "</script><script>alert('x')</script>"
escaped_html = renderer.render_document(data, template, runtime)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "threads-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(
renderer.DataValidationError,
match="Refusing to overwrite",
):
renderer._write_new(output, html)
def test_validation_rejects_missing_module_duplicate_and_remote_media() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data.pop("activity")
data["threads"]["items"][1]["id"] = data["threads"]["items"][0]["id"]
data["threads"]["items"][0]["media"]["type"] = "remote-image"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: activity" in message
assert "must be unique" in message
assert "media.type must be one of" in message
def test_remote_webpage_media_is_validated_and_supported_by_runtime() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["threads"]["items"][0]["media"] = {
"type": "webpage",
"pageUrl": "https://example.com/story",
"title": "演示网页",
"description": "串文中的网页卡片",
"embed": True,
}
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "threads-runtime.js").read_text(encoding="utf-8")
assert 'media.type === "webpage"' in runtime
assert 'sandbox="allow-forms allow-popups allow-scripts"' in runtime
def test_high_level_fast_spec_renders_once_without_deep_paths(tmp_path: Path) -> None:
spec_path = tmp_path / "threads-page-spec.json"
spec_path.write_text(
json.dumps(
{
"accountName": "茂达是🥚",
"handle": "@maoda",
"verified": True,
"postText": "今天想和大家分享茂达品牌日常,也听听大家的看法。",
"topic": "品牌日常",
"replyTopics": ["品牌表达", "产品故事"],
"displayReplyCount": 3,
"totalReplyCount": 128,
"replies": ["这个中文表达很自然,也很适合放在串文首页。"],
},
ensure_ascii=False,
),
encoding="utf-8",
)
command = [
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
]
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stdout + completed.stderr
result = json.loads(completed.stdout)
assert result["status"] == "ok"
assert result["action"] == "fast-threads-page"
data_path = Path(result["data"])
html_path = Path(result["html"])
assert data_path.name == "threads-page-data.json"
assert html_path.name == "threads-page.html"
data = json.loads(data_path.read_text(encoding="utf-8"))
assert data["site"]["currentUser"]["name"] == "茂达是🥚"
assert len(data["threads"]["items"]) == 1
thread = data["threads"]["items"][0]
assert thread["author"]["handle"] == "@maoda"
assert thread["author"]["verified"] is True
assert thread["metrics"]["replies"] == 128
assert len(thread["replies"]) == 3
assert thread["replies"][0]["text"] == "这个中文表达很自然,也很适合放在串文首页。"
assert thread["media"]["type"] == "autumn-grid"
assert "今天想和大家分享茂达品牌日常" in html_path.read_text(encoding="utf-8")
second = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert second.returncode == 0, second.stdout + second.stderr
second_result = json.loads(second.stdout)
assert Path(second_result["data"]).name == "threads-page-2-data.json"
assert Path(second_result["html"]).name == "threads-page-2.html"
def test_high_level_fast_spec_rejects_english_only_post(tmp_path: Path) -> None:
spec_path = tmp_path / "invalid-spec.json"
spec_path.write_text(
json.dumps(
{
"accountName": "茂达",
"postText": "This post should remain Chinese.",
},
ensure_ascii=False,
),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 2
result = json.loads(completed.stdout)
assert result["status"] == "error"
assert "must contain Simplified Chinese" in result["error"]
def test_threads_fast_workflow_forbids_generic_patch_paths() -> None:
skill_text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
prompt_text = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert '"accountName": "茂达是🥚"' in skill_text
assert "不得写六模块 JSON、`$set` 或深层字段路径" in skill_text
assert "Threads" in prompt_text
assert "禁止生成 `$set`、六模块补丁、深层字段路径" in prompt_text
@@ -0,0 +1,165 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "tiktok-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_tiktok_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"tiktok_page_renderer",
SCRIPT_PATH,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "tiktok-page-generator"
assert "TikTok" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["tiktok-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["tiktok-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert (
"/mnt/skills/public/tiktok-page-generator/SKILL.md"
in lead_prompt_source
)
assert "专用 TikTok 路由" in lead_prompt_source
def test_default_json_renders_editable_tiktok_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(
encoding="utf-8"
)
)
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "tiktok-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "tiktok-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["videos"] >= 1
assert renderer.DATA_MARKER not in html
assert renderer.RUNTIME_MARKER not in html
assert 'id="tiktokPageData"' in html
assert 'id="openSettings"' in html
assert 'id="settingsModal"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "tiktokInlineUpload" in html
assert 'host.closest(".video-card")' in html
assert "readAsDataURL" in html
assert '"brand": "TikTok"' in html
assert "<title>TikTok · 刷新此刻</title>" in html
assert '"tiktok-page-data-v1:" + pageFingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "const cachedData = localStorage.getItem" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
data["site"]["pageTitle"] = "TikTok · 智能体创作圈"
titled_html = renderer.render_document(data, template, runtime)
assert "<title>TikTok · 智能体创作圈</title>" in titled_html
assert "<title>TikTok · 刷新此刻</title>" not in titled_html
data["videos"]["items"][0]["text"] = (
"</script><script>alert('x')</script>"
)
escaped_html = renderer.render_document(data, template, runtime)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "tiktok-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(
renderer.DataValidationError,
match="Refusing to overwrite",
):
renderer._write_new(output, html)
def test_validation_rejects_missing_module_and_duplicate_videos() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(
encoding="utf-8"
)
)
data.pop("comments")
data["videos"]["items"][1]["id"] = data["videos"]["items"][0]["id"]
data["videos"]["items"][0]["visual"] = "remote-video"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: comments" in message
assert "must be unique" in message
assert "visual must be one of" in message
def test_remote_video_media_is_validated_and_supported_by_runtime() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["videos"]["items"][0]["media"] = {
"type": "video",
"mediaUrl": "https://cdn.example.com/short.mp4",
"posterUrl": "https://cdn.example.com/short.jpg",
}
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "tiktok-runtime.js").read_text(encoding="utf-8")
assert 'media.type === "video"' in runtime
assert "media.play().catch" in runtime
@@ -114,6 +114,46 @@ def _title_config() -> TitleConfig:
)
def test_admin_question_title_switch_finalizes_question_before_first_model_call(monkeypatch) -> None:
monkeypatch.setattr(
title_middleware_module,
"get_conversation_context_settings",
lambda: SimpleNamespace(use_question_as_title=True),
)
middleware = TitleMiddleware(
title_config=_title_config().model_copy(update={"enabled": False})
)
state = {
"messages": [SimpleNamespace(type="human", content="请帮我分析这份合同")],
}
result = middleware.before_model(state, runtime=SimpleNamespace())
assert result == {
"title": "请帮我分析这份合同",
"title_provisional": False,
}
def test_admin_question_title_switch_off_keeps_async_title_generation(monkeypatch) -> None:
monkeypatch.setattr(
title_middleware_module,
"get_conversation_context_settings",
lambda: SimpleNamespace(use_question_as_title=False),
)
middleware = TitleMiddleware(title_config=_title_config())
state = {
"messages": [SimpleNamespace(type="human", content="请帮我分析这份合同")],
}
result = middleware.before_model(state, runtime=SimpleNamespace())
assert result == {
"title": "请帮我分析这份合同",
"title_provisional": True,
}
@pytest.mark.asyncio
async def test_finalize_thread_title_updates_checkpoint_and_display_name(monkeypatch) -> None:
async def _fake_generate(self, state):
@@ -52,6 +52,7 @@ def _patch_config(monkeypatch: pytest.MonkeyPatch) -> None:
timeout_seconds=10.0,
verify_ssl=False,
ca_cert_path="",
mock_tokens={},
)
fake_cfg = types.SimpleNamespace(auth_login=types.SimpleNamespace(token_login=token_login))
monkeypatch.setattr(auth_module, "get_app_config", lambda: fake_cfg)
@@ -0,0 +1,407 @@
"""Uploads must not monopolize the gateway's event loop or worker capacity."""
from __future__ import annotations
import asyncio
import io
import threading
import time
from pathlib import Path
from types import SimpleNamespace
import httpx
import pytest
from fastapi import FastAPI, File, HTTPException, Request, UploadFile
from app.gateway.routers import uploads as uploads_router
from deerflow.persistence.upload_policy import MemoryUploadPolicyStore
from deerflow.runtime.user_context import get_current_user, require_current_user, reset_current_user, set_current_user
from deerflow.uploads.policy import default_upload_policy
async def _upload_app(monkeypatch, uploads_dir: Path, *, provider=None, separate_threads=False):
uploads_dir.mkdir(parents=True, exist_ok=True)
def ensure_uploads_dir(thread_id):
directory = uploads_dir / thread_id if separate_threads else uploads_dir
directory.mkdir(parents=True, exist_ok=True)
return directory
monkeypatch.setattr(uploads_router, "ensure_uploads_dir", ensure_uploads_dir)
monkeypatch.setattr(
uploads_router,
"get_paths",
lambda: SimpleNamespace(sandbox_uploads_dir=lambda thread_id, user_id=None: uploads_dir / thread_id if separate_threads else uploads_dir),
)
async def path_user(_thread_id):
return "user-1"
monkeypatch.setattr(uploads_router, "aresolve_path_user_id", path_user)
provider = provider or SimpleNamespace(uses_thread_data_mounts=True)
monkeypatch.setattr(uploads_router, "get_sandbox_provider", lambda: provider)
app = FastAPI()
store = MemoryUploadPolicyStore()
await store.ensure_seed(default_upload_policy())
app.state.upload_policy_store = store
app.state.upload_finished = False
@app.post("/upload/{thread_id}")
async def upload(thread_id: str, request: Request, files: list[UploadFile] = File(...)):
try:
return await uploads_router.upload_files.__wrapped__(thread_id, request, files, SimpleNamespace(uploads={"auto_convert_documents": False}))
finally:
app.state.upload_finished = True
@app.get("/probe")
async def probe():
return {"upload_finished": app.state.upload_finished}
return app
@pytest.mark.asyncio
@pytest.mark.parametrize("stage", ["write", "validate", "acquire", "sync", "commit", "cleanup"])
async def test_multifile_upload_keeps_other_requests_responsive(monkeypatch, tmp_path: Path, stage: str):
"""Exercise actual multipart requests alongside deterministic slow upload work.
Busy work simulates CPU-heavy validation; the other stages stand in for
slow storage and a remote sandbox. The ordering assertion avoids a fragile
machine-specific latency threshold while still failing on the old router.
"""
loop = asyncio.get_running_loop()
started = asyncio.Event()
started_at: list[float] = []
def slow_work():
if not started_at:
started_at.append(time.perf_counter())
loop.call_soon_threadsafe(started.set)
deadline = time.perf_counter() + 0.15
while time.perf_counter() < deadline:
pass
class Sandbox:
def update_file(self, _path, _content):
if stage == "sync":
slow_work()
def acquire(_thread_id):
if stage == "acquire":
slow_work()
return "sandbox-1"
provider = SimpleNamespace(uses_thread_data_mounts=False, acquire=acquire, get=lambda _id: Sandbox())
app = await _upload_app(monkeypatch, tmp_path / "uploads", provider=provider)
if stage == "write":
original_open = uploads_router.open_upload_file_no_symlink
class SlowWriter:
def __init__(self, handle):
self.handle = handle
def write(self, data):
slow_work()
return self.handle.write(data)
def __getattr__(self, name):
return getattr(self.handle, name)
def open_slow(*args, **kwargs):
path, handle = original_open(*args, **kwargs)
return path, SlowWriter(handle)
monkeypatch.setattr(uploads_router, "open_upload_file_no_symlink", open_slow)
elif stage in {"validate", "commit", "cleanup"}:
owner, name = {
"validate": (uploads_router, "validate_upload_content"),
"commit": (uploads_router, "_commit_staged_files"),
"cleanup": (uploads_router.shutil, "rmtree"),
}[stage]
original = getattr(owner, name)
def slow_call(*args, **kwargs):
slow_work()
return original(*args, **kwargs)
monkeypatch.setattr(owner, name, slow_call)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
async def probe_during_upload():
await asyncio.wait_for(started.wait(), timeout=5)
response = await client.get("/probe")
return response, time.perf_counter() - started_at[0]
upload, (probe, latency) = await asyncio.gather(
client.post(
"/upload/thread-1",
files=[("files", (f"sample-{index}.txt", b"test content", "text/plain")) for index in range(3)],
),
probe_during_upload(),
)
assert upload.status_code == 200, upload.text
assert len(upload.json()["files"]) == 3
assert probe.status_code == 200
print(f"{stage}: concurrent probe delay = {latency:.3f}s")
assert probe.json()["upload_finished"] is False, f"{stage} blocked other requests for {latency:.3f}s"
@pytest.mark.asyncio
async def test_upload_processing_has_a_global_concurrency_limit(monkeypatch, tmp_path: Path):
app = await _upload_app(monkeypatch, tmp_path / "uploads", separate_threads=True)
loop = asyncio.get_running_loop()
two_workers_started = asyncio.Event()
release = threading.Event()
lock = threading.Lock()
active = peak = entered = 0
original_validate = uploads_router.validate_upload_content
def validate(path, extension):
nonlocal active, peak, entered
with lock:
active += 1
entered += 1
peak = max(peak, active)
if active == 2:
loop.call_soon_threadsafe(two_workers_started.set)
try:
assert release.wait(timeout=5), "Upload workers were not released"
original_validate(path, extension)
finally:
with lock:
active -= 1
monkeypatch.setattr(uploads_router, "validate_upload_content", validate)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
uploads = [asyncio.create_task(client.post(f"/upload/thread-{index}", files={"files": ("file.txt", b"content")})) for index in range(6)]
try:
await asyncio.wait_for(two_workers_started.wait(), timeout=3)
await asyncio.sleep(0.05)
with lock:
assert entered == 2, "Excess uploads entered expensive work instead of waiting"
assert (await client.get("/probe")).status_code == 200
finally:
release.set()
results = await asyncio.gather(*uploads, return_exceptions=True)
assert all(isinstance(result, httpx.Response) and result.status_code == 200 for result in results), results
assert peak == 2
assert entered == 6
@pytest.mark.asyncio
@pytest.mark.parametrize("stage", ["validate", "sync"])
async def test_repeated_cancellation_waits_for_worker_and_restores_old_files(monkeypatch, tmp_path: Path, stage: str):
uploads_dir = tmp_path / "uploads"
loop = asyncio.get_running_loop()
started = asyncio.Event()
release = threading.Event()
original_validate = uploads_router.validate_upload_content
def pause_worker():
loop.call_soon_threadsafe(started.set)
assert release.wait(timeout=5), "Cancelled worker was not released"
class Sandbox:
def update_file(self, _path, _content):
if stage == "sync":
pause_worker()
provider = SimpleNamespace(uses_thread_data_mounts=False, acquire=lambda _id: "sandbox-1", get=lambda _id: Sandbox())
app = await _upload_app(monkeypatch, uploads_dir, provider=provider)
(uploads_dir / "same.txt").write_bytes(b"old content")
def validate(path, extension):
if stage == "validate":
pause_worker()
original_validate(path, extension)
monkeypatch.setattr(uploads_router, "validate_upload_content", validate)
upload_file = UploadFile(filename="same.txt", file=io.BytesIO(b"new content"))
task = asyncio.create_task(uploads_router.upload_files.__wrapped__("thread-1", SimpleNamespace(app=app), [upload_file], SimpleNamespace(uploads={"auto_convert_documents": False})))
try:
await asyncio.wait_for(started.wait(), timeout=3)
task.cancel()
await asyncio.sleep(0)
assert not task.done(), "Cancellation detached a still-running upload worker"
task.cancel()
await asyncio.sleep(0)
assert not task.done(), "A second cancellation detached the upload worker"
assert not upload_file.file.closed
finally:
release.set()
results = await asyncio.gather(task, return_exceptions=True)
await upload_file.close()
assert isinstance(results[0], asyncio.CancelledError)
assert (uploads_dir / "same.txt").read_bytes() == b"old content"
assert sorted(path.name for path in uploads_dir.iterdir()) == ["same.txt"]
@pytest.mark.asyncio
async def test_upload_worker_preserves_each_requests_user_context(monkeypatch, tmp_path: Path):
app = await _upload_app(monkeypatch, tmp_path / "uploads", separate_threads=True)
original_ensure = uploads_router.ensure_uploads_dir
original_validate = uploads_router.validate_upload_content
observed = []
caller_thread = threading.get_ident()
previous_user = get_current_user()
def ensure_directory(thread_id):
assert require_current_user().id == thread_id
assert threading.get_ident() != caller_thread
return original_ensure(thread_id)
def validate(path, extension):
assert require_current_user().id == path.stem
observed.append(require_current_user().id)
original_validate(path, extension)
monkeypatch.setattr(uploads_router, "ensure_uploads_dir", ensure_directory)
monkeypatch.setattr(uploads_router, "validate_upload_content", validate)
async def upload_as(user_id):
token = set_current_user(SimpleNamespace(id=user_id))
upload_file = UploadFile(filename=f"{user_id}.txt", file=io.BytesIO(b"content"))
try:
return await uploads_router.upload_files.__wrapped__(user_id, SimpleNamespace(app=app), [upload_file], SimpleNamespace(uploads={"auto_convert_documents": False}))
finally:
await upload_file.close()
reset_current_user(token)
results = await asyncio.gather(upload_as("user-alpha"), upload_as("user-beta"))
assert all(result.success for result in results)
assert sorted(observed) == ["user-alpha", "user-beta"]
assert get_current_user() is previous_user
@pytest.mark.asyncio
@pytest.mark.parametrize("collision", [False, True])
async def test_worker_conversion_sidecars_and_collision_rollback(monkeypatch, tmp_path: Path, collision: bool):
uploads_dir = tmp_path / "uploads"
app = await _upload_app(monkeypatch, uploads_dir)
(uploads_dir / "report.pdf").write_bytes(b"old pdf")
(uploads_dir / "report.pdf.md").write_bytes(b"old markdown")
caller_thread = threading.get_ident()
def convert(path):
assert threading.get_ident() != caller_thread
sidecar = path.with_name(f"{path.name}.md")
sidecar.write_text("# converted", encoding="utf-8")
return sidecar
monkeypatch.setattr(uploads_router, "convert_file_to_markdown_sync", convert)
files = [UploadFile(filename="report.pdf", file=io.BytesIO(b"%PDF-1.7\n"))]
if collision:
files.append(UploadFile(filename="report.pdf.md", file=io.BytesIO(b"user markdown")))
try:
operation = uploads_router.upload_files.__wrapped__("thread-1", SimpleNamespace(app=app), files, SimpleNamespace(uploads={"auto_convert_documents": True}))
if collision:
with pytest.raises(HTTPException) as exc_info:
await operation
assert exc_info.value.status_code == 409
assert (uploads_dir / "report.pdf").read_bytes() == b"old pdf"
assert (uploads_dir / "report.pdf.md").read_bytes() == b"old markdown"
else:
result = await operation
assert result.success
assert result.files[0]["markdown_file"] == "report.pdf.md"
assert (uploads_dir / "report.pdf.md").read_text(encoding="utf-8") == "# converted"
assert (uploads_dir / "report.pdf").read_bytes() == b"%PDF-1.7\n"
finally:
for upload_file in files:
await upload_file.close()
assert sorted(path.name for path in uploads_dir.iterdir()) == ["report.pdf", "report.pdf.md"]
@pytest.mark.asyncio
async def test_same_thread_uploads_cannot_undo_each_others_files(monkeypatch, tmp_path: Path):
uploads_dir = tmp_path / "uploads"
loop = asyncio.get_running_loop()
first_sync_started = asyncio.Event()
release = threading.Event()
class Sandbox:
def update_file(self, _path, content):
if content == b"first":
loop.call_soon_threadsafe(first_sync_started.set)
assert release.wait(timeout=5), "First upload was not released"
raise OSError("First sandbox sync failed")
provider = SimpleNamespace(uses_thread_data_mounts=False, acquire=lambda _id: "sandbox-1", get=lambda _id: Sandbox())
app = await _upload_app(monkeypatch, uploads_dir, provider=provider)
(uploads_dir / "same.txt").write_bytes(b"original")
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
first = asyncio.create_task(client.post("/upload/thread-1", files={"files": ("same.txt", b"first")}))
tasks = [first]
try:
await asyncio.wait_for(first_sync_started.wait(), timeout=3)
second = asyncio.create_task(client.post("/upload/thread-1", files={"files": ("same.txt", b"second")}))
tasks.append(second)
await asyncio.sleep(0.05)
assert not second.done(), "Overlapping upload modified a batch that can still roll back"
finally:
release.set()
results = await asyncio.gather(*tasks, return_exceptions=True)
assert [result.status_code for result in results] == [500, 200]
assert (uploads_dir / "same.txt").read_bytes() == b"second"
assert sorted(path.name for path in uploads_dir.iterdir()) == ["same.txt"]
@pytest.mark.asyncio
@pytest.mark.parametrize("operation", ["list", "delete"])
async def test_list_and_delete_wait_for_upload_rollback(monkeypatch, tmp_path: Path, operation: str):
"""No reader sees an uncommitted batch, and rollback never revives a deletion."""
uploads_dir = tmp_path / "uploads"
loop = asyncio.get_running_loop()
sync_started = asyncio.Event()
release = threading.Event()
class Sandbox:
def update_file(self, _path, _content):
loop.call_soon_threadsafe(sync_started.set)
assert release.wait(timeout=5), "Upload sync was not released"
raise OSError("Sandbox sync failed after the host commit")
provider = SimpleNamespace(uses_thread_data_mounts=False, acquire=lambda _id: "sandbox-1", get=lambda _id: Sandbox())
app = await _upload_app(monkeypatch, uploads_dir, provider=provider)
monkeypatch.setattr(uploads_router, "get_uploads_dir", lambda _thread_id: uploads_dir)
(uploads_dir / "same.txt").write_bytes(b"old")
request = SimpleNamespace(app=app)
files = [
UploadFile(filename="same.txt", file=io.BytesIO(b"new content")),
UploadFile(filename="added.txt", file=io.BytesIO(b"new file")),
]
upload = asyncio.create_task(uploads_router.upload_files.__wrapped__("thread-1", request, files, SimpleNamespace(uploads={"auto_convert_documents": False})))
tasks = [upload]
try:
await asyncio.wait_for(sync_started.wait(), timeout=3)
if operation == "list":
followup = asyncio.create_task(uploads_router.list_uploaded_files.__wrapped__("thread-1", request))
else:
followup = asyncio.create_task(uploads_router.delete_uploaded_file.__wrapped__("thread-1", "same.txt", request))
tasks.append(followup)
await asyncio.sleep(0.05)
assert not followup.done(), f"{operation} accessed files before the upload could finish rolling back"
finally:
release.set()
results = await asyncio.gather(*tasks, return_exceptions=True)
for upload_file in files:
await upload_file.close()
assert isinstance(results[0], HTTPException)
assert results[0].status_code == 500
result = results[1]
assert isinstance(result, dict)
if operation == "list":
assert result["count"] == 1
assert [(file["filename"], file["size"]) for file in result["files"]] == [("same.txt", "3")]
assert (uploads_dir / "same.txt").read_bytes() == b"old"
else:
assert result["success"]
assert list(uploads_dir.iterdir()) == []
assert not (uploads_dir / "added.txt").exists()
@@ -172,6 +172,31 @@ async def test_username_login_uses_oldest_account_when_canonical_duplicate_exist
assert user.id == older.id
@pytest.mark.asyncio
async def test_username_login_can_reuse_password_lookup_result(
monkeypatch: pytest.MonkeyPatch,
) -> None:
repo = FakeUserRepository()
seeded = _seed(repo, "alice@cm.com")
calls = {"find": 0}
original = repo.find_user_by_email_local_part
async def counted_find(local_part: str):
calls["find"] += 1
return await original(local_part)
monkeypatch.setattr(repo, "find_user_by_email_local_part", counted_find)
monkeypatch.setattr(auth_module, "get_local_provider", lambda: LocalAuthProvider(repo))
existing = await _lookup_user_by_username("alice")
assert existing is not None and existing.id == seeded.id
user, created = await _get_or_create_user_by_username("alice", existing_user=existing)
assert user.id == seeded.id
assert created is False
assert calls["find"] == 1
@pytest.mark.asyncio
async def test_password_lookup_uses_same_stable_username_account(
monkeypatch: pytest.MonkeyPatch,
@@ -0,0 +1,152 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "white-house-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_white_house_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"white_house_page_renderer", SCRIPT_PATH
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "white-house-page-generator"
assert "白宫" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["white-house-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["white-house-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/white-house-page-generator/SKILL.md" in lead_prompt_source
assert "专用白宫官网路由" in lead_prompt_source
def test_default_json_renders_editable_white_house_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
counts = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "white-house-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "white-house-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert counts["updates"] == 5
assert counts["priorities"] == 4
assert counts["months"] == 3
assert "__WHITE_HOUSE_PAGE_DATA__" not in html
assert "__WHITE_HOUSE_PAGE_RUNTIME__" not in html
assert 'id="whiteHousePageData"' in html
assert 'id="priorityGrid"' in html
assert 'id="menuOverlay"' in html
assert 'id="searchOverlay"' in html
assert 'id="liveOverlay"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert '"brandName":"美国白宫"' in html
assert '"language":"zh-CN"' in html
assert "JSON 编辑器" in html
assert '"white-house-page-data-v1:" + fingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "localStorage.getItem('executive-theme')" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
output = tmp_path / "white-house-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_visual_remote_url_and_english_copy() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
data.pop("footer")
data["priorities"]["items"][1]["id"] = data["priorities"]["items"][0]["id"]
data["updates"]["items"][0]["visual"] = "remote-image"
data["site"]["description"] = "https://example.com/remote"
data["hero"]["title"] = "For the People"
data["site"]["language"] = "en"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "missing top-level module(s): footer" in message
assert "duplicates" in message
assert "visual must be one of" in message
assert "remote URLs" in message
assert "site.language must be zh-CN" in message
assert "hero.title must use Simplified Chinese reader-facing copy" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/hero.jpg", "alt": "白宫图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/briefing.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/briefing", "title": "简报原文", "embed": False},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["hero"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "white-house-runtime.js").read_text(encoding="utf-8")
assert "externalMedia" in runtime
assert 'rel="noopener noreferrer"' in runtime
@@ -0,0 +1,86 @@
import asyncio
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from app.gateway.worker_pool.admission import Admission, bounded_worker_run, run_with_worker_admission
class Bridge:
def __init__(self): self.events = []
async def publish(self, run_id, event, data): self.events.append((run_id, event, data))
async def publish_end(self, run_id): self.events.append((run_id, "end", None))
async def cleanup(self, *args, **kwargs): pass
class Manager:
def __init__(self): self.records = {}
async def set_status(self, run_id, status, **kwargs): self.records[run_id].status = status
async def prepare(monkeypatch, capacity=4, queue=128, timeout=10):
async def owner(thread): return "owner"
monkeypatch.setattr("deerflow.runtime.thread_paths.aresolve_path_user_id", owner)
gate = Admission(capacity, queue, timeout); bridge = Bridge(); manager = Manager()
app = SimpleNamespace(state=SimpleNamespace(worker_admission=gate, worker_deferred_admission=True, stream_bridge=bridge, run_manager=manager))
return gate, bridge, manager, SimpleNamespace(app=app)
@pytest.mark.asyncio
async def test_70_streams_get_real_run_metadata_before_waiting_for_capacity(monkeypatch):
gate, bridge, manager, request = await prepare(monkeypatch)
release = asyncio.Event(); peak = 0
async def runner(**kwargs):
nonlocal peak
peak = max(peak, gate.active)
await release.wait()
kwargs["record"].status = "success"
@bounded_worker_run
async def start(body, thread_id, request):
record = SimpleNamespace(run_id=thread_id, thread_id=thread_id, status="pending")
manager.records[thread_id] = record
record.task = asyncio.create_task(run_with_worker_admission(runner, bridge=bridge, run_manager=manager, record=record))
return record
records = await asyncio.wait_for(asyncio.gather(*(start(None, str(i), request) for i in range(70))), 1)
await asyncio.sleep(.02)
assert len([e for e in bridge.events if e[1] == "metadata"]) == 70
assert gate.active == 4 and gate.waiting == 66
assert all(not r.task.done() for r in records)
release.set(); await asyncio.gather(*(r.task for r in records)); await asyncio.sleep(0)
assert peak == 4 and gate.active == gate.waiting == 0
@pytest.mark.asyncio
async def test_queued_cancel_timeout_and_overflow_finish_stream_without_capacity_leak(monkeypatch):
gate, bridge, manager, request = await prepare(monkeypatch, capacity=1, queue=2, timeout=.04)
release = asyncio.Event()
async def runner(**kwargs): await release.wait()
@bounded_worker_run
async def start(body, thread_id, request):
record = SimpleNamespace(run_id=thread_id, thread_id=thread_id, status="pending")
manager.records[thread_id] = record
record.task = asyncio.create_task(run_with_worker_admission(runner, bridge=bridge, run_manager=manager, record=record))
return record
first = await start(None, "one", request)
second = await start(None, "two", request)
third = await start(None, "three", request)
with pytest.raises(HTTPException) as exc: await start(None, "overflow", request)
assert exc.value.status_code == 429
await asyncio.sleep(.01); second.task.cancel()
await asyncio.gather(second.task, third.task, return_exceptions=True)
assert manager.records["two"].status.value == "interrupted"
assert manager.records["three"].status.value == "error"
assert any(e[0] == "three" and e[1] == "end" for e in bridge.events)
assert gate.active == 1 and gate.waiting == 0
release.set(); await first.task; await asyncio.sleep(0)
assert gate.active == 0
@pytest.mark.asyncio
async def test_preparation_exception_releases_reserved_place(monkeypatch):
gate, bridge, manager, request = await prepare(monkeypatch)
@bounded_worker_run
async def fail(*args): raise HTTPException(409, "conflict")
with pytest.raises(HTTPException): await fail(None, "one", request)
assert gate.active == gate.waiting == 0
@@ -0,0 +1,212 @@
import asyncio
import struct
import time
from types import SimpleNamespace
import pytest
from app.gateway.worker_pool.diagnostics import Diagnostics, container_metrics, redact_logs
from app.gateway.worker_pool.docker import DockerError, decode_docker_output, worker_spec
from app.gateway.worker_pool.host_metrics import rates, read_snapshot
from app.gateway.worker_pool.preflight import postgres_connection_string
from app.gateway.worker_pool.settings import PoolSettings, SettingsStore
from app.gateway.worker_pool.telemetry import RequestRecorder, RequestTelemetryMiddleware, route_name
def test_metrics_rates_use_elapsed_time_and_first_sample_is_unknown():
def sample(at, total, idle, read, rx):
return {
"available": True,
"at": at,
"cpu_total": total,
"cpu_idle": idle,
"cpu_iowait": 5,
"disks": {"sda": {"read_bytes": read, "write_bytes": 100, "io_ms": 100, "inflight": 0}},
"interfaces": {"eth0": {"rx_bytes": rx, "tx_bytes": 100}},
}
first = sample(1, 100, 50, 100, 100)
assert rates(first, None)["cpu_percent"] is None
result = rates(sample(3, 200, 75, 500, 900), first)
assert result["cpu_percent"] == 75
assert result["disks"]["sda"]["read_bps"] == 200
assert result["interfaces"]["eth0"]["rx_bps"] == 400
assert rates(sample(3, 200, 75, 500, 900), first)["iowait_percent"] == 0
def test_request_telemetry_aggregates_routes_per_execution_node():
recorder = RequestRecorder()
now = time.time()
for node, elapsed in (("main", 8), ("dfw-worker-1", 120)):
recorder.finish({
"id": node,
"started": now,
"method": "GET",
"path": "/api/threads/{thread_id}/state",
"status": 200,
"stream": False,
"headers_ms": elapsed,
"total_ms": elapsed,
"node": node,
})
summary = recorder.summary()
assert {(row["node"], row["max_ms"]) for row in summary["routes"]} == {
("main", 8),
("dfw-worker-1", 120),
}
assert recorder.worker_status()["count"] == 2
def test_host_parser_reads_host_network_not_container_network(tmp_path):
proc = tmp_path / "proc"
sys = tmp_path / "sys"
(proc / "1/net").mkdir(parents=True)
(sys / "class/block/sda1").mkdir(parents=True)
(sys / "class/block/sda1/partition").write_text("1")
(proc / "stat").write_text("cpu 10 0 20 100 5 0 0 0 0\ncpu0 1 0 0 0\ncpu1 1 0 0 0\n")
(proc / "meminfo").write_text("MemTotal: 1000 kB\nMemAvailable: 300 kB\n")
(proc / "loadavg").write_text("1.5 1.0 0.5 1/1 1")
(proc / "diskstats").write_text("8 0 sda 1 0 2 0 1 0 4 0 0 10 10\n8 1 sda1 1 0 2 0 1 0 4 0 0 10 10\n")
(proc / "1/net/dev").write_text("header\nheader\n eth0: 100 1 0 0 0 0 0 0 200 1 0 0 0 0 0 0\n veth1: 100 1 0 0 0 0 0 0 100 1 0 0 0 0 0 0\n")
result = read_snapshot(proc, sys)
assert result["cpu_count"] == 2
assert result["memory_available"] == 300 * 1024
assert list(result["interfaces"]) == ["eth0"]
assert list(result["disks"]) == ["sda"]
assert not read_snapshot(tmp_path / "absent", sys)["available"]
def test_docker_metrics_and_log_frames_and_secret_redaction():
detail = {"Name": "/node", "State": {"StartedAt": "today", "Health": {"Status": "healthy"}}, "HostConfig": {"NanoCpus": 2_000_000_000}}
stats = {
"cpu_stats": {"cpu_usage": {"total_usage": 300}, "system_cpu_usage": 1000, "online_cpus": 4},
"precpu_stats": {"cpu_usage": {"total_usage": 200}, "system_cpu_usage": 800},
"memory_stats": {"usage": 1000, "stats": {"inactive_file": 100}},
}
result = container_metrics(detail, stats)
assert result["cpu_percent"] == 200
assert result["memory_bytes"] == 900
value = b"password=secret\n"
frame = b"\x01\0\0\0" + struct.pack(">I", len(value)) + value
assert decode_docker_output(frame) == value.decode()
redacted = redact_logs('password=secret Authorization: Bearer hidden mysql://u:p@host/db api_key="abcdef" eyJabc.xxxxx.yyyyy')
assert all(secret not in redacted for secret in ("secret", "hidden", "u:p", "abcdef", "eyJabc"))
@pytest.mark.asyncio
async def test_stream_timing_does_not_buffer_or_read_request_body():
recorder = RequestRecorder()
recorder.enabled = True
started = asyncio.Event()
release = asyncio.Event()
bodies = []
async def app(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/event-stream")]})
await send({"type": "http.response.body", "body": b"event: metadata\n\n", "more_body": True})
started.set()
await release.wait()
await send({"type": "http.response.body", "body": b"event: end\n\n", "more_body": False})
async def receive():
pytest.fail("telemetry must never consume receive")
async def send(message):
if message["type"] == "http.response.body":
bodies.append(message["body"])
scope = {"type": "http", "app": SimpleNamespace(state=SimpleNamespace(request_recorder=recorder)), "method": "POST", "path": "/api/threads/private-id/runs/stream"}
task = asyncio.create_task(RequestTelemetryMiddleware(app)(scope, receive, send))
await started.wait()
assert bodies == [b"event: metadata\n\n"]
assert len(recorder.active) == 1 and not recorder.completed
release.set()
await task
row = recorder.completed[0]
assert row["stream"] and row["status"] == 200 and row["first_body_ms"] is not None
assert row["path"] == "/api/threads/{thread_id}/runs/stream"
assert not recorder.active
@pytest.mark.asyncio
async def test_disabled_or_monitor_request_is_not_recorded():
recorder = RequestRecorder()
async def app(scope, receive, send):
pass
scope = {"type": "http", "app": SimpleNamespace(state=SimpleNamespace(request_recorder=recorder)), "method": "GET", "path": "/api/models"}
await RequestTelemetryMiddleware(app)(scope, None, None)
recorder.enabled = True
scope["path"] = "/api/system-settings/worker-pool/diagnostics"
await RequestTelemetryMiddleware(app)(scope, None, None)
assert not recorder.active and not recorder.completed
@pytest.mark.asyncio
async def test_persistence_switch_archives_and_arbitrary_container_denied(tmp_path):
store = SettingsStore(tmp_path)
store.save(PoolSettings(source_container="main"))
pool = SimpleNamespace(store=store, nodes={})
app = SimpleNamespace(state=SimpleNamespace(config=None))
monitor = Diagnostics(pool, app)
await monitor.set_enabled(True)
monitor.recorder.finish({"id": "x", "started": 1, "method": "GET", "path": "/api/models", "status": 200, "stream": False, "total_ms": 2, "headers_ms": 1})
monitor.persist_requests()
class Engine:
def request(self, *args, **kwargs):
return b"password=do-not-keep\nhealthy"
await monitor.archive_logs(Engine(), "node")
archives = monitor.archives()
assert len(archives) == 1
assert "do-not-keep" not in str(monitor.read_archive(archives[0]["id"]))
with pytest.raises(DockerError):
monitor.read_archive("../../outside")
with pytest.raises(DockerError):
await monitor.logs("unrelated-database")
await monitor.close()
restored = Diagnostics(pool, app)
assert restored.enabled and len(restored.recorder.completed) == 1
await restored.set_enabled(False)
await restored.close()
@pytest.mark.asyncio
async def test_diagnostics_off_has_no_background_tasks_or_storage(tmp_path):
store = SettingsStore(tmp_path)
monitor = Diagnostics(SimpleNamespace(store=store, nodes={}), SimpleNamespace(state=SimpleNamespace(config=None)))
monitor.start()
assert monitor.task is None and monitor.lag_task is None
assert monitor.db is None
assert not (tmp_path / "diagnostics.sqlite3").exists()
assert not (tmp_path / "log-archive").exists()
await monitor.close()
def test_postgres_status_follows_actual_chat_configuration(monkeypatch):
monkeypatch.delenv("DEER_FLOW_USE_PG_CHECKPOINTER", raising=False)
db = SimpleNamespace(backend="mysql")
cp = SimpleNamespace(type="postgres", use_postgres=True, connection_string="postgresql://db/chat")
config = SimpleNamespace(database=db, checkpointer=cp)
assert postgres_connection_string(config) == cp.connection_string
cp.use_postgres = False
assert postgres_connection_string(config) is None
assert route_name("/api/threads/id/runs/run-id/stream") == "/api/threads/{thread_id}/runs/{run_id}/stream"
def test_worker_never_inherits_host_diagnostic_mounts():
source = {
"Image": "id",
"Config": {"Env": ["AUTH_JWT_SECRET=x"]},
"NetworkSettings": {"Networks": {"app": {}}},
"Mounts": [
{"Type": "bind", "Source": "/proc", "Destination": "/host/proc", "RW": False},
{"Type": "bind", "Source": "/sys", "Destination": "/host/sys", "RW": False},
{"Type": "bind", "Source": "/data", "Destination": "/app/data", "RW": True},
],
}
spec = worker_spec(source, PoolSettings(network="app"), "pool", "token", "/private")
assert [m["Target"] for m in spec["HostConfig"]["Mounts"]] == ["/app/data"]
@@ -0,0 +1,109 @@
import pytest
from app.gateway.worker_pool.discovery import discover, recommend
from app.gateway.worker_pool.settings import PoolSettings
def test_recommend_leaves_headroom_and_caps_starter():
result = recommend(32, 128 * 1024)
assert result["desired"] == 4
assert result["total_cpu"] == 8
assert result["total_memory_mb"] == 8192
@pytest.mark.parametrize("cpu,memory", [(1, 1024), (2, 2048), (0, 0)])
def test_small_host_has_no_viable_worker(cpu, memory):
assert recommend(cpu, memory)["desired"] == 0
def test_other_services_and_connections_limit_recommendation():
assert recommend(16, 32768, reserved_cpu=14)["desired"] == 1
assert recommend(32, 32768, reserved_memory_mb=30000)["desired"] == 0
assert recommend(32, 32768, connection_budget=12)["desired"] == 1
def test_discovery_excludes_own_workers_and_returns_no_secrets(monkeypatch):
def request(self, method, path):
if path == "/info":
return {"NCPU": 16, "MemTotal": 32768 * 1024**2}
if path == "/containers/json":
return [{"Id": "main", "Labels": {}}, {"Id": "worker", "Labels": {"deerflow.worker-pool": "mine"}}]
assert path == "/containers/main/json"
return {"Name": "/main", "Config": {"Env": ["PASSWORD=secret"]}, "HostConfig": {"NanoCpus": 2_000_000_000, "Memory": 4096 * 1024**2}, "NetworkSettings": {"Networks": {"bridge": {}, "app": {}}}}
monkeypatch.setattr("app.gateway.worker_pool.discovery.DockerEngine.request", request)
result = discover(PoolSettings(), "mine")
assert result["containers"] == [{"name": "main", "networks": ["app"]}]
assert result["recommendation"]["desired"] == 4
assert "secret" not in str(result)
@pytest.mark.asyncio
async def test_discovery_does_not_forward_saved_password_to_new_host(tmp_path, monkeypatch):
from types import SimpleNamespace
from app.gateway.worker_pool import router
from app.gateway.worker_pool.settings import SettingsStore
store = SettingsStore(tmp_path)
store.save(PoolSettings(host="old-host", username="root", host_key_sha256="SHA256:old", password="original-secret"))
async def controller(request): return SimpleNamespace(store=store)
monkeypatch.setattr(router, "controller", controller)
seen = []
def inspect(settings, instance):
seen.append(settings.password)
return {"ok": True}
monkeypatch.setattr("app.gateway.worker_pool.discovery.discover", inspect)
await router.discover_server(PoolSettings(host="new-host", username="root", host_key_sha256="SHA256:new"), None)
await router.discover_server(PoolSettings(host="old-host", username="root", host_key_sha256="SHA256:old"), None)
assert seen == [None, "original-secret"]
def test_ssh_host_key_mismatch_stops_before_authentication(monkeypatch):
import paramiko
from app.gateway.worker_pool.docker import DockerEngine, DockerError
authenticated = []
class Client:
def set_missing_host_key_policy(self, policy): self.policy = policy
def connect(self, *args, **kwargs):
from types import SimpleNamespace
self.policy.missing_host_key(self, "host", SimpleNamespace(asbytes=lambda: b"wrong-key"))
authenticated.append(kwargs["password"])
def close(self): pass
monkeypatch.setattr(paramiko, "SSHClient", Client)
with pytest.raises(DockerError, match="指纹不匹配"):
DockerEngine(PoolSettings(host="host", password="secret", host_key_sha256="SHA256:expected")).request("GET", "/info")
assert not authenticated
def test_snapshot_worker_runtime_commits_main_container_without_pausing(monkeypatch):
from app.gateway.worker_pool.docker import DockerEngine
calls = []
def request(self, method, path, body=None, **kwargs):
calls.append((method, path, body))
return {"Id": "sha256:worker-runtime-image"}
monkeypatch.setattr(DockerEngine, "request", request)
image = DockerEngine(PoolSettings()).snapshot_worker_runtime(
{"Id": "main-container-id"}, "instance123",
)
assert image == "sha256:worker-runtime-image"
assert calls == [(
"POST",
"/commit?container=main-container-id&repo=deerflow-worker-runtime-instance123&tag=main-contain&pause=false",
None,
)]
def test_snapshot_worker_runtime_reuses_image_for_tiny_writable_layer(monkeypatch):
from app.gateway.worker_pool.docker import DockerEngine
def unexpected_request(*args, **kwargs):
pytest.fail("a tiny writable layer should reuse the immutable image")
monkeypatch.setattr(DockerEngine, "request", unexpected_request)
image = DockerEngine(PoolSettings()).snapshot_worker_runtime(
{"Id": "main-container-id", "Image": "sha256:base-image", "SizeRw": 65536}, "instance123",
)
assert image == "sha256:base-image"
@@ -0,0 +1,874 @@
"""Managed-worker failure, affinity, admission, and 70-client regression tests."""
import asyncio
from types import SimpleNamespace
import httpx
import pytest
from fastapi import HTTPException
from app.gateway.worker_pool.admission import Admission, bounded_worker_run
from app.gateway.worker_pool.controller import Controller, PoolUnavailable
from app.gateway.worker_pool.docker import DockerError, worker_spec
from app.gateway.worker_pool.settings import PoolSettings, SettingsStore
def source():
return {
"Image": "sha256:fixed",
"Config": {"Env": ["AUTH_JWT_SECRET=fixed", "WEB_CONCURRENCY=8"], "WorkingDir": "/app"},
"NetworkSettings": {"Networks": {"deerflow": {}}},
"Mounts": [{"Type": "volume", "Name": "runtime", "Destination": "/data", "RW": True}, {"Type": "bind", "Source": "/var/run/docker.sock", "Destination": "/var/run/docker.sock", "RW": True}],
}
def test_secrets_encrypted_and_preserved(tmp_path):
store = SettingsStore(tmp_path)
store.save(PoolSettings(password="secret-sentinel"))
assert "secret-sentinel" not in store.path.read_text()
assert "password" not in store.public()
assert store.public()["password_set"]
store.save(PoolSettings(desired=2))
assert SettingsStore(tmp_path).load().password == "secret-sentinel"
assert store.load().desired == 2
def test_worker_template_has_limits_no_scheduler_no_socket():
spec = worker_spec(source(), PoolSettings(network="deerflow"), "instance", "token", "/private")
assert spec["Image"] == "sha256:fixed"
assert spec["HostConfig"]["Memory"] == 2048 * 1024**2
assert len(spec["HostConfig"]["Mounts"]) == 1
assert "SCHEDULED_TASKS_ENABLED=false" in spec["Env"]
assert not any(e.startswith("WEB_CONCURRENCY=") for e in spec["Env"])
assert "PortBindings" not in spec["HostConfig"]
assert spec["Cmd"][-2:] == ["--workers", "1"]
def test_worker_template_uses_snapshot_of_main_runtime():
spec = worker_spec(
source(), PoolSettings(network="deerflow"), "instance", "token", "/private",
runtime_image="sha256:runtime-snapshot",
)
assert spec["Image"] == "sha256:runtime-snapshot"
def test_postgres_worker_requires_shared_persistent_chat_archive_directory():
template = source()
template["Config"]["Env"].append("DEER_FLOW_HOME=/data")
spec = worker_spec(
template, PoolSettings(network="deerflow"), "instance", "token", "/private",
require_shared_archive=True,
)
assert any(mount["Target"] == "/data" and mount["ReadOnly"] is False for mount in spec["HostConfig"]["Mounts"])
template["Mounts"] = [
{"Type": "volume", "Name": "other", "Destination": "/other", "RW": True},
]
with pytest.raises(DockerError, match="共享的可写持久卷"):
worker_spec(
template, PoolSettings(network="deerflow"), "instance", "token", "/private",
require_shared_archive=True,
)
@pytest.mark.asyncio
async def test_live_storage_backend_change_reprobes_and_snapshots_main_runtime(tmp_path, monkeypatch):
from app.gateway.worker_pool import controller as controller_module
store = SettingsStore(tmp_path)
store.save(PoolSettings(
enabled=True, source_container="main", network="deerflow", desired=1,
))
pool = Controller(store)
storage_backends = iter(("sqlite", "postgres"))
discovered = []
snapshots = []
template = source()
template.update({
"Id": "main-container-id",
"Image": "sha256:base-image",
"Config": {"Env": ["AUTH_JWT_SECRET=fixed", "DEER_FLOW_HOME=/data"], "Labels": {}, "User": ""},
"NetworkSettings": {"Networks": {"deerflow": {}}},
})
class Engine:
def request(self, method, path, body=None):
assert method == "GET" and path == "/containers/main/json?size=true"
return template
def configured_storage_backend(self, source_detail, preferred, working_dir):
return next(storage_backends)
def discover_worker_python(self, source_detail, preferred, working_dir):
discovered.append(working_dir)
return "/opt/venv/bin/python"
def snapshot_worker_runtime(self, source_detail, instance):
image = f"sha256:runtime-{len(snapshots) + 1}"
snapshots.append((source_detail["Id"], instance))
return image
engine = Engine()
monkeypatch.setattr(controller_module, "DockerEngine", lambda settings: engine)
try:
_, sqlite_spec = await pool.inspect_template(store.load())
_, postgres_spec = await pool.inspect_template(store.load())
assert sqlite_spec["Image"] == "sha256:runtime-1"
assert postgres_spec["Image"] == "sha256:runtime-2"
assert len(discovered) == 2
assert len(snapshots) == 2
finally:
await pool.close()
def test_worker_keeps_service_networks_without_main_aliases_and_uses_private_health():
template = source()
template["NetworkSettings"]["Networks"]["knowledge"] = {"Aliases": ["main-only"], "IPAddress": "172.20.0.3"}
spec = worker_spec(template, PoolSettings(network="deerflow"), "instance", "token", "/private")
assert spec["NetworkingConfig"]["EndpointsConfig"] == {"deerflow": {}, "knowledge": {}}
health = spec["Healthcheck"]["Test"]
assert health[:3] == ["CMD", "/opt/venv/bin/python", "-c"]
assert "/internal/worker-status" in health[3]
assert "os.environ['DEER_FLOW_WORKER_TOKEN']" in health[3]
def test_dedicated_private_volume_is_excluded():
template = source()
template["Mounts"].append({"Type": "volume", "Name": "secrets", "Destination": "/private", "RW": True})
spec = worker_spec(template, PoolSettings(network="deerflow"), "instance", "token", "/private")
assert all(mount["Target"] != "/private" for mount in spec["HostConfig"]["Mounts"])
@pytest.mark.asyncio
async def test_orphaned_run_becomes_terminal_without_retry():
from app.gateway.worker_pool.manager import WorkerRunManager
from deerflow.runtime.runs.store.memory import MemoryRunStore
store = MemoryRunStore()
await store.put("old", thread_id="thread", status="running")
manager = WorkerRunManager(store=store)
assert await manager.get_persisted("old", authorized_thread_id="different-thread") is None
assert (await store.get("old"))["status"] == "running"
record = await manager.get_persisted("old", authorized_thread_id="thread")
assert record.status == "error"
assert (await store.get("old"))["status"] == "error"
assert record.task is None
def test_reject_shared_controller_secrets():
with pytest.raises(DockerError, match="密钥目录"):
worker_spec(source(), PoolSettings(network="deerflow"), "i", "t", "/data/control")
@pytest.mark.asyncio
async def test_70_simultaneous_runs_are_bounded_and_complete():
gate = Admission(4, 128, 10)
maximum = 0
completed = set()
async def run(i):
nonlocal maximum
await gate.acquire()
try:
maximum = max(maximum, gate.active)
await asyncio.sleep(0.003)
assert i not in completed
completed.add(i)
finally:
gate.release()
await asyncio.gather(*(run(i) for i in range(70)))
assert len(completed) == 70
assert maximum == 4
assert gate.active == gate.waiting == 0
@pytest.mark.asyncio
async def test_cancelled_waiter_and_timeout_do_not_leak_capacity():
gate = Admission(1, 1, 0.03)
await gate.acquire()
waiter = asyncio.create_task(gate.acquire())
await asyncio.sleep(0)
with pytest.raises(HTTPException) as error:
await gate.acquire()
assert error.value.status_code == 429
waiter.cancel()
await asyncio.gather(waiter, return_exceptions=True)
with pytest.raises(HTTPException):
await gate.acquire()
gate.release()
await gate.acquire()
gate.release()
assert gate.active == gate.waiting == 0
@pytest.mark.asyncio
async def test_background_run_holds_slot_after_http_returns():
gate = Admission(1)
event = asyncio.Event()
request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(worker_admission=gate)))
@bounded_worker_run
async def start(body, thread, request):
return SimpleNamespace(task=asyncio.create_task(event.wait()))
record = await start(None, "thread", request)
assert gate.active == 1
event.set()
await record.task
await asyncio.sleep(0)
assert gate.active == 0
@pytest.mark.asyncio
async def test_affinity_stable_on_scaleup_fail_closed_and_persisted(tmp_path):
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True))
pool = Controller(store)
pool.nodes["a"] = {"name": "a", "ready": True, "active": 0, "draining": False}
try:
first = await pool.reserve("thread", create=True)
pool.release("a")
pool.nodes["b"] = {"name": "b", "ready": True, "active": 0, "draining": False}
assert (await pool.reserve("thread", create=True))["name"] == first["name"]
pool.release("a")
pool.nodes["a"]["ready"] = False
with pytest.raises(PoolUnavailable):
await pool.reserve("thread", create=True)
assert pool._ensure_db().execute("SELECT node FROM affinity WHERE thread='thread'").fetchone()[0] == "a"
finally:
await pool.close()
restarted = Controller(store)
try:
with pytest.raises(PoolUnavailable):
await restarted.reserve("thread", create=True)
finally:
await restarted.close()
@pytest.mark.asyncio
async def test_scale_down_waits_for_active_runs_and_requests(tmp_path, monkeypatch):
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, source_container="main", network="deerflow", desired=0))
pool = Controller(store)
name = f"dfw-{store.raw()['instance']}-0"
calls = []
class Engine:
def request(self, method, path, body=None):
calls.append((method, path))
if path.startswith("/containers/json"):
return [{"Names": ["/" + name], "Id": "id", "State": "running"}]
return {}
async def template(settings):
return Engine(), {}
monkeypatch.setattr(pool, "inspect_template", template)
active = 1
async def status(request):
return httpx.Response(200, json={"ready": True, "active": active, "capacity": 4})
await pool.http.aclose()
pool.http = httpx.AsyncClient(transport=httpx.MockTransport(status))
try:
await pool.reconcile()
assert not any(method == "POST" for method, _ in calls)
active = 0
pool.inflight[name] = 1
await pool.reconcile()
assert not any(method == "POST" for method, _ in calls)
pool.release(name)
await pool.reconcile()
assert ("POST", "/containers/id/stop?t=30") in calls
assert ("DELETE", "/containers/id") in calls
assert not pool.nodes
finally:
await pool.close()
@pytest.mark.asyncio
async def test_changed_runtime_image_drains_and_recreates_old_workers(tmp_path, monkeypatch):
store = SettingsStore(tmp_path)
store.save(PoolSettings(
enabled=True, source_container="main", network="deerflow", desired=1,
))
pool = Controller(store)
name = f"dfw-{store.raw()['instance']}-0"
calls = []
class Engine:
def request(self, method, path, body=None):
calls.append((method, path, body))
if path.startswith("/containers/json"):
return [{
"Names": ["/" + name], "Id": "old-worker", "State": "running",
"ImageID": "sha256:old-runtime",
}]
if path.startswith("/containers/create?"):
assert body["Image"] == "sha256:new-runtime"
return {"Id": "new-worker"}
return {}
async def template(_settings):
return Engine(), {"Image": "sha256:new-runtime", "Entrypoint": ["python"], "WorkingDir": "/app"}
async def status(_request):
return httpx.Response(200, json={"ready": True, "active": 0, "capacity": 4})
monkeypatch.setattr(pool, "inspect_template", template)
await pool.http.aclose()
pool.http = httpx.AsyncClient(transport=httpx.MockTransport(status))
try:
await pool.reconcile()
paths = [(method, path) for method, path, _ in calls]
assert ("POST", "/containers/old-worker/stop?t=30") in paths
assert ("DELETE", "/containers/old-worker") in paths
create_index = next(i for i, (_, path) in enumerate(paths) if path.startswith("/containers/create?"))
assert paths.index(("DELETE", "/containers/old-worker")) < create_index
assert ("POST", "/containers/new-worker/start") in paths
assert pool.nodes[name]["id"] == "new-worker"
finally:
await pool.close()
@pytest.mark.asyncio
async def test_70_http_clients_distribute_to_four_workers_with_sse(tmp_path, monkeypatch):
from fastapi import FastAPI
from app.gateway import deps
from app.gateway.worker_pool.proxy import WorkerProxy
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, concurrency=32))
pool = Controller(store)
for i in range(4):
name = f"worker-{i}"
pool.nodes[name] = {"name": name, "ready": True, "active": 0, "capacity": 32, "draining": False, "url": f"http://{name}:8001"}
async def user(request):
return SimpleNamespace(id="user")
async def access(*args, **kwargs):
return True
async def metadata(*args, **kwargs):
return {"metadata": {}}
monkeypatch.setattr(deps, "get_current_user_from_request", user)
monkeypatch.setattr(deps, "get_thread_store", lambda request: SimpleNamespace(check_access=access, get=metadata))
gates = {name: Admission(32) for name in pool.nodes}
counts = {name: 0 for name in pool.nodes}
class EventStream(httpx.AsyncByteStream):
async def __aiter__(self):
yield b"event: end\ndata: {}\n\n"
async def worker(request):
assert request.headers["cookie"] == "session=preserved"
assert request.headers["host"] == "gateway"
assert request.headers["x-deerflow-worker-token"] == store.raw()["token"]
name = request.url.host
await gates[name].acquire()
try:
await asyncio.sleep(0.005)
counts[name] += 1
return httpx.Response(200, stream=EventStream(), headers={"content-type": "text/event-stream", "location": "/threads/t/runs/r/stream"})
finally:
gates[name].release()
await pool.http.aclose()
pool.http = httpx.AsyncClient(transport=httpx.MockTransport(worker))
app = FastAPI()
app.state.worker_pool = pool
app.add_middleware(WorkerProxy)
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://gateway") as client:
responses = await asyncio.gather(*(client.post(f"/api/threads/t-{i}/runs/stream", headers={"cookie": "session=preserved"}, json={"input": {}}) for i in range(70)))
assert all(response.status_code == 200 and "event: end" in response.text for response in responses)
assert all(response.headers["location"] == "/threads/t/runs/r/stream" for response in responses)
assert sorted(counts.values()) == [17, 17, 18, 18]
assert all(value == 0 for value in pool.inflight.values())
finally:
await pool.close()
@pytest.mark.asyncio
async def test_70_concurrent_runs_fall_back_to_main_without_worker_queue(tmp_path, monkeypatch):
from fastapi import FastAPI, Request
from starlette.responses import StreamingResponse
from app.gateway import deps
from app.gateway.worker_pool.proxy import WorkerProxy
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, concurrency=4))
pool = Controller(store)
pool.nodes["worker-0"] = {
"name": "worker-0", "ready": True, "active": 0, "capacity": 4,
"draining": False, "url": "http://worker-0:8001",
}
async def user(request):
return SimpleNamespace(id="user")
async def access(*args, **kwargs):
return True
async def metadata(*args, **kwargs):
return {"metadata": {}}
monkeypatch.setattr(deps, "get_current_user_from_request", user)
monkeypatch.setattr(deps, "get_thread_store", lambda request: SimpleNamespace(check_access=access, get=metadata))
worker_count = 0
main_count = 0
class EventStream(httpx.AsyncByteStream):
async def __aiter__(self):
await asyncio.sleep(0)
yield b"event: end\ndata: {}\n\n"
async def worker(request):
nonlocal worker_count
assert request.headers["x-deerflow-worker-token"] == store.raw()["token"]
worker_count += 1
await asyncio.sleep(0.01)
return httpx.Response(200, stream=EventStream(), headers={"content-type": "text/event-stream"})
await pool.http.aclose()
pool.http = httpx.AsyncClient(transport=httpx.MockTransport(worker))
app = FastAPI()
app.state.worker_pool = pool
app.state.checkpointer = object()
app.state.store = object()
app.add_middleware(WorkerProxy)
@app.post("/api/threads/{thread_id}/runs/stream")
async def main_run(thread_id: str, request: Request):
nonlocal main_count
assert request.state.worker_pool_fallback is True
main_count += 1
return StreamingResponse(iter([b"event: end\ndata: {}\n\n"]), media_type="text/event-stream")
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://gateway") as client:
responses = await asyncio.gather(*(
client.post(f"/api/threads/t-{i}/runs/stream", json={"input": {}})
for i in range(70)
))
assert all(response.status_code == 200 and "event: end" in response.text for response in responses)
assert worker_count == 4
assert main_count == 66
assert pool._ensure_db().execute("SELECT count(*) FROM affinity").fetchone()[0] == 4
assert all(value == 0 for value in pool.inflight.values())
finally:
await pool.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("worker_thread_active,status_code", [(False, 200), (True, 409)])
async def test_affined_thread_uses_main_fallback_only_when_worker_thread_is_idle(tmp_path, monkeypatch, worker_thread_active, status_code):
from fastapi import FastAPI, Request
from app.gateway import deps
from app.gateway.worker_pool.proxy import WorkerProxy
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, concurrency=1))
pool = Controller(store)
pool.nodes["worker-0"] = {
"name": "worker-0", "ready": True, "active": 1, "capacity": 1,
"draining": False, "url": "http://worker-0:8001",
}
pool._ensure_db().execute("INSERT INTO affinity VALUES (?, ?)", ("thread-1", "worker-0"))
pool._ensure_db().commit()
async def user(request):
return SimpleNamespace(id="user")
async def access(*args, **kwargs):
return True
async def metadata(*args, **kwargs):
return {"metadata": {}}
monkeypatch.setattr(deps, "get_current_user_from_request", user)
monkeypatch.setattr(deps, "get_thread_store", lambda request: SimpleNamespace(check_access=access, get=metadata))
checked = []
async def worker(request):
checked.append(request.url.path)
return httpx.Response(200, json={"active": worker_thread_active})
await pool.http.aclose()
pool.http = httpx.AsyncClient(transport=httpx.MockTransport(worker))
app = FastAPI()
app.state.worker_pool = pool
app.state.checkpointer = object()
app.state.store = object()
app.add_middleware(WorkerProxy)
@app.post("/api/threads/thread-1/runs/stream")
async def main_run(request: Request):
assert request.state.worker_pool_fallback is True
return {"handled_by": "main"}
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://gateway") as client:
result = await client.post("/api/threads/thread-1/runs/stream", json={"input": {}})
assert result.status_code == status_code
assert checked == ["/internal/worker-thread-active/thread-1"]
if status_code == 200:
assert result.json() == {"handled_by": "main"}
finally:
await pool.close()
def test_unified_pg_and_legacy_override(monkeypatch):
from app.gateway.worker_pool.preflight import shared_postgres_configured
cfg = SimpleNamespace(database=SimpleNamespace(backend="postgres"), checkpointer=None)
assert shared_postgres_configured(cfg)
cfg.checkpointer = SimpleNamespace(type="sqlite", use_postgres=True)
assert not shared_postgres_configured(cfg)
cfg.checkpointer.type = "postgres"
assert shared_postgres_configured(cfg)
monkeypatch.setenv("DEER_FLOW_USE_PG_CHECKPOINTER", "false")
assert not shared_postgres_configured(cfg)
def test_mysql_business_supports_both_chat_backends(monkeypatch, tmp_path):
from app.gateway.worker_pool.preflight import shared_storage_configured, validate_sqlite_mount
cfg = SimpleNamespace(database=SimpleNamespace(backend="mysql"), checkpointer=SimpleNamespace(type="postgres", use_postgres=True))
monkeypatch.delenv("DEER_FLOW_USE_PG_CHECKPOINTER", raising=False)
assert shared_storage_configured(cfg)
cfg.checkpointer = SimpleNamespace(type="sqlite", connection_string=str(tmp_path / "chat.db"))
assert shared_storage_configured(cfg)
mountinfo = f"31 1 8:1 / {tmp_path.as_posix()} rw - ext4 /dev/sda rw"
validate_sqlite_mount(cfg, mountinfo)
with pytest.raises(RuntimeError, match="network filesystem"):
validate_sqlite_mount(cfg, mountinfo.replace("ext4", "nfs4"))
with pytest.raises(RuntimeError, match="persistently mounted"):
validate_sqlite_mount(cfg, "31 1 8:1 / / rw - ext4 /dev/sda rw")
cfg.checkpointer.connection_string = ":memory:"
assert not shared_storage_configured(cfg)
@pytest.mark.asyncio
async def test_delayed_health_probe_does_not_hide_existing_run(tmp_path):
from app.gateway.worker_pool.controller import Controller, PoolUnavailable
from app.gateway.worker_pool.settings import SettingsStore
pool = Controller(SettingsStore(tmp_path))
pool.nodes['worker'] = dict(name='worker', ready=False, ever_ready=True, draining=False, url='http://worker:8001')
pool._ensure_db().execute("INSERT INTO affinity VALUES (?, ?)", ('thread', 'worker'))
pool._ensure_db().commit()
try:
assert (await pool.reserve('thread', create=False))['name'] == 'worker'
pool.release('worker')
with pytest.raises(PoolUnavailable):
await pool.reserve('thread', create=True)
pool.nodes['worker']['ever_ready'] = False
with pytest.raises(PoolUnavailable):
await pool.reserve('thread', create=False)
finally:
await pool.close()
@pytest.mark.asyncio
async def test_reconcile_retains_previous_readiness_during_probe_timeout(tmp_path, monkeypatch):
import httpx
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, desired=1, source_container='main', network='network'))
pool = Controller(store)
name = f"dfw-{store.raw()['instance']}-0"
pool.nodes[name] = dict(name=name, ready=True, ever_ready=True, active=1, draining=False)
pool._ensure_db().execute('INSERT INTO affinity VALUES (?, ?)', ('thread', name))
pool._ensure_db().commit()
class Engine:
def request(self, method, path, body=None):
assert path.startswith('/containers/json')
return [{'Names': ['/' + name], 'Id': 'id', 'State': 'running'}]
async def inspect(_settings):
return Engine(), {}
async def timeout(*args, **kwargs):
raise httpx.ReadTimeout('busy')
monkeypatch.setattr(pool, 'inspect_template', inspect)
monkeypatch.setattr(pool.http, 'get', timeout)
try:
await pool.reconcile()
await pool.reconcile()
assert (await pool.reserve('thread', create=False))['name'] == name
with pytest.raises(PoolUnavailable):
await pool.reserve('thread', create=True)
finally:
await pool.close()
@pytest.mark.asyncio
async def test_read_transport_retries_but_never_replays_model_submission():
import httpx
from app.gateway.worker_pool.proxy import send_upstream
attempts = []
async def transport(request):
attempts.append(request.method)
if request.method == 'POST' or len(attempts) == 1:
raise httpx.RemoteProtocolError('server closed keep-alive')
return httpx.Response(200, json={'status': 'success'})
async with httpx.AsyncClient(transport=httpx.MockTransport(transport)) as client:
response = await send_upstream(client, 'GET', 'http://worker/run', [], None)
assert response.status_code == 200
await response.aclose()
with pytest.raises(httpx.RemoteProtocolError):
await send_upstream(client, 'POST', 'http://worker/run', [], b'{}')
assert attempts == ['GET', 'GET', 'POST']
@pytest.mark.asyncio
async def test_second_gateway_cannot_start_schedulers(tmp_path, monkeypatch):
from app.gateway.worker_pool.lifecycle import managed_controller
from app.gateway.worker_pool.settings import PoolSettings, SettingsStore
monkeypatch.setenv("DEER_FLOW_WORKER_CONTROL_HOME", str(tmp_path))
async with managed_controller(SimpleNamespace(state=SimpleNamespace())):
async with managed_controller(SimpleNamespace(state=SimpleNamespace())):
pass
assert not (tmp_path / "affinity.sqlite3").exists()
assert not (tmp_path / "diagnostics.sqlite3").exists()
assert not (tmp_path / "controller.lock").exists()
assert not (tmp_path / "settings.json").exists()
assert not (tmp_path / "secret.key").exists()
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, source_container="main"))
async with managed_controller(SimpleNamespace(state=SimpleNamespace())) as first:
assert first is not None
first.start()
with pytest.raises(RuntimeError, match="只能由一个主服务进程"):
async with managed_controller(SimpleNamespace(state=SimpleNamespace())) as second:
second.start()
@pytest.mark.asyncio
async def test_disabled_pool_ignores_saved_docker_settings(tmp_path, monkeypatch):
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=False, transport="socket", source_container="main", network="deerflow"))
pool = Controller(store)
def unexpected_request(*args, **kwargs):
pytest.fail("disabled Worker pool must not contact Docker")
monkeypatch.setattr("app.gateway.worker_pool.controller.DockerEngine.request", unexpected_request)
pool.start()
await asyncio.sleep(0)
assert pool.task is None
assert pool.db is None
assert not (tmp_path / "controller.lock").exists()
assert not (tmp_path / "affinity.sqlite3").exists()
await pool.close()
@pytest.mark.asyncio
async def test_non_admin_cannot_read_host_credentials(monkeypatch):
from app.gateway.worker_pool import router
async def user(request):
return SimpleNamespace(system_role="user")
monkeypatch.setattr(router, "get_current_user_from_request", user)
with pytest.raises(HTTPException) as error:
await router.controller(SimpleNamespace())
assert error.value.status_code == 403
@pytest.mark.asyncio
async def test_disabled_proxy_and_reserved_search_are_unchanged(tmp_path):
from fastapi import FastAPI
from app.gateway.worker_pool.proxy import WorkerProxy
pool = Controller(SettingsStore(tmp_path))
app = FastAPI()
app.state.worker_pool = pool
app.add_middleware(WorkerProxy)
@app.post("/api/threads/search")
@app.post("/api/threads/t/runs/stream")
async def endpoint():
return {"local": True}
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client:
assert (await client.post("/api/threads/search")).json() == {"local": True}
assert (await client.post("/api/threads/t/runs/stream")).json() == {"local": True}
finally:
await pool.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("allowed,system,status_code", [(False, False, 404), (True, True, 200)])
async def test_private_and_scheduler_threads_are_not_assigned(tmp_path, monkeypatch, allowed, system, status_code):
from fastapi import FastAPI
from app.gateway import deps
from app.gateway.worker_pool.proxy import WorkerProxy
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True))
pool = Controller(store)
async def user(request):
return SimpleNamespace(id="user")
async def access(*args, **kwargs):
return allowed
async def metadata(*args, **kwargs):
return {"metadata": {"thread_type": "scheduler"} if system else {}}
monkeypatch.setattr(deps, "get_current_user_from_request", user)
monkeypatch.setattr(deps, "get_thread_store", lambda request: SimpleNamespace(check_access=access, get=metadata))
app = FastAPI()
app.state.worker_pool = pool
app.add_middleware(WorkerProxy)
@app.post("/api/threads/t/runs/stream")
async def endpoint():
return {"scheduler_stays_local": True}
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://gateway") as client:
result = await client.post("/api/threads/t/runs/stream")
assert result.status_code == status_code
assert pool._ensure_db().execute("SELECT count(*) FROM affinity").fetchone()[0] == 0
finally:
await pool.close()
@pytest.mark.asyncio
async def test_worker_boot_disables_background_and_refuses_memory_fallback(monkeypatch):
from contextlib import asynccontextmanager
from langgraph.checkpoint.memory import InMemorySaver
from app.gateway.worker_pool import worker
from deerflow.config.app_config import AppConfig
from deerflow.config.database_config import DatabaseConfig
cfg = AppConfig.model_construct(
database=DatabaseConfig.model_construct(backend="postgres"),
checkpointer=None,
sandbox=SimpleNamespace(use="LocalSandboxProvider"),
)
monkeypatch.setattr(worker, "get_app_config", lambda: cfg)
async def configure(config):
pass
monkeypatch.setattr(worker, "configure_runtime_executor", configure)
@asynccontextmanager
async def runtime(app, *, background):
assert background is False
app.state.checkpointer = InMemorySaver()
app.state.store = object()
yield
monkeypatch.setattr(worker, "langgraph_runtime", runtime)
app = SimpleNamespace(state=SimpleNamespace())
with pytest.raises(RuntimeError, match="fallback"):
async with worker.worker_lifespan(app):
pytest.fail("memory worker became ready")
@pytest.mark.asyncio
async def test_skill_disable_invalidates_other_worker_cache_immediately(monkeypatch):
from app.gateway.worker_pool.refresh import SharedSkillRefresh
from deerflow.agents.lead_agent import prompt
from deerflow.config import extensions_config
refresh = SharedSkillRefresh()
stamp = 1
calls = []
monkeypatch.setattr(refresh, "extension_stamp", lambda: stamp)
monkeypatch.setattr(refresh, "skill_stamp", lambda: ())
monkeypatch.setattr(extensions_config, "reload_extensions_config", lambda: calls.append("reload"))
async def invalidate():
calls.append("refresh")
monkeypatch.setattr(prompt, "refresh_skills_system_prompt_cache_async", invalidate)
await refresh.check()
await refresh.check()
assert calls == ["reload", "refresh"]
stamp = 2
await refresh.check()
assert calls == ["reload", "refresh", "reload", "refresh"]
@pytest.mark.asyncio
async def test_scaleup_partial_failure_remains_owned(tmp_path, monkeypatch):
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True, source_container="main", network="deerflow", desired=1))
pool = Controller(store)
class Engine:
def request(self, method, path, body=None):
if path.startswith("/containers/json"):
return []
if path.startswith("/containers/create?"):
return {"Id": "created-before-failure"}
raise DockerError("start failed")
async def template(settings):
return Engine(), {}
monkeypatch.setattr(pool, "inspect_template", template)
try:
with pytest.raises(DockerError):
await pool.reconcile()
assert len(pool.nodes) == 1
assert next(iter(pool.nodes.values()))["id"] == "created-before-failure"
assert not next(iter(pool.nodes.values()))["ready"]
finally:
await pool.close()
@pytest.mark.asyncio
async def test_local_preparation_cannot_race_remote_assignment(tmp_path):
store = SettingsStore(tmp_path)
store.save(PoolSettings(enabled=True))
pool = Controller(store)
request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(worker_pool=pool)))
entered, finish = asyncio.Event(), asyncio.Event()
@bounded_worker_run
async def start(body, thread_id, request):
entered.set()
await finish.wait()
return SimpleNamespace(task=None)
task = asyncio.create_task(start(None, "thread", request))
try:
await entered.wait()
with pytest.raises(PoolUnavailable, match="主服务执行"):
await pool.reserve("thread", create=True)
finish.set()
await task
assert not pool.local_threads
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await pool.close()
@@ -0,0 +1,91 @@
import asyncio
from types import SimpleNamespace
import pytest
from deerflow.config.checkpointer_config import CheckpointerConfig
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
from deerflow.runtime.store.async_provider import make_store
from deerflow.runtime.runs.manager import RunRecord
from deerflow.runtime.runs.schemas import DisconnectMode, RunStatus
from deerflow.runtime.runs.worker import _finalize_run
@pytest.mark.asyncio
async def test_concurrent_sqlite_initialization_and_shared_store(tmp_path):
config = SimpleNamespace(checkpointer=CheckpointerConfig(type="sqlite", connection_string=str(tmp_path / "chat.db")))
async def node(index):
async with make_checkpointer(config) as saver, make_store(config) as store:
assert (await (await saver.conn.execute("PRAGMA busy_timeout")).fetchone())[0] == 30000
await store.aput(("qa",), str(index), {"index": index})
assert (await store.aget(("qa",), str(index))).value == {"index": index}
await asyncio.gather(*(node(i) for i in range(4)))
async with make_store(config) as store:
assert len(await store.asearch(("qa",))) == 4
@pytest.mark.asyncio
async def test_pg_disabled_checkpoint_and_store_use_same_sqlite(tmp_path, monkeypatch):
monkeypatch.setenv("DEER_FLOW_USE_PG_CHECKPOINTER", "false")
path = str(tmp_path / "fallback.db")
config = SimpleNamespace(checkpointer=CheckpointerConfig(type="postgres", connection_string="postgresql://unused", sqlite_connection_string=path))
async with make_checkpointer(config) as saver, make_store(config) as store:
saver_path = (await (await saver.conn.execute("PRAGMA database_list")).fetchone())[2]
store_path = (await (await store.conn.execute("PRAGMA database_list")).fetchone())[2]
assert saver_path == store_path
await store.aput(("qa",), "fallback", {"ok": True})
assert (await store.aget(("qa",), "fallback")).value == {"ok": True}
@pytest.mark.asyncio
async def test_run_finalization_does_not_wait_for_title_checkpoint_read():
title_read_started = asyncio.Event()
release_title_read = asyncio.Event()
status_updates: list[tuple[str, str]] = []
titles: list[tuple[str, str]] = []
class Checkpointer:
async def aget_tuple(self, _config):
title_read_started.set()
await release_title_read.wait()
return SimpleNamespace(checkpoint={"channel_values": {"title": "异步标题"}})
class ThreadStore:
async def update_status(self, thread_id: str, status: str) -> None:
status_updates.append((thread_id, status))
async def update_display_name(self, thread_id: str, title: str) -> None:
titles.append((thread_id, title))
class Manager:
def is_latest(self, _record: RunRecord) -> bool:
return True
record = RunRecord(
run_id="run-title-test",
thread_id="thread-title-test",
assistant_id="assistant",
status=RunStatus.success,
on_disconnect=DisconnectMode.continue_,
)
await asyncio.wait_for(
_finalize_run(
journal=None,
run_manager=Manager(),
record=record,
ctx=SimpleNamespace(checkpointer=Checkpointer(), thread_store=ThreadStore(), app_config=None),
),
timeout=0.1,
)
assert status_updates == [("thread-title-test", "idle")]
await asyncio.wait_for(title_read_started.wait(), timeout=0.1)
assert titles == []
release_title_read.set()
for _ in range(10):
if titles:
break
await asyncio.sleep(0)
assert titles == [("thread-title-test", "异步标题")]
@@ -0,0 +1,373 @@
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "x-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_x_page.py"
FAST_SCRIPT_PATH = SKILL_ROOT / "scripts" / "fast_single_post.py"
LEAD_PROMPT_PATH = BACKEND_ROOT / "packages" / "harness" / "deerflow" / "agents" / "lead_agent" / "prompt.py"
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"x_page_renderer",
SCRIPT_PATH,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "x-page-generator"
assert "Twitter" in skill.description
discovered = LocalSkillStorage(host_path=str(SKILL_ROOT.parent.parent)).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert discovered_by_name["x-page-generator"].category == SkillCategory.PUBLIC
assert discovered_by_name["x-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/x-page-generator/SKILL.md" in lead_prompt_source
assert "专用 X 路由" in lead_prompt_source
assert "fast_single_post.py --spec" in lead_prompt_source
assert "推文与回复正文必须使用简体中文" in lead_prompt_source
def test_default_json_renders_editable_x_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "x-template.html").read_text(encoding="utf-8")
runtime = (SKILL_ROOT / "assets" / "x-runtime.js").read_text(encoding="utf-8")
html = renderer.render_document(data, template, runtime)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["posts"] >= 1
assert renderer.DATA_MARKER not in html
assert renderer.RUNTIME_MARKER not in html
assert 'id="xPageData"' in html
assert 'id="openSettings"' in html
assert 'id="settingsModal"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert '"brand": "X"' in html
assert "<title>X · 正在发生</title>" in html
assert '"x-page-data-v1:" + pageFingerprint' in html
assert "const readStorage = (key)" in html
assert "const storedTheme = readStorage(themeKey)" in html
assert "window.localStorage.getItem" in html
assert "const storedTheme = localStorage.getItem" not in html
data["site"]["pageTitle"] = "X · 智能体开发圈"
titled_html = renderer.render_document(data, template, runtime)
assert "<title>X · 智能体开发圈</title>" in titled_html
assert "<title>X · 正在发生</title>" not in titled_html
data["posts"]["items"][0]["text"] = "</script><script>alert('x')</script>"
escaped_html = renderer.render_document(data, template, runtime)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "x-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(
renderer.DataValidationError,
match="Refusing to overwrite",
):
renderer._write_new(output, html)
def test_validation_rejects_missing_module_and_duplicate_posts() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data.pop("trends")
data["posts"]["items"][1]["id"] = data["posts"]["items"][0]["id"]
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: trends" in message
assert "must be unique" in message
def test_remote_video_media_is_validated_and_supported_by_runtime() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["posts"]["items"][0]["media"] = {
"type": "video",
"mediaUrl": "https://cdn.example.com/post.mp4",
"posterUrl": "https://cdn.example.com/poster.jpg",
"pageUrl": "https://example.com/post",
}
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "x-runtime.js").read_text(encoding="utf-8")
assert 'media.type === "video"' in runtime
assert 'media.type === "webpage"' in runtime
assert 'rel="noopener noreferrer"' in runtime
def test_single_post_supports_fifty_preloaded_hot_replies() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
data["posts"]["items"] = [post]
post["metrics"]["replies"] = 50
post["repliesExpanded"] = True
post["disclosure"] = "情景模拟页面 · 帖文与热评为演示内容"
post["sourceUrl"] = "https://example.com/source-post"
post["replies"] = [
{
"id": f"reply-{index:02d}",
"author": f"演示用户{index:02d}",
"handle": f"@demo{index:02d}",
"initial": str(index % 10),
"colors": ["#5662f6", "#9f45d8"],
"verified": index % 11 == 0,
"time": f"{index + 1} 分钟",
"text": f"这是第 {index + 1} 条演示热评。",
"likes": 500 - index,
"replyCount": index % 7,
}
for index in range(50)
]
summary = renderer.validate_document(data)
html = renderer.render_document(
data,
(SKILL_ROOT / "assets" / "x-template.html").read_text(encoding="utf-8"),
(SKILL_ROOT / "assets" / "x-runtime.js").read_text(encoding="utf-8"),
)
assert summary["counts"]["posts"] == 1
assert summary["counts"]["replies"] == 50
assert '"id": "reply-49"' in html
assert "reply-pagination" in html
assert 'data-sort="hot"' in html
assert "showAllLabel" in html
assert "post.disclosure" not in html
assert "post-source" in html
assert "查看原帖" in html
def test_reply_limit_rejects_more_than_one_hundred_items() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
post["metrics"]["replies"] = 101
post["replies"] = [{"author": f"用户{index}", "text": "演示回复"} for index in range(101)]
with pytest.raises(renderer.DataValidationError, match="at most 100"):
renderer.validate_document(data)
def test_fast_single_post_builds_json_and_html_in_one_call(tmp_path: Path) -> None:
command = [
sys.executable,
str(FAST_SCRIPT_PATH),
"--output-dir",
str(tmp_path),
"--author",
"测试账号",
"--handle",
"@test_account",
"--text",
"这是一条用于验证快速生成路径的推文。",
"--topic",
"快速生成",
"--reply-count",
"50",
"--media-type",
"image",
"--media-url",
"https://example.com/image.jpg",
"--tag",
"快速生成",
]
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stderr
result = json.loads(completed.stdout)
assert result["status"] == "ok"
assert result["action"] == "fast-single-post"
assert result["counts"]["posts"] == 1
assert result["counts"]["replies"] == 50
data_path = Path(result["data"])
html_path = Path(result["html"])
assert data_path.exists()
assert html_path.exists()
data = json.loads(data_path.read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
assert len(post["replies"]) == 50
assert post["replies"][-1]["id"] == "reply-050"
assert post["media"]["mediaUrl"] == "https://example.com/image.jpg"
assert "disclosure" not in post
html = html_path.read_text(encoding="utf-8")
assert '"id": "reply-050"' in html
assert "settings-trigger" in html
second = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert second.returncode == 0, second.stderr
second_result = json.loads(second.stdout)
assert second_result["data"] != result["data"]
assert second_result["html"] != result["html"]
assert Path(second_result["data"]).exists()
assert Path(second_result["html"]).exists()
def test_fast_single_post_spec_separates_total_and_displayed_replies(
tmp_path: Path,
) -> None:
spec_path = tmp_path / "x-spec.json"
spec_path.write_text(
json.dumps(
{
"author": "Donald J. Trump",
"handle": "@realDonaldTrump",
"verified": True,
"text": "刚刚结束一次重要会晤,双方将继续推进对话与合作。",
"topic": "中美元首会晤",
"tags": ["中美关系"],
"displayReplyCount": 20,
"totalReplyCount": 10000,
"replyTopics": ["中美经济影响", "两岸关系"],
"replies": ["关税调整将直接影响企业成本和供应链安排。"],
"likes": 214000,
"reposts": 38600,
"views": 2380000,
},
ensure_ascii=False,
),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 0, completed.stderr
result = json.loads(completed.stdout)
data = json.loads(Path(result["data"]).read_text(encoding="utf-8"))
post = data["posts"]["items"][0]
assert post["metrics"]["replies"] == 10000
assert len(post["replies"]) == 20
assert post["replies"][0]["text"] == "关税调整将直接影响企业成本和供应链安排。"
assert any(topic in post["replies"][1]["text"] for topic in ("中美经济影响", "两岸关系"))
assert post["media"]["type"] == "skyline"
assert result["counts"]["replies"] == 20
def test_fast_single_post_rejects_english_only_tweet(tmp_path: Path) -> None:
spec_path = tmp_path / "x-english-spec.json"
spec_path.write_text(
json.dumps(
{
"author": "Example",
"handle": "@example",
"text": "This post is written only in English.",
}
),
encoding="utf-8",
)
completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert completed.returncode == 2
result = json.loads(completed.stdout)
assert "Simplified Chinese" in result["error"]
spec_path.write_text(
json.dumps(
{
"author": "Example",
"handle": "@example",
"text": "这是一条中文推文。",
"replies": ["This reply is written only in English."],
},
ensure_ascii=False,
),
encoding="utf-8",
)
reply_completed = subprocess.run(
[
sys.executable,
str(FAST_SCRIPT_PATH),
"--spec",
str(spec_path),
"--output-dir",
str(tmp_path),
],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
assert reply_completed.returncode == 2
assert "Simplified Chinese" in json.loads(reply_completed.stdout)["error"]
@@ -0,0 +1,149 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "xinhua-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_xinhua_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location("xinhua_page_renderer", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "xinhua-page-generator"
assert "新华社" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["xinhua-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["xinhua-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/xinhua-page-generator/SKILL.md" in lead_prompt_source
assert "专用新华社路由" in lead_prompt_source
def test_default_json_renders_editable_xinhua_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
counts = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "xinhua-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "xinhua-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert counts["spotlight"] == 5
assert counts["focus_tabs"] == 3
assert counts["provinces"] >= 1
assert "__XINHUA_PAGE_DATA__" not in html
assert "__XINHUA_PAGE_RUNTIME__" not in html
assert 'id="xinhuaPageData"' in html
assert 'id="spotlight"' in html
assert 'id="focusTabs"' in html
assert 'id="provinceBar"' in html
assert 'id="liveModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert 'data-spot-id="' in html
assert 'class="spot-select"' in html
assert ".focus-thumb,.spot-card,.involve-bg" in html
assert "readAsDataURL" in html
assert '"brandName":"新华社"' in html
assert '"xinhua-page-data-v1:" + fingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "localStorage.getItem('huaxun-theme')" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
output = tmp_path / "xinhua-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_visual_and_remote_url() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
data.pop("footer")
data["specials"]["spotlightItems"][1]["id"] = data["specials"][
"spotlightItems"
][0]["id"]
data["focus"]["tabs"][0]["columns"][0]["visual"] = "remote-image"
data["site"]["description"] = "https://example.com/remote"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "missing top-level module(s): footer" in message
assert "duplicates" in message
assert "visual must be one of" in message
assert "remote URLs" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/news.jpg", "alt": "新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/news.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "新闻原文", "embed": False},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["specials"]["spotlightItems"][0]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "xinhua-runtime.js").read_text(encoding="utf-8")
assert "externalMedia" in runtime
assert 'rel="noopener noreferrer"' in runtime
@@ -0,0 +1,163 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "yahoo-japan-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_yahoo_japan_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"yahoo_japan_page_renderer", SCRIPT_PATH
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "yahoo-japan-page-generator"
assert "Yahoo! JAPAN" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["yahoo-japan-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["yahoo-japan-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/yahoo-japan-page-generator/SKILL.md" in lead_prompt_source
assert "专用雅虎日本路由" in lead_prompt_source
def test_default_json_renders_editable_yahoo_japan_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
counts = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "yahoo-japan-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "yahoo-japan-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert counts["search_tabs"] == 7
assert counts["services"] == 15
assert counts["news_tabs"] == 6
assert counts["weather_locations"] == 3
assert counts["markets"] == 3
assert counts["products"] == 5
assert counts["sports_tabs"] == 3
assert counts["trend_tabs"] == 3
assert "__YAHOO_JAPAN_PAGE_DATA__" not in html
assert "__YAHOO_JAPAN_PAGE_RUNTIME__" not in html
assert 'id="yahooJapanPageData"' in html
assert 'id="searchTabs"' in html
assert 'id="serviceMore"' in html
assert 'id="weatherSelect"' in html
assert 'id="marketTabs"' in html
assert 'id="loginModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "external-media::before" in html
assert "data-media-path" in html
assert "exactPathTarget" in html
assert 'const visualHostSelector = "[data-media-path],' in html
assert "readAsDataURL" in html
assert '"brandPrimary":"Yahoo!"' in html
assert '"brandAccent":"JAPAN"' in html
assert '"language":"zh-CN"' in html
assert "JSON 编辑器" in html
assert '"yahoo-japan-page-data-v1:" + fingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "localStorage.getItem('hinode-theme')" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
output = tmp_path / "yahoo-japan-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_visual_remote_url_and_foreign_copy() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
data.pop("footer")
data["search"]["tabs"][1]["id"] = data["search"]["tabs"][0]["id"]
data["shopping"]["products"][0]["visual"] = "remote-image"
data["site"]["description"] = "https://example.com/remote"
data["news"]["feature"]["title"] = "新しい交通ハブ"
data["site"]["language"] = "ja"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "missing top-level module(s): footer" in message
assert "duplicates" in message
assert "visual must be one of" in message
assert "remote URLs" in message
assert "site.language must be zh-CN" in message
assert "news.feature.title must use Simplified Chinese reader-facing copy" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/yahoo.jpg", "alt": "门户新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/yahoo.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "报道原文", "embed": False},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["news"]["feature"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "yahoo-japan-runtime.js").read_text(encoding="utf-8")
assert "externalMedia" in runtime
assert 'rel="noopener noreferrer"' in runtime
@@ -0,0 +1,160 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "yomiuri-news-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_yomiuri_news_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"yomiuri_news_page_renderer", SCRIPT_PATH
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "yomiuri-news-page-generator"
assert "读卖新闻" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["yomiuri-news-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["yomiuri-news-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert "/mnt/skills/public/yomiuri-news-page-generator/SKILL.md" in lead_prompt_source
assert "专用读卖新闻路由" in lead_prompt_source
def test_default_json_renders_editable_yomiuri_news_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
counts = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "yomiuri-news-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "yomiuri-news-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert counts["breaking_items"] == 4
assert counts["lead_stories"] == 3
assert counts["edition_pages"] == 4
assert counts["news_tabs"] == 5
assert counts["opinion_columns"] == 3
assert counts["videos"] == 3
assert "__YOMIURI_NEWS_PAGE_DATA__" not in html
assert "__YOMIURI_NEWS_PAGE_RUNTIME__" not in html
assert 'id="yomiuriNewsPageData"' in html
assert 'id="flashPanel"' in html
assert 'id="leadMain"' in html
assert 'id="paperModal"' in html
assert 'id="categoryTabs"' in html
assert 'id="videoModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "external-media::before" in html
assert "data-media-path" in html
assert "exactPathTarget" in html
assert 'const visualHostSelector = "[data-media-path],' in html
assert "readAsDataURL" in html
assert '"brandPrimary":"读卖新闻"' in html
assert '"brandRoman":"YOMIURI ONLINE"' in html
assert '"language":"zh-CN"' in html
assert "JSON 编辑器" in html
assert '"yomiuri-news-page-data-v1:"+fingerprint' in html
assert "const readStorage=(key)" in html
assert "window.localStorage.getItem" in html
assert '<img src="http' not in html
assert '<video src="http' not in html
output = tmp_path / "yomiuri-news-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(renderer.DataValidationError, match="refusing to overwrite"):
renderer._write_new(output, html)
def test_validation_rejects_missing_duplicate_visual_remote_url_and_foreign_copy() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8")
)
data.pop("footer")
data["breaking"]["items"][1]["id"] = data["breaking"]["items"][0]["id"]
data["media"]["videos"][0]["visual"] = "remote-image"
data["site"]["description"] = "https://example.com/remote"
data["lead"]["main"]["title"] = "新しい交通拠点"
data["site"]["language"] = "ja"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "missing top-level module(s): footer" in message
assert "duplicates" in message
assert "visual must be one of" in message
assert "remote URLs" in message
assert "site.language must be zh-CN" in message
assert "lead.main.title must use Simplified Chinese reader-facing copy" in message
def test_remote_media_contract_supports_image_video_and_webpage() -> None:
renderer = _load_renderer()
source = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
for media in (
{"type": "image", "mediaUrl": "https://cdn.example.com/yomiuri.jpg", "alt": "读卖新闻图片"},
{"type": "video", "mediaUrl": "https://cdn.example.com/yomiuri.mp4", "posterUrl": "https://cdn.example.com/poster.jpg"},
{"type": "webpage", "pageUrl": "https://example.com/story", "title": "报道原文", "embed": True},
):
data = json.loads(json.dumps(source, ensure_ascii=False))
data["lead"]["main"]["media"] = media
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "yomiuri-news-runtime.js").read_text(encoding="utf-8")
assert "externalMedia" in runtime
assert 'sandbox="allow-forms allow-popups allow-scripts"' in runtime
@@ -0,0 +1,177 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
from deerflow.skills.types import SkillCategory
from deerflow.skills.validation import _validate_skill_frontmatter
BACKEND_ROOT = Path(__file__).resolve().parents[1]
SKILL_ROOT = BACKEND_ROOT / "skills" / "public" / "youtube-page-generator"
SCRIPT_PATH = SKILL_ROOT / "scripts" / "render_youtube_page.py"
LEAD_PROMPT_PATH = (
BACKEND_ROOT
/ "packages"
/ "harness"
/ "deerflow"
/ "agents"
/ "lead_agent"
/ "prompt.py"
)
def _load_renderer():
spec = importlib.util.spec_from_file_location(
"youtube_page_renderer",
SCRIPT_PATH,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_skill_is_discoverable_and_valid() -> None:
valid, error, _metadata = _validate_skill_frontmatter(SKILL_ROOT)
assert valid, error
skill = parse_skill_file(SKILL_ROOT / "SKILL.md", SkillCategory.PUBLIC)
assert skill is not None
assert skill.name == "youtube-page-generator"
assert "YouTube" in skill.description
discovered = LocalSkillStorage(
host_path=str(SKILL_ROOT.parent.parent)
).load_skills(enabled_only=True)
discovered_by_name = {item.name: item for item in discovered}
assert (
discovered_by_name["youtube-page-generator"].category
== SkillCategory.PUBLIC
)
assert discovered_by_name["youtube-page-generator"].enabled is True
lead_prompt_source = LEAD_PROMPT_PATH.read_text(encoding="utf-8")
assert (
"/mnt/skills/public/youtube-page-generator/SKILL.md"
in lead_prompt_source
)
assert "专用 YouTube 路由" in lead_prompt_source
def test_default_json_renders_editable_youtube_html(tmp_path: Path) -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(
encoding="utf-8"
)
)
summary = renderer.validate_document(data)
template = (SKILL_ROOT / "assets" / "youtube-template.html").read_text(
encoding="utf-8"
)
runtime = (SKILL_ROOT / "assets" / "youtube-runtime.js").read_text(
encoding="utf-8"
)
html = renderer.render_document(data, template, runtime)
assert summary["modules"] == list(renderer.MODULES)
assert summary["counts"]["videos"] >= 1
assert renderer.DATA_MARKER not in html
assert renderer.RUNTIME_MARKER not in html
assert 'id="youtubePageData"' in html
assert 'id="videoGrid"' in html
assert 'id="watchModal"' in html
assert 'id="openSettings"' in html
assert 'id="moduleTabs"' in html
assert 'id="applyJson"' in html
assert "uploadJsonMedia" in html
assert "installInlineMediaReplace" in html
assert "dfInlineMediaInput" in html
assert 'input.accept = "image/*,video/*"' in html
assert 'button.title = "上传图片或视频并替换"' in html
assert "active.expected" not in html
assert "previous.posterUrl" not in html
assert ".thumbnail-visual>video" in html
assert ".watch-visual>video{object-fit:contain;object-position:center center}" in html
assert 'class="watch-visual" id="watchVisual"' in html
assert 'class="duration"' not in html
assert "video.duration" not in html
assert 'muted playsinline preload="metadata"' in html
assert "decorateBuiltInVisuals" in html
assert "replaceBuiltInVisuals" not in html
assert "readAsDataURL" in html
assert '"brand": "YouTube"' in html
assert "<title>YouTube · 精彩视频</title>" in html
assert '"youtube-page-data-v1:" + pageFingerprint' in html
assert "const readStorage = (key)" in html
assert "window.localStorage.getItem" in html
assert "const cachedData = localStorage.getItem" not in html
assert '<img src="http' not in html
assert '<video src="http' not in html
data["site"]["pageTitle"] = "YouTube · 智能体频道"
titled_html = renderer.render_document(data, template, runtime)
assert "<title>YouTube · 智能体频道</title>" in titled_html
assert "<title>YouTube · 精彩视频</title>" not in titled_html
data["videos"]["items"][0]["title"] = (
"</script><script>alert('x')</script>"
)
escaped_html = renderer.render_document(data, template, runtime)
assert "</script><script>alert" not in escaped_html
assert "<\\/script><script>alert" in escaped_html
output = tmp_path / "youtube-page.html"
renderer._write_new(output, html)
assert output.is_file()
with pytest.raises(
renderer.DataValidationError,
match="Refusing to overwrite",
):
renderer._write_new(output, html)
def test_validation_rejects_missing_module_and_duplicate_videos() -> None:
renderer = _load_renderer()
data = json.loads(
(SKILL_ROOT / "assets" / "default-data.json").read_text(
encoding="utf-8"
)
)
data.pop("shorts")
data["videos"]["items"][1]["id"] = data["videos"]["items"][0]["id"]
data["videos"]["items"][0]["visual"] = "remote-thumbnail"
with pytest.raises(renderer.DataValidationError) as exc_info:
renderer.validate_document(data)
message = str(exc_info.value)
assert "Missing top-level module: shorts" in message
assert "must be unique" in message
assert "visual must be one of" in message
def test_remote_image_and_webpage_media_are_supported() -> None:
renderer = _load_renderer()
data = json.loads((SKILL_ROOT / "assets" / "default-data.json").read_text(encoding="utf-8"))
data["videos"]["items"][0]["media"] = {
"type": "webpage",
"pageUrl": "https://example.com/watch",
"title": "演示网页",
"mediaUrl": "https://cdn.example.com/watch.jpg",
"embed": False,
}
data["shorts"]["items"][0]["media"] = {
"type": "image",
"mediaUrl": "https://cdn.example.com/short.jpg",
"alt": "Shorts 封面",
}
renderer.validate_document(data)
runtime = (SKILL_ROOT / "assets" / "youtube-runtime.js").read_text(encoding="utf-8")
assert 'media.type === "webpage"' in runtime
assert "mediaMarkup(video.media" in runtime