sync update from LAN repo (source master @ ca48c4ec)
This commit is contained in:
1 parent
8bc8b0e082
commit
ea9dcff9c0
873 files changed
+107866
-14011
No files matched your search
@@ -1,3 +1,5 @@
|
||||
# ruff: noqa: E402, I001
|
||||
# The dotenv load below must precede application imports that read environment.
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
@@ -7,11 +9,16 @@ import threading
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
# Load .env BEFORE any module that reads os.environ at import time.
|
||||
# Without this, GATEWAY_CORS_ORIGINS / AUTH_JWT_SECRET / DEER_FLOW_AUTH_DISABLED
|
||||
@@ -28,16 +35,20 @@ from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
|
||||
from app.gateway.routers import (
|
||||
admin_active_runs,
|
||||
admin_audit_retention as admin_audit_retention_routes,
|
||||
admin_users,
|
||||
agents,
|
||||
ai_writing,
|
||||
app_identity,
|
||||
artifacts,
|
||||
assistant_knowledge,
|
||||
assistants_compat,
|
||||
auth,
|
||||
service_tokens,
|
||||
browser_context,
|
||||
business_mapping,
|
||||
channels,
|
||||
chat_archive,
|
||||
checkpoint_migration,
|
||||
compact_menu_overrides,
|
||||
dashboard_sessions,
|
||||
@@ -52,6 +63,7 @@ from app.gateway.routers import (
|
||||
html_images,
|
||||
html_page_favorites,
|
||||
html_runtime,
|
||||
html_templates,
|
||||
intent,
|
||||
knowledge,
|
||||
knowledge_ingest,
|
||||
@@ -62,8 +74,8 @@ from app.gateway.routers import (
|
||||
mcp,
|
||||
memory,
|
||||
menu_overrides,
|
||||
model_management,
|
||||
model_exchanges,
|
||||
model_management,
|
||||
models,
|
||||
multi_agent,
|
||||
notifications,
|
||||
@@ -89,11 +101,10 @@ from app.gateway.routers import (
|
||||
roundtable_draft_shares,
|
||||
roundtable_drafts,
|
||||
roundtable_jobs,
|
||||
roundtable_rooms,
|
||||
roundtable_task_drafts,
|
||||
runs,
|
||||
scheduled_tasks,
|
||||
task_analysis,
|
||||
task_analysis_qa,
|
||||
sensitive_words,
|
||||
sentiment_agent,
|
||||
skill_addresses,
|
||||
@@ -101,10 +112,14 @@ from app.gateway.routers import (
|
||||
skill_view_diagnostics,
|
||||
skills,
|
||||
strategy_agents,
|
||||
stream_truncations,
|
||||
suggestions,
|
||||
system_settings,
|
||||
tags,
|
||||
task_analysis,
|
||||
task_analysis_qa,
|
||||
task_buttons,
|
||||
task_deeplink_details,
|
||||
task_deeplink_threads,
|
||||
task_reports,
|
||||
taskcop_tasks,
|
||||
@@ -112,8 +127,8 @@ from app.gateway.routers import (
|
||||
thread_shares,
|
||||
threads,
|
||||
tool_metrics,
|
||||
uploads,
|
||||
upload_policy,
|
||||
uploads,
|
||||
user_preferences,
|
||||
user_prompts,
|
||||
web_app_sites,
|
||||
@@ -144,6 +159,11 @@ logging.basicConfig(
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Swagger UI and ReDoc are vendored so the API documentation works in isolated
|
||||
# intranets. The route intentionally lives below /docs so it is covered by the
|
||||
# existing anonymous docs allow-list in the auth middleware.
|
||||
_DOCS_STATIC_DIR = Path(__file__).with_name("static") / "docs"
|
||||
|
||||
# Upper bound (seconds) each lifespan shutdown hook is allowed to run.
|
||||
# Bounds worker exit time so uvicorn's reload supervisor does not keep
|
||||
# firing signals into a worker that is stuck waiting for shutdown cleanup.
|
||||
@@ -242,7 +262,7 @@ async def _iter_store_items(store, namespace, *, page_size: int = 500):
|
||||
offset += page_size
|
||||
|
||||
|
||||
async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
async def _sync_legacy_skills(app: FastAPI) -> list[Any] | None:
|
||||
"""Idempotently register existing on-disk ``skills/custom/*`` directories
|
||||
in the ``skills`` DB table, owned by the first admin and published=True.
|
||||
|
||||
@@ -262,11 +282,11 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
return
|
||||
return None
|
||||
|
||||
skill_store = getattr(app.state, "skill_store", None)
|
||||
if skill_store is None:
|
||||
return
|
||||
return None
|
||||
|
||||
async with sf() as session:
|
||||
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
|
||||
@@ -284,7 +304,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
except Exception:
|
||||
logger.exception("_sync_legacy_skills: could not list on-disk skills")
|
||||
return
|
||||
return None
|
||||
|
||||
inserted = 0
|
||||
for skill in skills:
|
||||
@@ -302,6 +322,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
logger.warning("_sync_legacy_skills: could not register %s", skill.name, exc_info=True)
|
||||
if inserted:
|
||||
logger.info("Skill ownership migration: registered %d legacy custom skill(s)", inserted)
|
||||
return skills
|
||||
|
||||
|
||||
async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
|
||||
@@ -548,6 +569,35 @@ _CONCURRENCY_SAMPLE_RETENTION_HOURS = _env_float(
|
||||
"CONCURRENCY_SAMPLE_RETENTION_HOURS",
|
||||
168.0, # 7 days
|
||||
)
|
||||
# Administrative audit data is operational rather than archival. The actual
|
||||
# cutoff lives with the shared query policy; these controls intentionally make
|
||||
# cleanup a low-priority, incremental task instead of a large startup delete.
|
||||
_ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS = _env_float(
|
||||
"ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS",
|
||||
15 * 60,
|
||||
minimum=60.0,
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS = _env_float(
|
||||
"ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS",
|
||||
300.0,
|
||||
minimum=0.0,
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_BATCH_SIZE = min(
|
||||
500,
|
||||
_env_int("ADMIN_AUDIT_RETENTION_BATCH_SIZE", 100, minimum=1),
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE = min(
|
||||
16,
|
||||
_env_int("ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE", 4, minimum=1),
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_INTER_BATCH_PAUSE_SECONDS = min(
|
||||
1.0,
|
||||
_env_float("ADMIN_AUDIT_RETENTION_INTER_BATCH_PAUSE_SECONDS", 0.05, minimum=0.0),
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS = min(
|
||||
60.0,
|
||||
_env_float("ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS", 8.0, minimum=0.1),
|
||||
)
|
||||
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
|
||||
# *previous* day's snapshot once, so records that landed near midnight are fully
|
||||
# captured. Default 1 ? "???? 1 ????????".
|
||||
@@ -638,10 +688,11 @@ async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql:
|
||||
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.gateway.admin_query_window import ADMIN_QUERY_WINDOW_DAYS
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
today = datetime.now(BEIJING_TZ).date()
|
||||
start = today - timedelta(days=max(1, days) - 1)
|
||||
start = today - timedelta(days=min(ADMIN_QUERY_WINDOW_DAYS, max(1, days)) - 1)
|
||||
return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
|
||||
|
||||
|
||||
@@ -946,6 +997,141 @@ async def _concurrency_sampler_loop(app: FastAPI) -> None:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
async def _admin_audit_retention_loop(app: FastAPI) -> None:
|
||||
"""Keep bounded admin-audit tables at the three-month retention target."""
|
||||
from app.gateway.admin_audit_retention import sweep_admin_audit_retention
|
||||
from app.gateway.admin_audit_retention_control import (
|
||||
is_paused,
|
||||
read_status,
|
||||
write_status,
|
||||
)
|
||||
from app.gateway.admin_query_window import ADMIN_AUDIT_RETENTION_DAYS
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".admin_audit_retention.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Admin audit retention: another gateway worker holds the lock; this worker skips")
|
||||
return
|
||||
|
||||
root = runtime_home()
|
||||
status = read_status(root)
|
||||
status.update(
|
||||
{
|
||||
"retention_days": ADMIN_AUDIT_RETENTION_DAYS,
|
||||
"interval_seconds": _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS,
|
||||
"startup_delay_seconds": _ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS,
|
||||
"batch_size": _ADMIN_AUDIT_RETENTION_BATCH_SIZE,
|
||||
"max_rounds": _ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE,
|
||||
"max_sweep_seconds": _ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS,
|
||||
}
|
||||
)
|
||||
|
||||
def save_status(**updates: Any) -> None:
|
||||
status.update(updates)
|
||||
write_status(root, dict(status))
|
||||
|
||||
async def publish_progress(progress: dict[str, Any]) -> None:
|
||||
await asyncio.to_thread(save_status, **progress)
|
||||
|
||||
logger.info(
|
||||
"Admin audit retention started: interval=%ss startup_delay=%ss batch_size=%s "
|
||||
"max_batches_per_store=%s max_sweep_seconds=%s",
|
||||
_ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS,
|
||||
_ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS,
|
||||
_ADMIN_AUDIT_RETENTION_BATCH_SIZE,
|
||||
_ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE,
|
||||
_ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS,
|
||||
)
|
||||
try:
|
||||
if _ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS:
|
||||
# Do not compete with migrations, warmup, or users arriving just
|
||||
# after a Gateway restart. The scheduled passes are deliberately
|
||||
# maintenance work, not a startup requirement.
|
||||
save_status(
|
||||
status="scheduled",
|
||||
current_source=None,
|
||||
next_run_at=(
|
||||
datetime.now().astimezone().timestamp()
|
||||
+ _ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS
|
||||
),
|
||||
)
|
||||
await asyncio.sleep(_ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
if is_paused(root):
|
||||
save_status(status="paused", current_source=None, next_run_at=None)
|
||||
await asyncio.sleep(min(2.0, _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS))
|
||||
continue
|
||||
|
||||
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
save_status(
|
||||
status="running",
|
||||
current_source=None,
|
||||
round=0,
|
||||
deleted={
|
||||
"llm_metrics": 0,
|
||||
"tool_metrics": 0,
|
||||
"run_summaries": 0,
|
||||
"leaderboard_snapshots": 0,
|
||||
},
|
||||
started_at=started_at,
|
||||
next_run_at=None,
|
||||
last_error=None,
|
||||
)
|
||||
try:
|
||||
result = await sweep_admin_audit_retention(
|
||||
session_factory=get_session_factory(),
|
||||
llm_metrics_store=getattr(app.state, "llm_metrics_store", None),
|
||||
tool_metrics_store=getattr(app.state, "tool_metrics_store", None),
|
||||
run_store=getattr(app.state, "run_store", None),
|
||||
batch_size=_ADMIN_AUDIT_RETENTION_BATCH_SIZE,
|
||||
max_batches_per_store=_ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE,
|
||||
inter_batch_pause_seconds=_ADMIN_AUDIT_RETENTION_INTER_BATCH_PAUSE_SECONDS,
|
||||
max_sweep_seconds=_ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS,
|
||||
should_stop=lambda: is_paused(root),
|
||||
on_progress=publish_progress,
|
||||
)
|
||||
finished_at = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
paused = is_paused(root)
|
||||
last_sweep = {
|
||||
"started_at": started_at,
|
||||
"finished_at": finished_at,
|
||||
"deleted": result,
|
||||
"stopped_by_admin": paused,
|
||||
}
|
||||
save_status(
|
||||
status="paused" if paused else "scheduled",
|
||||
current_source=None,
|
||||
round=status.get("round", 0),
|
||||
deleted=result,
|
||||
started_at=started_at,
|
||||
finished_at=finished_at,
|
||||
last_sweep=last_sweep,
|
||||
next_run_at=None if paused else (
|
||||
datetime.now().astimezone().timestamp()
|
||||
+ _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
save_status(
|
||||
status="error",
|
||||
current_source=None,
|
||||
last_error="清理轮次发生异常,请查看后台日志",
|
||||
finished_at=datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
)
|
||||
logger.exception("Admin audit retention sweep failed (non-fatal)")
|
||||
|
||||
# Wake periodically so an administrator can pause/resume without
|
||||
# waiting for the full 15-minute cleanup interval.
|
||||
remaining = _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS
|
||||
while remaining > 0 and not is_paused(root):
|
||||
sleep_for = min(2.0, remaining)
|
||||
await asyncio.sleep(sleep_for)
|
||||
remaining -= sleep_for
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""?? uvicorn / gunicorn ? worker ???? ERROR ???
|
||||
|
||||
@@ -1000,7 +1186,7 @@ def _warn_if_multi_worker() -> None:
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
|
||||
@@ -1017,6 +1203,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
error_msg = f"Failed to load configuration during gateway startup: {e}"
|
||||
logger.exception(error_msg)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
config = get_gateway_config()
|
||||
logger.info(f"Starting API Gateway on {config.host}:{config.port}")
|
||||
|
||||
@@ -1069,11 +1256,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
# Migrate "ownerless" on-disk skills into the new SkillStore table ?
|
||||
# makes the multitenancy switch transparent for existing deployments.
|
||||
catalog_skills = None
|
||||
try:
|
||||
await _sync_legacy_skills(app)
|
||||
catalog_skills = await _sync_legacy_skills(app)
|
||||
except Exception:
|
||||
logger.exception("Skill ownership migration failed (non-fatal)")
|
||||
|
||||
# Project the compact per-skill catalog metadata into MySQL before the
|
||||
# Gateway accepts requests. List APIs read indexed scalar columns and
|
||||
# never load skill markdown, display samples, or usage sidecars.
|
||||
try:
|
||||
from app.gateway.skill_catalog_cache import refresh_skill_catalog_projection
|
||||
|
||||
catalog_size = await refresh_skill_catalog_projection(
|
||||
app.state.config,
|
||||
skills=catalog_skills,
|
||||
)
|
||||
logger.info("Skill MySQL catalog projection synchronized: skills=%d", catalog_size)
|
||||
except Exception:
|
||||
logger.exception("Skill MySQL catalog projection sync failed (non-fatal)")
|
||||
|
||||
# ????? agent(intent / recommender / coordinator)???????
|
||||
# ??? _sync_legacy_agents **??**??????????????????
|
||||
# ??? upsert ? agents ?,?????? /api/agents ?????,
|
||||
@@ -1133,6 +1335,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"Template-builder agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# HTML 模板库:把随包内置模板 seed 进 {base_dir}/html-templates/,模板管理页
|
||||
# 与定时任务下拉即可见(缺失才复制,绝不覆盖管理页/线上修改)。
|
||||
try:
|
||||
from deerflow.runtime.scheduler import template_store
|
||||
|
||||
template_store.ensure_seeded()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"HTML template store seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 实验性「页面研报设计师」:在 _sync_legacy_agents 前补齐 agent 目录与
|
||||
# 公共页面生成技能,首次启动即可在智能体管理页和普通对话中复用。
|
||||
try:
|
||||
@@ -1328,6 +1541,15 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Legacy agent sync failed at startup (non-fatal)")
|
||||
|
||||
# Refresh after database initialization and all built-in skill seeders;
|
||||
# import-time priming may have run before either was ready.
|
||||
try:
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
except Exception:
|
||||
logger.exception("Skills prompt cache warm-up failed at startup (non-fatal)")
|
||||
|
||||
# ?? skill curator ???:??????,?????????????
|
||||
# (per-user interval_hours ??????????????)?
|
||||
try:
|
||||
@@ -1358,6 +1580,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler failed to start (non-fatal)")
|
||||
|
||||
try:
|
||||
app.state.admin_audit_retention_task = asyncio.create_task(
|
||||
_admin_audit_retention_loop(app)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Admin audit retention scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
@@ -1399,6 +1628,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Task analysis service failed to start")
|
||||
|
||||
if getattr(app.state, "worker_pool", None) is not None:
|
||||
app.state.worker_pool.start()
|
||||
yield
|
||||
|
||||
artifact_library_tasks = list(
|
||||
@@ -1466,6 +1697,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler shutdown error")
|
||||
|
||||
admin_audit_retention_task = getattr(app.state, "admin_audit_retention_task", None)
|
||||
if admin_audit_retention_task is not None:
|
||||
admin_audit_retention_task.cancel()
|
||||
try:
|
||||
await admin_audit_retention_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Admin audit retention scheduler shutdown error")
|
||||
|
||||
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
|
||||
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
|
||||
# ``langgraph_runtime()`` ? contextmanager ???????????????
|
||||
@@ -1505,6 +1746,25 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.info("Shutting down API Gateway")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from app.gateway.worker_pool.lifecycle import managed_controller
|
||||
from app.gateway.perf_logging import event_loop_watchdog, register_stack_dump_signal
|
||||
|
||||
register_stack_dump_signal()
|
||||
watchdog = asyncio.create_task(event_loop_watchdog(app), name="gateway-event-loop-watchdog")
|
||||
try:
|
||||
async with managed_controller(app):
|
||||
async with _gateway_lifespan(app):
|
||||
yield
|
||||
finally:
|
||||
watchdog.cancel()
|
||||
await asyncio.gather(watchdog, return_exceptions=True)
|
||||
from app.gateway.api_read_cache import close_api_read_cache
|
||||
|
||||
await close_api_read_cache()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create and configure the FastAPI application.
|
||||
|
||||
@@ -1512,7 +1772,12 @@ def create_app() -> FastAPI:
|
||||
Configured FastAPI application instance.
|
||||
"""
|
||||
config = get_gateway_config()
|
||||
docs_kwargs = {"docs_url": "/docs", "redoc_url": "/redoc", "openapi_url": "/openapi.json"} if config.enable_docs else {"docs_url": None, "redoc_url": None, "openapi_url": None}
|
||||
logger.info("API documentation endpoints enabled: %s", config.enable_docs)
|
||||
docs_kwargs = {
|
||||
"docs_url": None,
|
||||
"redoc_url": None,
|
||||
"openapi_url": "/openapi.json",
|
||||
} if config.enable_docs else {"docs_url": None, "redoc_url": None, "openapi_url": None}
|
||||
|
||||
app = FastAPI(
|
||||
title="DeerFlow API Gateway",
|
||||
@@ -1598,7 +1863,83 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
],
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def invalidate_catalog_cache_after_write(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if request.method.upper() in {"POST", "PUT", "PATCH", "DELETE"} and response.status_code < 400:
|
||||
path = request.url.path.rstrip("/")
|
||||
collections: set[str] = set()
|
||||
if path == "/api/agents" or path.startswith("/api/agents/"):
|
||||
collections.update(("agents", "tags"))
|
||||
if path == "/api/skills" or path.startswith("/api/skills/"):
|
||||
collections.update(("skills", "tags"))
|
||||
if path == "/api/tags" or path.startswith("/api/tags/"):
|
||||
collections.update(("agents", "skills", "recommended_questions", "light_apps", "tags"))
|
||||
if path == "/api/positions" or path.startswith("/api/positions/"):
|
||||
collections.update(("agents", "skills", "recommended_questions"))
|
||||
if path == "/api/user-preferences" or path.startswith("/api/user-preferences/"):
|
||||
collections.add("user_preferences")
|
||||
if path == "/api/user-prompts" or path.startswith("/api/user-prompts/"):
|
||||
collections.add("user_prompts")
|
||||
if path.startswith("/api/recommended-questions"):
|
||||
collections.update(("recommended_questions", "tags"))
|
||||
if path == "/api/scheduled-tasks" or path.startswith("/api/scheduled-tasks/"):
|
||||
collections.add("tags")
|
||||
if path == "/api/light-apps" or path.startswith("/api/light-apps/"):
|
||||
collections.add("tags")
|
||||
if path == "/api/sensitive-words":
|
||||
collections.add("sensitive_words")
|
||||
config = getattr(request.app.state, "config", None)
|
||||
if config is not None and collections:
|
||||
from app.gateway.api_read_cache import invalidate_cached_collection
|
||||
|
||||
await asyncio.gather(
|
||||
*(
|
||||
invalidate_cached_collection(
|
||||
config,
|
||||
collection,
|
||||
force=True,
|
||||
)
|
||||
for collection in collections
|
||||
)
|
||||
)
|
||||
return response
|
||||
|
||||
if config.enable_docs:
|
||||
# Keep these paths relative to the page URL. Nginx strips the
|
||||
# /deerflow deployment prefix before forwarding to this app, and
|
||||
# root-relative URLs would otherwise escape that prefix in browsers.
|
||||
app.mount(
|
||||
"/docs/static",
|
||||
StaticFiles(directory=_DOCS_STATIC_DIR),
|
||||
name="docs-static",
|
||||
)
|
||||
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
async def swagger_ui() -> HTMLResponse:
|
||||
return get_swagger_ui_html(
|
||||
openapi_url="./openapi.json",
|
||||
title=f"{app.title} - Swagger UI",
|
||||
swagger_js_url="./docs/static/swagger-ui-bundle.js",
|
||||
swagger_css_url="./docs/static/swagger-ui.css",
|
||||
swagger_favicon_url="./docs/static/favicon-32x32.png",
|
||||
swagger_ui_parameters=app.swagger_ui_parameters,
|
||||
)
|
||||
|
||||
@app.get("/redoc", include_in_schema=False)
|
||||
async def redoc() -> HTMLResponse:
|
||||
return get_redoc_html(
|
||||
openapi_url="./openapi.json",
|
||||
title=f"{app.title} - ReDoc",
|
||||
redoc_js_url="./docs/static/redoc.standalone.js",
|
||||
redoc_favicon_url="./docs/static/favicon-32x32.png",
|
||||
with_google_fonts=False,
|
||||
)
|
||||
|
||||
# Auth: reject unauthenticated requests to non-public paths (fail-closed safety net)
|
||||
from app.gateway.worker_pool.proxy import WorkerProxy
|
||||
|
||||
app.add_middleware(WorkerProxy)
|
||||
app.add_middleware(AuthMiddleware)
|
||||
|
||||
# CSRF: Double Submit Cookie pattern for state-changing requests
|
||||
@@ -1756,8 +2097,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
app.include_router(analysis_projects.router)
|
||||
app.include_router(threads.router)
|
||||
# Dedicated taskId + goPath history lookup for 3qfx/rwfx/xdfx single-agent chats.
|
||||
# Dedicated taskId + goPath history lookup for single-agent task deep-link chats.
|
||||
app.include_router(task_deeplink_threads.router)
|
||||
app.include_router(task_deeplink_details.router)
|
||||
# Agents API is mounted at /api/agents
|
||||
app.include_router(agents.router)
|
||||
app.include_router(strategy_agents.router)
|
||||
@@ -1866,6 +2208,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# Roundtable-planning background jobs at /api/roundtable-jobs
|
||||
app.include_router(roundtable_jobs.router)
|
||||
app.include_router(roundtable_rooms.router)
|
||||
app.include_router(roundtable_diagnostics.router)
|
||||
|
||||
# Deep Research (深度研究) API at /api/deep-research
|
||||
@@ -1904,6 +2247,8 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(html_runtime.router)
|
||||
app.include_router(html_images.router)
|
||||
app.include_router(html_page_favorites.router)
|
||||
# HTML 模板管理(模板管理页;读开放,写仅 admin)。
|
||||
app.include_router(html_templates.router)
|
||||
|
||||
# Product knowledge base API is mounted at /api/knowledge
|
||||
app.include_router(knowledge.router)
|
||||
@@ -1916,6 +2261,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# System settings API is mounted at /api/system-settings
|
||||
app.include_router(system_settings.router)
|
||||
app.include_router(app_identity.router)
|
||||
app.include_router(app_identity.public_router)
|
||||
from app.gateway.worker_pool.router import router as worker_pool_router
|
||||
|
||||
app.include_router(worker_pool_router)
|
||||
|
||||
# Admin LLM metrics audit + export
|
||||
app.include_router(llm_metrics.router)
|
||||
@@ -1923,6 +2273,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Admin window into other users' threads / messages / memory
|
||||
app.include_router(admin_users.router)
|
||||
app.include_router(admin_users.self_router)
|
||||
app.include_router(admin_audit_retention_routes.router)
|
||||
|
||||
app.include_router(admin_active_runs.router)
|
||||
app.include_router(skill_view_diagnostics.router)
|
||||
@@ -1931,17 +2282,23 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Admin tool / skill call metrics audit + export
|
||||
app.include_router(tool_metrics.router)
|
||||
|
||||
# Admin model-stream truncation event log
|
||||
app.include_router(stream_truncations.router)
|
||||
|
||||
# Channels API is mounted at /api/channels
|
||||
app.include_router(channels.router)
|
||||
|
||||
# Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
|
||||
app.include_router(checkpoint_migration.router)
|
||||
# Admin monthly archive/rollback for PostgreSQL chat checkpoints.
|
||||
app.include_router(chat_archive.router)
|
||||
|
||||
# Assistants compatibility API (LangGraph Platform stub)
|
||||
app.include_router(assistants_compat.router)
|
||||
|
||||
# Auth API is mounted at /api/v1/auth
|
||||
app.include_router(auth.router)
|
||||
app.include_router(service_tokens.router)
|
||||
|
||||
# Feedback API is mounted at /api/threads/{thread_id}/runs/{run_id}/feedback
|
||||
app.include_router(feedback.router)
|
||||
@@ -2012,6 +2369,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(llmwiki.proxy_router, prefix="/deerflow")
|
||||
app.include_router(llmwiki.proxy_router)
|
||||
|
||||
from app.gateway.worker_pool.telemetry import RequestTelemetryMiddleware
|
||||
app.add_middleware(RequestTelemetryMiddleware)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user