sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -11,9 +11,10 @@ from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
|
||||
from fastapi.responses import HTMLResponse
|
||||
@@ -34,16 +35,20 @@ from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
|
||||
from app.gateway.routers import (
|
||||
admin_active_runs,
|
||||
admin_audit_retention as admin_audit_retention_routes,
|
||||
admin_users,
|
||||
agents,
|
||||
ai_writing,
|
||||
app_identity,
|
||||
artifacts,
|
||||
assistant_knowledge,
|
||||
assistants_compat,
|
||||
auth,
|
||||
service_tokens,
|
||||
browser_context,
|
||||
business_mapping,
|
||||
channels,
|
||||
chat_archive,
|
||||
checkpoint_migration,
|
||||
compact_menu_overrides,
|
||||
dashboard_sessions,
|
||||
@@ -58,6 +63,7 @@ from app.gateway.routers import (
|
||||
html_images,
|
||||
html_page_favorites,
|
||||
html_runtime,
|
||||
html_templates,
|
||||
intent,
|
||||
knowledge,
|
||||
knowledge_ingest,
|
||||
@@ -68,6 +74,7 @@ from app.gateway.routers import (
|
||||
mcp,
|
||||
memory,
|
||||
menu_overrides,
|
||||
model_exchanges,
|
||||
model_management,
|
||||
models,
|
||||
multi_agent,
|
||||
@@ -94,6 +101,7 @@ from app.gateway.routers import (
|
||||
roundtable_draft_shares,
|
||||
roundtable_drafts,
|
||||
roundtable_jobs,
|
||||
roundtable_rooms,
|
||||
roundtable_task_drafts,
|
||||
runs,
|
||||
scheduled_tasks,
|
||||
@@ -104,6 +112,7 @@ from app.gateway.routers import (
|
||||
skill_view_diagnostics,
|
||||
skills,
|
||||
strategy_agents,
|
||||
stream_truncations,
|
||||
suggestions,
|
||||
system_settings,
|
||||
tags,
|
||||
@@ -118,8 +127,8 @@ from app.gateway.routers import (
|
||||
thread_shares,
|
||||
threads,
|
||||
tool_metrics,
|
||||
uploads,
|
||||
upload_policy,
|
||||
uploads,
|
||||
user_preferences,
|
||||
user_prompts,
|
||||
web_app_sites,
|
||||
@@ -253,7 +262,7 @@ async def _iter_store_items(store, namespace, *, page_size: int = 500):
|
||||
offset += page_size
|
||||
|
||||
|
||||
async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
async def _sync_legacy_skills(app: FastAPI) -> list[Any] | None:
|
||||
"""Idempotently register existing on-disk ``skills/custom/*`` directories
|
||||
in the ``skills`` DB table, owned by the first admin and published=True.
|
||||
|
||||
@@ -273,11 +282,11 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
return
|
||||
return None
|
||||
|
||||
skill_store = getattr(app.state, "skill_store", None)
|
||||
if skill_store is None:
|
||||
return
|
||||
return None
|
||||
|
||||
async with sf() as session:
|
||||
stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
|
||||
@@ -295,7 +304,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
except Exception:
|
||||
logger.exception("_sync_legacy_skills: could not list on-disk skills")
|
||||
return
|
||||
return None
|
||||
|
||||
inserted = 0
|
||||
for skill in skills:
|
||||
@@ -313,6 +322,7 @@ async def _sync_legacy_skills(app: FastAPI) -> None:
|
||||
logger.warning("_sync_legacy_skills: could not register %s", skill.name, exc_info=True)
|
||||
if inserted:
|
||||
logger.info("Skill ownership migration: registered %d legacy custom skill(s)", inserted)
|
||||
return skills
|
||||
|
||||
|
||||
async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
|
||||
@@ -347,6 +357,12 @@ async def _run_db_migrations() -> None:
|
||||
logger.info("Persistence backend=memory ? skipping Alembic migrations")
|
||||
return
|
||||
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
if is_gauss_mysql_product():
|
||||
logger.warning("Database product=gauss_mysql; skipping MySQL-specific Alembic migrations")
|
||||
return
|
||||
|
||||
if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
|
||||
logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations")
|
||||
return
|
||||
@@ -553,6 +569,35 @@ _CONCURRENCY_SAMPLE_RETENTION_HOURS = _env_float(
|
||||
"CONCURRENCY_SAMPLE_RETENTION_HOURS",
|
||||
168.0, # 7 days
|
||||
)
|
||||
# Administrative audit data is operational rather than archival. The actual
|
||||
# cutoff lives with the shared query policy; these controls intentionally make
|
||||
# cleanup a low-priority, incremental task instead of a large startup delete.
|
||||
_ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS = _env_float(
|
||||
"ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS",
|
||||
15 * 60,
|
||||
minimum=60.0,
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS = _env_float(
|
||||
"ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS",
|
||||
300.0,
|
||||
minimum=0.0,
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_BATCH_SIZE = min(
|
||||
500,
|
||||
_env_int("ADMIN_AUDIT_RETENTION_BATCH_SIZE", 100, minimum=1),
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE = min(
|
||||
16,
|
||||
_env_int("ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE", 4, minimum=1),
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_INTER_BATCH_PAUSE_SECONDS = min(
|
||||
1.0,
|
||||
_env_float("ADMIN_AUDIT_RETENTION_INTER_BATCH_PAUSE_SECONDS", 0.05, minimum=0.0),
|
||||
)
|
||||
_ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS = min(
|
||||
60.0,
|
||||
_env_float("ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS", 8.0, minimum=0.1),
|
||||
)
|
||||
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
|
||||
# *previous* day's snapshot once, so records that landed near midnight are fully
|
||||
# captured. Default 1 ? "???? 1 ????????".
|
||||
@@ -643,10 +688,11 @@ async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql:
|
||||
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.gateway.admin_query_window import ADMIN_QUERY_WINDOW_DAYS
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
today = datetime.now(BEIJING_TZ).date()
|
||||
start = today - timedelta(days=max(1, days) - 1)
|
||||
start = today - timedelta(days=min(ADMIN_QUERY_WINDOW_DAYS, max(1, days)) - 1)
|
||||
return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
|
||||
|
||||
|
||||
@@ -951,6 +997,141 @@ async def _concurrency_sampler_loop(app: FastAPI) -> None:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
async def _admin_audit_retention_loop(app: FastAPI) -> None:
|
||||
"""Keep bounded admin-audit tables at the three-month retention target."""
|
||||
from app.gateway.admin_audit_retention import sweep_admin_audit_retention
|
||||
from app.gateway.admin_audit_retention_control import (
|
||||
is_paused,
|
||||
read_status,
|
||||
write_status,
|
||||
)
|
||||
from app.gateway.admin_query_window import ADMIN_AUDIT_RETENTION_DAYS
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".admin_audit_retention.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Admin audit retention: another gateway worker holds the lock; this worker skips")
|
||||
return
|
||||
|
||||
root = runtime_home()
|
||||
status = read_status(root)
|
||||
status.update(
|
||||
{
|
||||
"retention_days": ADMIN_AUDIT_RETENTION_DAYS,
|
||||
"interval_seconds": _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS,
|
||||
"startup_delay_seconds": _ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS,
|
||||
"batch_size": _ADMIN_AUDIT_RETENTION_BATCH_SIZE,
|
||||
"max_rounds": _ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE,
|
||||
"max_sweep_seconds": _ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS,
|
||||
}
|
||||
)
|
||||
|
||||
def save_status(**updates: Any) -> None:
|
||||
status.update(updates)
|
||||
write_status(root, dict(status))
|
||||
|
||||
async def publish_progress(progress: dict[str, Any]) -> None:
|
||||
await asyncio.to_thread(save_status, **progress)
|
||||
|
||||
logger.info(
|
||||
"Admin audit retention started: interval=%ss startup_delay=%ss batch_size=%s "
|
||||
"max_batches_per_store=%s max_sweep_seconds=%s",
|
||||
_ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS,
|
||||
_ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS,
|
||||
_ADMIN_AUDIT_RETENTION_BATCH_SIZE,
|
||||
_ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE,
|
||||
_ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS,
|
||||
)
|
||||
try:
|
||||
if _ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS:
|
||||
# Do not compete with migrations, warmup, or users arriving just
|
||||
# after a Gateway restart. The scheduled passes are deliberately
|
||||
# maintenance work, not a startup requirement.
|
||||
save_status(
|
||||
status="scheduled",
|
||||
current_source=None,
|
||||
next_run_at=(
|
||||
datetime.now().astimezone().timestamp()
|
||||
+ _ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS
|
||||
),
|
||||
)
|
||||
await asyncio.sleep(_ADMIN_AUDIT_RETENTION_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
if is_paused(root):
|
||||
save_status(status="paused", current_source=None, next_run_at=None)
|
||||
await asyncio.sleep(min(2.0, _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS))
|
||||
continue
|
||||
|
||||
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
save_status(
|
||||
status="running",
|
||||
current_source=None,
|
||||
round=0,
|
||||
deleted={
|
||||
"llm_metrics": 0,
|
||||
"tool_metrics": 0,
|
||||
"run_summaries": 0,
|
||||
"leaderboard_snapshots": 0,
|
||||
},
|
||||
started_at=started_at,
|
||||
next_run_at=None,
|
||||
last_error=None,
|
||||
)
|
||||
try:
|
||||
result = await sweep_admin_audit_retention(
|
||||
session_factory=get_session_factory(),
|
||||
llm_metrics_store=getattr(app.state, "llm_metrics_store", None),
|
||||
tool_metrics_store=getattr(app.state, "tool_metrics_store", None),
|
||||
run_store=getattr(app.state, "run_store", None),
|
||||
batch_size=_ADMIN_AUDIT_RETENTION_BATCH_SIZE,
|
||||
max_batches_per_store=_ADMIN_AUDIT_RETENTION_MAX_BATCHES_PER_STORE,
|
||||
inter_batch_pause_seconds=_ADMIN_AUDIT_RETENTION_INTER_BATCH_PAUSE_SECONDS,
|
||||
max_sweep_seconds=_ADMIN_AUDIT_RETENTION_MAX_SWEEP_SECONDS,
|
||||
should_stop=lambda: is_paused(root),
|
||||
on_progress=publish_progress,
|
||||
)
|
||||
finished_at = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
paused = is_paused(root)
|
||||
last_sweep = {
|
||||
"started_at": started_at,
|
||||
"finished_at": finished_at,
|
||||
"deleted": result,
|
||||
"stopped_by_admin": paused,
|
||||
}
|
||||
save_status(
|
||||
status="paused" if paused else "scheduled",
|
||||
current_source=None,
|
||||
round=status.get("round", 0),
|
||||
deleted=result,
|
||||
started_at=started_at,
|
||||
finished_at=finished_at,
|
||||
last_sweep=last_sweep,
|
||||
next_run_at=None if paused else (
|
||||
datetime.now().astimezone().timestamp()
|
||||
+ _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
save_status(
|
||||
status="error",
|
||||
current_source=None,
|
||||
last_error="清理轮次发生异常,请查看后台日志",
|
||||
finished_at=datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
)
|
||||
logger.exception("Admin audit retention sweep failed (non-fatal)")
|
||||
|
||||
# Wake periodically so an administrator can pause/resume without
|
||||
# waiting for the full 15-minute cleanup interval.
|
||||
remaining = _ADMIN_AUDIT_RETENTION_INTERVAL_SECONDS
|
||||
while remaining > 0 and not is_paused(root):
|
||||
sleep_for = min(2.0, remaining)
|
||||
await asyncio.sleep(sleep_for)
|
||||
remaining -= sleep_for
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""?? uvicorn / gunicorn ? worker ???? ERROR ???
|
||||
|
||||
@@ -1005,7 +1186,7 @@ def _warn_if_multi_worker() -> None:
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
|
||||
# ?????? worker ?? ?? ?? AI ?? / ? agent ???????
|
||||
@@ -1022,6 +1203,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
error_msg = f"Failed to load configuration during gateway startup: {e}"
|
||||
logger.exception(error_msg)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
config = get_gateway_config()
|
||||
logger.info(f"Starting API Gateway on {config.host}:{config.port}")
|
||||
|
||||
@@ -1045,20 +1227,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Run Alembic migrations before anything else touches the schema.
|
||||
await _run_db_migrations()
|
||||
|
||||
# Local Wiki mirror/index scans are opt-in and run outside request
|
||||
# handling. Per-library database leases keep multi-worker startup safe.
|
||||
app.state.llmwiki_index_scheduler_task = None
|
||||
local_wiki_index = app.state.config.llmwiki.local_wiki_index
|
||||
if (
|
||||
local_wiki_index.enabled
|
||||
and local_wiki_index.auto_sync
|
||||
and getattr(app.state, "llmwiki_sync_service", None) is not None
|
||||
):
|
||||
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
|
||||
# The lightweight Wiki coordinator always runs: it upgrades persisted
|
||||
# generation prompts and applies strict focus filters even when the
|
||||
# optional local mirror is disabled. Mirror/index scans themselves
|
||||
# remain opt-in inside the loop, with per-library database leases for
|
||||
# multi-worker safety.
|
||||
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
|
||||
|
||||
app.state.llmwiki_index_scheduler_task = asyncio.create_task(
|
||||
local_wiki_index_scheduler_loop(app)
|
||||
)
|
||||
app.state.llmwiki_index_scheduler_task = asyncio.create_task(
|
||||
local_wiki_index_scheduler_loop(app)
|
||||
)
|
||||
|
||||
# ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
|
||||
try:
|
||||
@@ -1078,11 +1256,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
# Migrate "ownerless" on-disk skills into the new SkillStore table ?
|
||||
# makes the multitenancy switch transparent for existing deployments.
|
||||
catalog_skills = None
|
||||
try:
|
||||
await _sync_legacy_skills(app)
|
||||
catalog_skills = await _sync_legacy_skills(app)
|
||||
except Exception:
|
||||
logger.exception("Skill ownership migration failed (non-fatal)")
|
||||
|
||||
# Project the compact per-skill catalog metadata into MySQL before the
|
||||
# Gateway accepts requests. List APIs read indexed scalar columns and
|
||||
# never load skill markdown, display samples, or usage sidecars.
|
||||
try:
|
||||
from app.gateway.skill_catalog_cache import refresh_skill_catalog_projection
|
||||
|
||||
catalog_size = await refresh_skill_catalog_projection(
|
||||
app.state.config,
|
||||
skills=catalog_skills,
|
||||
)
|
||||
logger.info("Skill MySQL catalog projection synchronized: skills=%d", catalog_size)
|
||||
except Exception:
|
||||
logger.exception("Skill MySQL catalog projection sync failed (non-fatal)")
|
||||
|
||||
# ????? agent(intent / recommender / coordinator)???????
|
||||
# ??? _sync_legacy_agents **??**??????????????????
|
||||
# ??? upsert ? agents ?,?????? /api/agents ?????,
|
||||
@@ -1142,6 +1335,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"Template-builder agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# HTML 模板库:把随包内置模板 seed 进 {base_dir}/html-templates/,模板管理页
|
||||
# 与定时任务下拉即可见(缺失才复制,绝不覆盖管理页/线上修改)。
|
||||
try:
|
||||
from deerflow.runtime.scheduler import template_store
|
||||
|
||||
template_store.ensure_seeded()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"HTML template store seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 实验性「页面研报设计师」:在 _sync_legacy_agents 前补齐 agent 目录与
|
||||
# 公共页面生成技能,首次启动即可在智能体管理页和普通对话中复用。
|
||||
try:
|
||||
@@ -1223,6 +1427,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"Forced-research responder agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 内置「查询变慢排查助手」:Linux + MySQL 间歇性查询变慢。目录已存在则不覆盖。
|
||||
try:
|
||||
from app.gateway.routers._query_slowdown_diagnosis_seed import (
|
||||
ensure_query_slowdown_diagnosis_agent,
|
||||
)
|
||||
|
||||
ensure_query_slowdown_diagnosis_agent()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Query-slowdown diagnosis agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
# 内置「耗时排查助手」:根据执行记录和日志判断空等原因。目录已存在则不覆盖。
|
||||
try:
|
||||
from app.gateway.routers._latency_diagnosis_seed import (
|
||||
@@ -1235,6 +1450,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"Latency-diagnosis agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 内置 Elasticsearch 离线运维助手:随离线包提供部署知识、只读采集脚本
|
||||
# 和规则分析器。必须在 legacy agent 同步前补建,目录存在时不覆盖现场修改。
|
||||
try:
|
||||
from app.gateway.routers._elasticsearch_offline_ops_seed import (
|
||||
ensure_elasticsearch_offline_ops_agent,
|
||||
)
|
||||
|
||||
ensure_elasticsearch_offline_ops_agent()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Elasticsearch offline operations agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
# 内置「任务研判报告助手」(agentfx 任务深链固定使用):同样必须在
|
||||
# _sync_legacy_agents 之前补建目录 + 补建配套 task-report-import 技能
|
||||
# (skills/ 被 .gitignore 排除),由后者 upsert 进 agents 表。
|
||||
@@ -1314,6 +1541,15 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Legacy agent sync failed at startup (non-fatal)")
|
||||
|
||||
# Refresh after database initialization and all built-in skill seeders;
|
||||
# import-time priming may have run before either was ready.
|
||||
try:
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
except Exception:
|
||||
logger.exception("Skills prompt cache warm-up failed at startup (non-fatal)")
|
||||
|
||||
# ?? skill curator ???:??????,?????????????
|
||||
# (per-user interval_hours ??????????????)?
|
||||
try:
|
||||
@@ -1344,6 +1580,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler failed to start (non-fatal)")
|
||||
|
||||
try:
|
||||
app.state.admin_audit_retention_task = asyncio.create_task(
|
||||
_admin_audit_retention_loop(app)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Admin audit retention scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
@@ -1385,6 +1628,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Task analysis service failed to start")
|
||||
|
||||
if getattr(app.state, "worker_pool", None) is not None:
|
||||
app.state.worker_pool.start()
|
||||
yield
|
||||
|
||||
artifact_library_tasks = list(
|
||||
@@ -1452,6 +1697,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Concurrency sampler shutdown error")
|
||||
|
||||
admin_audit_retention_task = getattr(app.state, "admin_audit_retention_task", None)
|
||||
if admin_audit_retention_task is not None:
|
||||
admin_audit_retention_task.cancel()
|
||||
try:
|
||||
await admin_audit_retention_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Admin audit retention scheduler shutdown error")
|
||||
|
||||
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
|
||||
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
|
||||
# ``langgraph_runtime()`` ? contextmanager ???????????????
|
||||
@@ -1491,6 +1746,25 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.info("Shutting down API Gateway")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from app.gateway.worker_pool.lifecycle import managed_controller
|
||||
from app.gateway.perf_logging import event_loop_watchdog, register_stack_dump_signal
|
||||
|
||||
register_stack_dump_signal()
|
||||
watchdog = asyncio.create_task(event_loop_watchdog(app), name="gateway-event-loop-watchdog")
|
||||
try:
|
||||
async with managed_controller(app):
|
||||
async with _gateway_lifespan(app):
|
||||
yield
|
||||
finally:
|
||||
watchdog.cancel()
|
||||
await asyncio.gather(watchdog, return_exceptions=True)
|
||||
from app.gateway.api_read_cache import close_api_read_cache
|
||||
|
||||
await close_api_read_cache()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create and configure the FastAPI application.
|
||||
|
||||
@@ -1589,6 +1863,48 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
],
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def invalidate_catalog_cache_after_write(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if request.method.upper() in {"POST", "PUT", "PATCH", "DELETE"} and response.status_code < 400:
|
||||
path = request.url.path.rstrip("/")
|
||||
collections: set[str] = set()
|
||||
if path == "/api/agents" or path.startswith("/api/agents/"):
|
||||
collections.update(("agents", "tags"))
|
||||
if path == "/api/skills" or path.startswith("/api/skills/"):
|
||||
collections.update(("skills", "tags"))
|
||||
if path == "/api/tags" or path.startswith("/api/tags/"):
|
||||
collections.update(("agents", "skills", "recommended_questions", "light_apps", "tags"))
|
||||
if path == "/api/positions" or path.startswith("/api/positions/"):
|
||||
collections.update(("agents", "skills", "recommended_questions"))
|
||||
if path == "/api/user-preferences" or path.startswith("/api/user-preferences/"):
|
||||
collections.add("user_preferences")
|
||||
if path == "/api/user-prompts" or path.startswith("/api/user-prompts/"):
|
||||
collections.add("user_prompts")
|
||||
if path.startswith("/api/recommended-questions"):
|
||||
collections.update(("recommended_questions", "tags"))
|
||||
if path == "/api/scheduled-tasks" or path.startswith("/api/scheduled-tasks/"):
|
||||
collections.add("tags")
|
||||
if path == "/api/light-apps" or path.startswith("/api/light-apps/"):
|
||||
collections.add("tags")
|
||||
if path == "/api/sensitive-words":
|
||||
collections.add("sensitive_words")
|
||||
config = getattr(request.app.state, "config", None)
|
||||
if config is not None and collections:
|
||||
from app.gateway.api_read_cache import invalidate_cached_collection
|
||||
|
||||
await asyncio.gather(
|
||||
*(
|
||||
invalidate_cached_collection(
|
||||
config,
|
||||
collection,
|
||||
force=True,
|
||||
)
|
||||
for collection in collections
|
||||
)
|
||||
)
|
||||
return response
|
||||
|
||||
if config.enable_docs:
|
||||
# Keep these paths relative to the page URL. Nginx strips the
|
||||
# /deerflow deployment prefix before forwarding to this app, and
|
||||
@@ -1621,6 +1937,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
)
|
||||
|
||||
# Auth: reject unauthenticated requests to non-public paths (fail-closed safety net)
|
||||
from app.gateway.worker_pool.proxy import WorkerProxy
|
||||
|
||||
app.add_middleware(WorkerProxy)
|
||||
app.add_middleware(AuthMiddleware)
|
||||
|
||||
# CSRF: Double Submit Cookie pattern for state-changing requests
|
||||
@@ -1889,6 +2208,7 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# Roundtable-planning background jobs at /api/roundtable-jobs
|
||||
app.include_router(roundtable_jobs.router)
|
||||
app.include_router(roundtable_rooms.router)
|
||||
app.include_router(roundtable_diagnostics.router)
|
||||
|
||||
# Deep Research (深度研究) API at /api/deep-research
|
||||
@@ -1927,6 +2247,8 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(html_runtime.router)
|
||||
app.include_router(html_images.router)
|
||||
app.include_router(html_page_favorites.router)
|
||||
# HTML 模板管理(模板管理页;读开放,写仅 admin)。
|
||||
app.include_router(html_templates.router)
|
||||
|
||||
# Product knowledge base API is mounted at /api/knowledge
|
||||
app.include_router(knowledge.router)
|
||||
@@ -1939,6 +2261,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# System settings API is mounted at /api/system-settings
|
||||
app.include_router(system_settings.router)
|
||||
app.include_router(app_identity.router)
|
||||
app.include_router(app_identity.public_router)
|
||||
from app.gateway.worker_pool.router import router as worker_pool_router
|
||||
|
||||
app.include_router(worker_pool_router)
|
||||
|
||||
# Admin LLM metrics audit + export
|
||||
app.include_router(llm_metrics.router)
|
||||
@@ -1946,24 +2273,32 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Admin window into other users' threads / messages / memory
|
||||
app.include_router(admin_users.router)
|
||||
app.include_router(admin_users.self_router)
|
||||
app.include_router(admin_audit_retention_routes.router)
|
||||
|
||||
app.include_router(admin_active_runs.router)
|
||||
app.include_router(skill_view_diagnostics.router)
|
||||
app.include_router(model_exchanges.router)
|
||||
|
||||
# Admin tool / skill call metrics audit + export
|
||||
app.include_router(tool_metrics.router)
|
||||
|
||||
# Admin model-stream truncation event log
|
||||
app.include_router(stream_truncations.router)
|
||||
|
||||
# Channels API is mounted at /api/channels
|
||||
app.include_router(channels.router)
|
||||
|
||||
# Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
|
||||
app.include_router(checkpoint_migration.router)
|
||||
# Admin monthly archive/rollback for PostgreSQL chat checkpoints.
|
||||
app.include_router(chat_archive.router)
|
||||
|
||||
# Assistants compatibility API (LangGraph Platform stub)
|
||||
app.include_router(assistants_compat.router)
|
||||
|
||||
# Auth API is mounted at /api/v1/auth
|
||||
app.include_router(auth.router)
|
||||
app.include_router(service_tokens.router)
|
||||
|
||||
# Feedback API is mounted at /api/threads/{thread_id}/runs/{run_id}/feedback
|
||||
app.include_router(feedback.router)
|
||||
@@ -2034,6 +2369,9 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(llmwiki.proxy_router, prefix="/deerflow")
|
||||
app.include_router(llmwiki.proxy_router)
|
||||
|
||||
from app.gateway.worker_pool.telemetry import RequestTelemetryMiddleware
|
||||
app.add_middleware(RequestTelemetryMiddleware)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user