Files

106 lines
4.0 KiB
Python

"""Best-effort periodic full scans for the DeerFlow-local Wiki index."""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Coroutine
from time import monotonic
from typing import Any
from fastapi import FastAPI
logger = logging.getLogger(__name__)
_CONFIG_REFRESH_SECONDS = 10.0
def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
"""Keep fire-and-forget index jobs strongly referenced until completion."""
jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None)
if jobs is None:
jobs = set()
app.state.llmwiki_index_jobs = jobs
task = asyncio.create_task(coroutine)
jobs.add(task)
task.add_done_callback(jobs.discard)
return task
async def _refresh_local_wiki_config(app: FastAPI):
from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime
return await rebuild_local_wiki_index_runtime(app)
def _auto_sync_ready(app: FastAPI, config: Any) -> bool:
return bool(
config.enabled
and config.auto_sync
and getattr(app.state, "llmwiki_sync_service", None) is not None
and getattr(app.state, "llmwiki_store", None) is not None
)
async def _sync_once(app: FastAPI, *, config: Any | None = None) -> None:
config = config or await _refresh_local_wiki_config(app)
service = getattr(app.state, "llmwiki_sync_service", None)
mapping_store = getattr(app.state, "llmwiki_store", None)
if not _auto_sync_ready(app, config) or service is None or mapping_store is None:
return
mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
semaphore = asyncio.Semaphore(config.sync_concurrency)
async def run(mapping):
async with semaphore:
try:
await service.sync_mapping(mapping)
except asyncio.CancelledError:
raise
except Exception:
logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True)
await asyncio.gather(*(run(mapping) for mapping in mappings))
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
logger.info(
"Local Wiki auto-sync scheduler started; runtime config refresh interval=%s seconds",
_CONFIG_REFRESH_SECONDS,
)
last_sync_started_at: float | None = None
last_synced_runtime_signature: str | None = None
while True:
try:
# Rebuild checks both config.yaml mtime and persisted runtime
# settings every poll, independently from the much longer sync
# cadence. Provider/URL changes therefore take effect promptly.
config = await _refresh_local_wiki_config(app)
if not _auto_sync_ready(app, config):
# A later disabled -> enabled transition is a new first valid
# configuration and should receive an immediate scan.
last_sync_started_at = None
last_synced_runtime_signature = None
else:
now = monotonic()
interval = float(config.sync_interval_seconds)
runtime_signature = getattr(
app.state,
"llmwiki_runtime_signature",
None,
)
if (
last_sync_started_at is None
or runtime_signature != last_synced_runtime_signature
or now - last_sync_started_at >= interval
):
sync_started_at = now
await _sync_once(app, config=config)
last_sync_started_at = sync_started_at
last_synced_runtime_signature = runtime_signature
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)")
await asyncio.sleep(_CONFIG_REFRESH_SECONDS)