"""Best-effort periodic full scans for the DeerFlow-local Wiki index.""" from __future__ import annotations import asyncio import logging from collections.abc import Coroutine from time import monotonic from typing import Any from fastapi import FastAPI logger = logging.getLogger(__name__) _CONFIG_REFRESH_SECONDS = 10.0 def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task: """Keep fire-and-forget index jobs strongly referenced until completion.""" jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None) if jobs is None: jobs = set() app.state.llmwiki_index_jobs = jobs task = asyncio.create_task(coroutine) jobs.add(task) task.add_done_callback(jobs.discard) return task async def _refresh_local_wiki_config(app: FastAPI): from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime return await rebuild_local_wiki_index_runtime(app) def _auto_sync_ready(app: FastAPI, config: Any) -> bool: return bool( config.enabled and config.auto_sync and getattr(app.state, "llmwiki_sync_service", None) is not None and getattr(app.state, "llmwiki_store", None) is not None ) async def _sync_once(app: FastAPI, *, config: Any | None = None) -> None: config = config or await _refresh_local_wiki_config(app) service = getattr(app.state, "llmwiki_sync_service", None) mapping_store = getattr(app.state, "llmwiki_store", None) if not _auto_sync_ready(app, config) or service is None or mapping_store is None: return mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)] semaphore = asyncio.Semaphore(config.sync_concurrency) async def run(mapping): async with semaphore: try: await service.sync_mapping(mapping) except asyncio.CancelledError: raise except Exception: logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True) await asyncio.gather(*(run(mapping) for mapping in mappings)) async def local_wiki_index_scheduler_loop(app: FastAPI) -> None: logger.info( "Local Wiki auto-sync scheduler started; runtime config refresh interval=%s seconds", _CONFIG_REFRESH_SECONDS, ) last_sync_started_at: float | None = None last_synced_runtime_signature: str | None = None while True: try: # Rebuild checks both config.yaml mtime and persisted runtime # settings every poll, independently from the much longer sync # cadence. Provider/URL changes therefore take effect promptly. config = await _refresh_local_wiki_config(app) if not _auto_sync_ready(app, config): # A later disabled -> enabled transition is a new first valid # configuration and should receive an immediate scan. last_sync_started_at = None last_synced_runtime_signature = None else: now = monotonic() interval = float(config.sync_interval_seconds) runtime_signature = getattr( app.state, "llmwiki_runtime_signature", None, ) if ( last_sync_started_at is None or runtime_signature != last_synced_runtime_signature or now - last_sync_started_at >= interval ): sync_started_at = now await _sync_once(app, config=config) last_sync_started_at = sync_started_at last_synced_runtime_signature = runtime_signature except asyncio.CancelledError: raise except Exception: logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)") await asyncio.sleep(_CONFIG_REFRESH_SECONDS)