"""Best-effort periodic full scans for the DeerFlow-local Wiki index.""" from __future__ import annotations import asyncio import logging from collections.abc import Coroutine from datetime import UTC, datetime from time import monotonic from typing import Any from fastapi import FastAPI logger = logging.getLogger(__name__) _CONFIG_REFRESH_SECONDS = 10.0 _MUTATION_SYNC_DELAYS_SECONDS = (1.0, 8.0, 30.0, 90.0, 300.0) _MUTATION_SYNC_MAX_SECONDS = 10 * 60.0 _MUTATION_SYNC_CONFIRM_GRACE_SECONDS = 30.0 _FULL_SCAN_FALLBACK_SECONDS = 10 * 60.0 def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool: """Normalize processing markers across old/new WeKnora response shapes.""" raw_flag = remote.get("is_processing", remote.get("isProcessing")) if isinstance(raw_flag, str): is_processing = raw_flag.strip().lower() in {"1", "true", "yes", "on"} else: is_processing = bool(raw_flag) raw_count = remote.get("processing_count", remote.get("processingCount")) try: processing_count = int(raw_count or 0) except (TypeError, ValueError): processing_count = 0 return is_processing or processing_count > 0 def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task: """Keep fire-and-forget index jobs strongly referenced until completion.""" jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None) if jobs is None: jobs = set() app.state.llmwiki_index_jobs = jobs task = asyncio.create_task(coroutine) jobs.add(task) task.add_done_callback(jobs.discard) return task def schedule_local_wiki_mutation_sync( app: FastAPI, mapping: dict[str, Any], *, delays: tuple[float, ...] | None = None, ) -> asyncio.Task | None: """Debounce one mapping's mutation and retry while WeKnora finishes Wiki work. Document ingestion is asynchronous upstream, so a single immediate scan can legitimately observe the old Wiki snapshot. One coordinator per mapping performs a few inexpensive metadata-first scans. A newer write only bumps the generation and resets that coordinator; it never cancels an active sync. Database leases remain the cross-process source of truth. """ service = getattr(app.state, "llmwiki_sync_service", None) mapping_id = str(mapping.get("id") or "") if service is None or not mapping_id: return None retry_delays = tuple(max(0.0, float(value)) for value in (delays or _MUTATION_SYNC_DELAYS_SECONDS)) if not retry_delays: return None plans: dict[str, dict[str, Any]] = getattr(app.state, "llmwiki_mutation_sync_plans", None) if plans is None: plans = {} app.state.llmwiki_mutation_sync_plans = plans tasks: dict[str, asyncio.Task] = getattr(app.state, "llmwiki_mutation_sync_tasks", None) if tasks is None: tasks = {} app.state.llmwiki_mutation_sync_tasks = tasks prior = plans.get(mapping_id) or {} wakeup = prior.get("wakeup") if not isinstance(wakeup, asyncio.Event): wakeup = asyncio.Event() plans[mapping_id] = { "generation": int(prior.get("generation") or 0) + 1, "mapping": dict(mapping), "wakeup": wakeup, } running = tasks.get(mapping_id) if running is not None and not running.done(): # Interrupt a long backoff, but never cancel an in-flight sync. The # coordinator will observe the newer generation at its next await # boundary and restart the bounded settle sequence from one second. wakeup.set() return running async def _remote_processing(current_mapping: dict[str, Any]) -> bool | None: """Read the cheap KB status without listing every Wiki page. ``None`` deliberately means "unknown": a transient status-read error must not suppress the actual incremental scan. """ client = getattr(service, "client", None) get_remote = getattr(client, "get_knowledge_base", None) remote_id = str(current_mapping.get("weknora_id") or "") if not callable(get_remote) or not remote_id: return None try: remote = await get_remote(remote_id) except asyncio.CancelledError: raise except Exception: logger.debug( "Could not read WeKnora processing status mapping_id=%s", mapping_id, exc_info=True, ) return None if not isinstance(remote, dict): return None return _knowledge_base_is_processing(remote) def _sync_changed(result: Any) -> bool: if not isinstance(result, dict): return False if str(result.get("status") or "").lower() not in {"completed", "idle"}: return False return any( int(result.get(key) or 0) > 0 for key in ( "added_pages", "updated_pages", "deleted_pages", ) ) async def _run() -> None: observed_generation = -1 delay_index = 0 quiet_scans = 0 observed_processing = False post_processing_confirmations = 0 deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS try: while True: plan = plans.get(mapping_id) if plan is None: return generation = int(plan["generation"]) if generation != observed_generation: observed_generation = generation delay_index = 0 quiet_scans = 0 observed_processing = False post_processing_confirmations = 0 deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS if not observed_processing and quiet_scans >= len(retry_delays): return remaining = deadline - monotonic() if remaining <= 0: return delay = min(retry_delays[min(delay_index, len(retry_delays) - 1)], remaining) wakeup = plan.get("wakeup") if isinstance(wakeup, asyncio.Event): wakeup.clear() # Re-read after clearing: a mutation scheduled from # another callback/thread between the earlier generation # read and ``clear()`` must not be hidden for a long # backoff interval. latest_after_clear = plans.get(mapping_id) if ( latest_after_clear is None or int(latest_after_clear["generation"]) != observed_generation ): continue try: await asyncio.wait_for(wakeup.wait(), timeout=delay) except TimeoutError: pass else: # A newer mutation resets the generation and the # backoff on the next loop iteration. continue else: await asyncio.sleep(delay) latest = plans.get(mapping_id) if latest is None: return if int(latest["generation"]) != observed_generation: continue current_mapping = dict(latest["mapping"]) delay_index = min(delay_index + 1, len(retry_delays) - 1) # Do not repeatedly enumerate the whole Wiki while WeKnora is # still parsing the uploaded document. Processing polls do # not consume one of the reserved quiet-scan opportunities, # so a final post-processing scan is always retained within # the bounded ten-minute settle window. if await _remote_processing(current_mapping) is True: observed_processing = True post_processing_confirmations = 0 continue try: result = await service.sync_mapping(current_mapping, force=False) except asyncio.CancelledError: raise except Exception: logger.warning( "Mutation-triggered local Wiki sync failed mapping_id=%s attempt=%s", mapping_id, quiet_scans + 1, exc_info=True, ) continue # A periodic/manual worker may temporarily own the mapping. # That is not a completed settle scan; retry after its lease # releases instead of silently dropping this mutation. if isinstance(result, dict) and result.get("status") == "already_running": continue post_processing = await _remote_processing(current_mapping) latest = plans.get(mapping_id) if latest is None: return if int(latest["generation"]) != observed_generation: continue if post_processing is True: observed_processing = True post_processing_confirmations = 0 continue if isinstance(result, dict) and result.get("deferred_deletion"): # The service's destructive-phase fence observed a race # that the cheap pre-check missed (or could not verify). # Treat it exactly like an explicit processing window and # retain the post-processing confirmation scans. observed_processing = True post_processing_confirmations = 0 delay_index = min(1, len(retry_delays) - 1) deadline = max( deadline, monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS, ) continue quiet_scans += 1 # Once WeKnora has explicitly transitioned out of processing, # require one short, quiet confirmation when the first scan # sees no delta: some WeKnora builds clear ``is_processing`` # just before publishing the final Wiki directory snapshot. # A synchronous edit or a post-processing delta can finish # immediately. if _sync_changed(result): return if observed_processing: post_processing_confirmations += 1 if post_processing_confirmations >= 2: return delay_index = min(1, len(retry_delays) - 1) deadline = max( deadline, monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS, ) finally: current = asyncio.current_task() if tasks.get(mapping_id) is current: tasks.pop(mapping_id, None) plan = plans.get(mapping_id) if plan is not None and int(plan.get("generation") or 0) == observed_generation: plans.pop(mapping_id, None) task = spawn_local_wiki_index_job(app, _run()) tasks[mapping_id] = task return task def _as_utc(value: Any) -> datetime | None: if isinstance(value, datetime): parsed = value else: raw = str(value or "").strip() if not raw: return None try: parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) except ValueError: return None if parsed.tzinfo is None: return parsed.replace(tzinfo=UTC) return parsed.astimezone(UTC) def _mapping_is_due( mapping: dict[str, Any], state: dict[str, Any] | None, remote: dict[str, Any] | None, *, force_scan: bool, ) -> bool: # Even the low-frequency reconciliation pass must not enumerate a Wiki # directory while WeKnora says it is transiently processing. ``force`` # bypasses only the timestamp preflight; the destructive-phase fence in # the sync service remains the final race/data-loss guard. if remote is not None and _knowledge_base_is_processing(remote): return False if force_scan or state is None or int(state.get("index_revision") or 0) <= 0: return True if str(state.get("state") or "") in {"failed", "partial", "rebuild_required"}: return True if str(state.get("state") or "") in {"running", "rebuilding"}: return False if remote is None: # Preserve the local snapshot on a transient/incomplete directory read; # the low-frequency forced scan remains the deletion/error authority. return False remote_updated = _as_utc(remote.get("updated_at") or remote.get("updatedAt")) last_success = _as_utc(state.get("last_success_at")) if remote_updated is None or last_success is None: return True return remote_updated > last_success async def _refresh_local_wiki_config(app: FastAPI): from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime return await rebuild_local_wiki_index_runtime(app) def _auto_sync_ready(app: FastAPI, config: Any) -> bool: return bool( config.enabled and config.auto_sync and getattr(app.state, "llmwiki_sync_service", None) is not None and getattr(app.state, "llmwiki_store", None) is not None ) async def _sync_once( app: FastAPI, *, config: Any | None = None, force_scan: bool | None = None, ) -> None: config = config or await _refresh_local_wiki_config(app) if force_scan is None: force_scan = bool( getattr(app.state, "llmwiki_force_full_scan_once", False) ) service = getattr(app.state, "llmwiki_sync_service", None) mapping_store = getattr(app.state, "llmwiki_store", None) if not _auto_sync_ready(app, config) or service is None or mapping_store is None: return mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)] index_store = getattr(app.state, "llmwiki_index_store", None) states: dict[str, dict[str, Any]] = {} if index_store is not None: states = { str(row.get("knowledge_base_mapping_id") or ""): row for row in await index_store.list_index_status( [str(mapping["id"]) for mapping in mappings] ) if isinstance(row, dict) } remote_by_id: dict[str, dict[str, Any]] = {} client = getattr(service, "client", None) list_remote = getattr(client, "list_knowledge_bases", None) if callable(list_remote): try: remote_rows = await list_remote(refresh=True) except TypeError: remote_rows = await list_remote() remote_by_id = { str(row.get("id") or ""): row for row in remote_rows if isinstance(row, dict) and row.get("id") } if remote_by_id: mappings = [ mapping for mapping in mappings if _mapping_is_due( mapping, states.get(str(mapping["id"])), remote_by_id.get(str(mapping.get("weknora_id") or "")), force_scan=bool(force_scan), ) ] semaphore = asyncio.Semaphore(config.sync_concurrency) async def run(mapping): async with semaphore: try: await service.sync_mapping(mapping) except asyncio.CancelledError: raise except Exception: logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True) await asyncio.gather(*(run(mapping) for mapping in mappings)) async def local_wiki_index_scheduler_loop(app: FastAPI) -> None: logger.info( "Local Wiki auto-sync scheduler started; runtime config refresh interval=%s seconds", _CONFIG_REFRESH_SECONDS, ) last_sync_started_at: float | None = None last_synced_runtime_signature: str | None = None last_full_scan_started_at: float | None = None while True: try: # Rebuild checks both config.yaml mtime and persisted runtime # settings every poll, independently from the much longer sync # cadence. Provider/URL changes therefore take effect promptly. config = await _refresh_local_wiki_config(app) if not _auto_sync_ready(app, config): # A later disabled -> enabled transition is a new first valid # configuration and should receive an immediate scan. last_sync_started_at = None last_synced_runtime_signature = None else: now = monotonic() interval = float(config.sync_interval_seconds) runtime_signature = getattr( app.state, "llmwiki_runtime_signature", None, ) if ( last_sync_started_at is None or runtime_signature != last_synced_runtime_signature or now - last_sync_started_at >= interval ): sync_started_at = now force_scan = bool( last_full_scan_started_at is None or runtime_signature != last_synced_runtime_signature or now - last_full_scan_started_at >= max( _FULL_SCAN_FALLBACK_SECONDS, interval, ) ) app.state.llmwiki_force_full_scan_once = force_scan try: await _sync_once(app, config=config) finally: app.state.llmwiki_force_full_scan_once = False last_sync_started_at = sync_started_at last_synced_runtime_signature = runtime_signature if force_scan: last_full_scan_started_at = sync_started_at except asyncio.CancelledError: raise except Exception: logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)") await asyncio.sleep(_CONFIG_REFRESH_SECONDS)