462 lines
18 KiB
Python
462 lines
18 KiB
Python
"""Best-effort periodic full scans for the DeerFlow-local Wiki index."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from collections.abc import Coroutine
|
|
from datetime import UTC, datetime
|
|
from time import monotonic
|
|
from typing import Any
|
|
|
|
from fastapi import FastAPI
|
|
|
|
logger = logging.getLogger(__name__)
|
|
_CONFIG_REFRESH_SECONDS = 10.0
|
|
_MUTATION_SYNC_DELAYS_SECONDS = (1.0, 8.0, 30.0, 90.0, 300.0)
|
|
_MUTATION_SYNC_MAX_SECONDS = 10 * 60.0
|
|
_MUTATION_SYNC_CONFIRM_GRACE_SECONDS = 30.0
|
|
_FULL_SCAN_FALLBACK_SECONDS = 10 * 60.0
|
|
|
|
|
|
def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool:
|
|
"""Normalize processing markers across old/new WeKnora response shapes."""
|
|
|
|
raw_flag = remote.get("is_processing", remote.get("isProcessing"))
|
|
if isinstance(raw_flag, str):
|
|
is_processing = raw_flag.strip().lower() in {"1", "true", "yes", "on"}
|
|
else:
|
|
is_processing = bool(raw_flag)
|
|
raw_count = remote.get("processing_count", remote.get("processingCount"))
|
|
try:
|
|
processing_count = int(raw_count or 0)
|
|
except (TypeError, ValueError):
|
|
processing_count = 0
|
|
return is_processing or processing_count > 0
|
|
|
|
|
|
def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
|
|
"""Keep fire-and-forget index jobs strongly referenced until completion."""
|
|
|
|
jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None)
|
|
if jobs is None:
|
|
jobs = set()
|
|
app.state.llmwiki_index_jobs = jobs
|
|
task = asyncio.create_task(coroutine)
|
|
jobs.add(task)
|
|
task.add_done_callback(jobs.discard)
|
|
return task
|
|
|
|
|
|
def schedule_local_wiki_mutation_sync(
|
|
app: FastAPI,
|
|
mapping: dict[str, Any],
|
|
*,
|
|
delays: tuple[float, ...] | None = None,
|
|
) -> asyncio.Task | None:
|
|
"""Debounce one mapping's mutation and retry while WeKnora finishes Wiki work.
|
|
|
|
Document ingestion is asynchronous upstream, so a single immediate scan can
|
|
legitimately observe the old Wiki snapshot. One coordinator per mapping
|
|
performs a few inexpensive metadata-first scans. A newer write only bumps
|
|
the generation and resets that coordinator; it never cancels an active
|
|
sync. Database leases remain the cross-process source of truth.
|
|
"""
|
|
|
|
service = getattr(app.state, "llmwiki_sync_service", None)
|
|
mapping_id = str(mapping.get("id") or "")
|
|
if service is None or not mapping_id:
|
|
return None
|
|
retry_delays = tuple(max(0.0, float(value)) for value in (delays or _MUTATION_SYNC_DELAYS_SECONDS))
|
|
if not retry_delays:
|
|
return None
|
|
|
|
plans: dict[str, dict[str, Any]] = getattr(app.state, "llmwiki_mutation_sync_plans", None)
|
|
if plans is None:
|
|
plans = {}
|
|
app.state.llmwiki_mutation_sync_plans = plans
|
|
tasks: dict[str, asyncio.Task] = getattr(app.state, "llmwiki_mutation_sync_tasks", None)
|
|
if tasks is None:
|
|
tasks = {}
|
|
app.state.llmwiki_mutation_sync_tasks = tasks
|
|
|
|
prior = plans.get(mapping_id) or {}
|
|
wakeup = prior.get("wakeup")
|
|
if not isinstance(wakeup, asyncio.Event):
|
|
wakeup = asyncio.Event()
|
|
plans[mapping_id] = {
|
|
"generation": int(prior.get("generation") or 0) + 1,
|
|
"mapping": dict(mapping),
|
|
"wakeup": wakeup,
|
|
}
|
|
running = tasks.get(mapping_id)
|
|
if running is not None and not running.done():
|
|
# Interrupt a long backoff, but never cancel an in-flight sync. The
|
|
# coordinator will observe the newer generation at its next await
|
|
# boundary and restart the bounded settle sequence from one second.
|
|
wakeup.set()
|
|
return running
|
|
|
|
async def _remote_processing(current_mapping: dict[str, Any]) -> bool | None:
|
|
"""Read the cheap KB status without listing every Wiki page.
|
|
|
|
``None`` deliberately means "unknown": a transient status-read error
|
|
must not suppress the actual incremental scan.
|
|
"""
|
|
|
|
client = getattr(service, "client", None)
|
|
get_remote = getattr(client, "get_knowledge_base", None)
|
|
remote_id = str(current_mapping.get("weknora_id") or "")
|
|
if not callable(get_remote) or not remote_id:
|
|
return None
|
|
try:
|
|
remote = await get_remote(remote_id)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.debug(
|
|
"Could not read WeKnora processing status mapping_id=%s",
|
|
mapping_id,
|
|
exc_info=True,
|
|
)
|
|
return None
|
|
if not isinstance(remote, dict):
|
|
return None
|
|
return _knowledge_base_is_processing(remote)
|
|
|
|
def _sync_changed(result: Any) -> bool:
|
|
if not isinstance(result, dict):
|
|
return False
|
|
if str(result.get("status") or "").lower() not in {"completed", "idle"}:
|
|
return False
|
|
return any(
|
|
int(result.get(key) or 0) > 0
|
|
for key in (
|
|
"added_pages",
|
|
"updated_pages",
|
|
"deleted_pages",
|
|
)
|
|
)
|
|
|
|
async def _run() -> None:
|
|
observed_generation = -1
|
|
delay_index = 0
|
|
quiet_scans = 0
|
|
observed_processing = False
|
|
post_processing_confirmations = 0
|
|
deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS
|
|
try:
|
|
while True:
|
|
plan = plans.get(mapping_id)
|
|
if plan is None:
|
|
return
|
|
generation = int(plan["generation"])
|
|
if generation != observed_generation:
|
|
observed_generation = generation
|
|
delay_index = 0
|
|
quiet_scans = 0
|
|
observed_processing = False
|
|
post_processing_confirmations = 0
|
|
deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS
|
|
if not observed_processing and quiet_scans >= len(retry_delays):
|
|
return
|
|
remaining = deadline - monotonic()
|
|
if remaining <= 0:
|
|
return
|
|
delay = min(retry_delays[min(delay_index, len(retry_delays) - 1)], remaining)
|
|
wakeup = plan.get("wakeup")
|
|
if isinstance(wakeup, asyncio.Event):
|
|
wakeup.clear()
|
|
# Re-read after clearing: a mutation scheduled from
|
|
# another callback/thread between the earlier generation
|
|
# read and ``clear()`` must not be hidden for a long
|
|
# backoff interval.
|
|
latest_after_clear = plans.get(mapping_id)
|
|
if (
|
|
latest_after_clear is None
|
|
or int(latest_after_clear["generation"])
|
|
!= observed_generation
|
|
):
|
|
continue
|
|
try:
|
|
await asyncio.wait_for(wakeup.wait(), timeout=delay)
|
|
except TimeoutError:
|
|
pass
|
|
else:
|
|
# A newer mutation resets the generation and the
|
|
# backoff on the next loop iteration.
|
|
continue
|
|
else:
|
|
await asyncio.sleep(delay)
|
|
latest = plans.get(mapping_id)
|
|
if latest is None:
|
|
return
|
|
if int(latest["generation"]) != observed_generation:
|
|
continue
|
|
current_mapping = dict(latest["mapping"])
|
|
delay_index = min(delay_index + 1, len(retry_delays) - 1)
|
|
|
|
# Do not repeatedly enumerate the whole Wiki while WeKnora is
|
|
# still parsing the uploaded document. Processing polls do
|
|
# not consume one of the reserved quiet-scan opportunities,
|
|
# so a final post-processing scan is always retained within
|
|
# the bounded ten-minute settle window.
|
|
if await _remote_processing(current_mapping) is True:
|
|
observed_processing = True
|
|
post_processing_confirmations = 0
|
|
continue
|
|
try:
|
|
result = await service.sync_mapping(current_mapping, force=False)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.warning(
|
|
"Mutation-triggered local Wiki sync failed mapping_id=%s attempt=%s",
|
|
mapping_id,
|
|
quiet_scans + 1,
|
|
exc_info=True,
|
|
)
|
|
continue
|
|
|
|
# A periodic/manual worker may temporarily own the mapping.
|
|
# That is not a completed settle scan; retry after its lease
|
|
# releases instead of silently dropping this mutation.
|
|
if isinstance(result, dict) and result.get("status") == "already_running":
|
|
continue
|
|
|
|
post_processing = await _remote_processing(current_mapping)
|
|
latest = plans.get(mapping_id)
|
|
if latest is None:
|
|
return
|
|
if int(latest["generation"]) != observed_generation:
|
|
continue
|
|
if post_processing is True:
|
|
observed_processing = True
|
|
post_processing_confirmations = 0
|
|
continue
|
|
if isinstance(result, dict) and result.get("deferred_deletion"):
|
|
# The service's destructive-phase fence observed a race
|
|
# that the cheap pre-check missed (or could not verify).
|
|
# Treat it exactly like an explicit processing window and
|
|
# retain the post-processing confirmation scans.
|
|
observed_processing = True
|
|
post_processing_confirmations = 0
|
|
delay_index = min(1, len(retry_delays) - 1)
|
|
deadline = max(
|
|
deadline,
|
|
monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS,
|
|
)
|
|
continue
|
|
|
|
quiet_scans += 1
|
|
# Once WeKnora has explicitly transitioned out of processing,
|
|
# require one short, quiet confirmation when the first scan
|
|
# sees no delta: some WeKnora builds clear ``is_processing``
|
|
# just before publishing the final Wiki directory snapshot.
|
|
# A synchronous edit or a post-processing delta can finish
|
|
# immediately.
|
|
if _sync_changed(result):
|
|
return
|
|
if observed_processing:
|
|
post_processing_confirmations += 1
|
|
if post_processing_confirmations >= 2:
|
|
return
|
|
delay_index = min(1, len(retry_delays) - 1)
|
|
deadline = max(
|
|
deadline,
|
|
monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS,
|
|
)
|
|
finally:
|
|
current = asyncio.current_task()
|
|
if tasks.get(mapping_id) is current:
|
|
tasks.pop(mapping_id, None)
|
|
plan = plans.get(mapping_id)
|
|
if plan is not None and int(plan.get("generation") or 0) == observed_generation:
|
|
plans.pop(mapping_id, None)
|
|
|
|
task = spawn_local_wiki_index_job(app, _run())
|
|
tasks[mapping_id] = task
|
|
return task
|
|
|
|
|
|
def _as_utc(value: Any) -> datetime | None:
|
|
if isinstance(value, datetime):
|
|
parsed = value
|
|
else:
|
|
raw = str(value or "").strip()
|
|
if not raw:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
if parsed.tzinfo is None:
|
|
return parsed.replace(tzinfo=UTC)
|
|
return parsed.astimezone(UTC)
|
|
|
|
|
|
def _mapping_is_due(
|
|
mapping: dict[str, Any],
|
|
state: dict[str, Any] | None,
|
|
remote: dict[str, Any] | None,
|
|
*,
|
|
force_scan: bool,
|
|
) -> bool:
|
|
# Even the low-frequency reconciliation pass must not enumerate a Wiki
|
|
# directory while WeKnora says it is transiently processing. ``force``
|
|
# bypasses only the timestamp preflight; the destructive-phase fence in
|
|
# the sync service remains the final race/data-loss guard.
|
|
if remote is not None and _knowledge_base_is_processing(remote):
|
|
return False
|
|
if force_scan or state is None or int(state.get("index_revision") or 0) <= 0:
|
|
return True
|
|
if str(state.get("state") or "") in {"failed", "partial", "rebuild_required"}:
|
|
return True
|
|
if str(state.get("state") or "") in {"running", "rebuilding"}:
|
|
return False
|
|
if remote is None:
|
|
# Preserve the local snapshot on a transient/incomplete directory read;
|
|
# the low-frequency forced scan remains the deletion/error authority.
|
|
return False
|
|
remote_updated = _as_utc(remote.get("updated_at") or remote.get("updatedAt"))
|
|
last_success = _as_utc(state.get("last_success_at"))
|
|
if remote_updated is None or last_success is None:
|
|
return True
|
|
return remote_updated > last_success
|
|
|
|
|
|
async def _refresh_local_wiki_config(app: FastAPI):
|
|
from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime
|
|
|
|
return await rebuild_local_wiki_index_runtime(app)
|
|
|
|
|
|
def _auto_sync_ready(app: FastAPI, config: Any) -> bool:
|
|
return bool(
|
|
config.enabled
|
|
and config.auto_sync
|
|
and getattr(app.state, "llmwiki_sync_service", None) is not None
|
|
and getattr(app.state, "llmwiki_store", None) is not None
|
|
)
|
|
|
|
|
|
async def _sync_once(
|
|
app: FastAPI,
|
|
*,
|
|
config: Any | None = None,
|
|
force_scan: bool | None = None,
|
|
) -> None:
|
|
config = config or await _refresh_local_wiki_config(app)
|
|
if force_scan is None:
|
|
force_scan = bool(
|
|
getattr(app.state, "llmwiki_force_full_scan_once", False)
|
|
)
|
|
service = getattr(app.state, "llmwiki_sync_service", None)
|
|
mapping_store = getattr(app.state, "llmwiki_store", None)
|
|
if not _auto_sync_ready(app, config) or service is None or mapping_store is None:
|
|
return
|
|
mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
|
|
index_store = getattr(app.state, "llmwiki_index_store", None)
|
|
states: dict[str, dict[str, Any]] = {}
|
|
if index_store is not None:
|
|
states = {
|
|
str(row.get("knowledge_base_mapping_id") or ""): row
|
|
for row in await index_store.list_index_status(
|
|
[str(mapping["id"]) for mapping in mappings]
|
|
)
|
|
if isinstance(row, dict)
|
|
}
|
|
remote_by_id: dict[str, dict[str, Any]] = {}
|
|
client = getattr(service, "client", None)
|
|
list_remote = getattr(client, "list_knowledge_bases", None)
|
|
if callable(list_remote):
|
|
try:
|
|
remote_rows = await list_remote(refresh=True)
|
|
except TypeError:
|
|
remote_rows = await list_remote()
|
|
remote_by_id = {
|
|
str(row.get("id") or ""): row
|
|
for row in remote_rows
|
|
if isinstance(row, dict) and row.get("id")
|
|
}
|
|
if remote_by_id:
|
|
mappings = [
|
|
mapping
|
|
for mapping in mappings
|
|
if _mapping_is_due(
|
|
mapping,
|
|
states.get(str(mapping["id"])),
|
|
remote_by_id.get(str(mapping.get("weknora_id") or "")),
|
|
force_scan=bool(force_scan),
|
|
)
|
|
]
|
|
semaphore = asyncio.Semaphore(config.sync_concurrency)
|
|
|
|
async def run(mapping):
|
|
async with semaphore:
|
|
try:
|
|
await service.sync_mapping(mapping)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True)
|
|
|
|
await asyncio.gather(*(run(mapping) for mapping in mappings))
|
|
|
|
|
|
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
|
|
logger.info(
|
|
"Local Wiki auto-sync scheduler started; runtime config refresh interval=%s seconds",
|
|
_CONFIG_REFRESH_SECONDS,
|
|
)
|
|
last_sync_started_at: float | None = None
|
|
last_synced_runtime_signature: str | None = None
|
|
last_full_scan_started_at: float | None = None
|
|
while True:
|
|
try:
|
|
# Rebuild checks both config.yaml mtime and persisted runtime
|
|
# settings every poll, independently from the much longer sync
|
|
# cadence. Provider/URL changes therefore take effect promptly.
|
|
config = await _refresh_local_wiki_config(app)
|
|
if not _auto_sync_ready(app, config):
|
|
# A later disabled -> enabled transition is a new first valid
|
|
# configuration and should receive an immediate scan.
|
|
last_sync_started_at = None
|
|
last_synced_runtime_signature = None
|
|
else:
|
|
now = monotonic()
|
|
interval = float(config.sync_interval_seconds)
|
|
runtime_signature = getattr(
|
|
app.state,
|
|
"llmwiki_runtime_signature",
|
|
None,
|
|
)
|
|
if (
|
|
last_sync_started_at is None
|
|
or runtime_signature != last_synced_runtime_signature
|
|
or now - last_sync_started_at >= interval
|
|
):
|
|
sync_started_at = now
|
|
force_scan = bool(
|
|
last_full_scan_started_at is None
|
|
or runtime_signature != last_synced_runtime_signature
|
|
or now - last_full_scan_started_at >= max(
|
|
_FULL_SCAN_FALLBACK_SECONDS,
|
|
interval,
|
|
)
|
|
)
|
|
app.state.llmwiki_force_full_scan_once = force_scan
|
|
try:
|
|
await _sync_once(app, config=config)
|
|
finally:
|
|
app.state.llmwiki_force_full_scan_once = False
|
|
last_sync_started_at = sync_started_at
|
|
last_synced_runtime_signature = runtime_signature
|
|
if force_scan:
|
|
last_full_scan_started_at = sync_started_at
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)")
|
|
await asyncio.sleep(_CONFIG_REFRESH_SECONDS)
|