sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -5,11 +5,34 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Coroutine
|
||||
from datetime import UTC, datetime
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_CONFIG_REFRESH_SECONDS = 10.0
|
||||
_MUTATION_SYNC_DELAYS_SECONDS = (1.0, 8.0, 30.0, 90.0, 300.0)
|
||||
_MUTATION_SYNC_MAX_SECONDS = 10 * 60.0
|
||||
_MUTATION_SYNC_CONFIRM_GRACE_SECONDS = 30.0
|
||||
_FULL_SCAN_FALLBACK_SECONDS = 10 * 60.0
|
||||
|
||||
|
||||
def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool:
|
||||
"""Normalize processing markers across old/new WeKnora response shapes."""
|
||||
|
||||
raw_flag = remote.get("is_processing", remote.get("isProcessing"))
|
||||
if isinstance(raw_flag, str):
|
||||
is_processing = raw_flag.strip().lower() in {"1", "true", "yes", "on"}
|
||||
else:
|
||||
is_processing = bool(raw_flag)
|
||||
raw_count = remote.get("processing_count", remote.get("processingCount"))
|
||||
try:
|
||||
processing_count = int(raw_count or 0)
|
||||
except (TypeError, ValueError):
|
||||
processing_count = 0
|
||||
return is_processing or processing_count > 0
|
||||
|
||||
|
||||
def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
|
||||
@@ -25,13 +48,348 @@ def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]
|
||||
return task
|
||||
|
||||
|
||||
async def _sync_once(app: FastAPI) -> None:
|
||||
def schedule_local_wiki_mutation_sync(
|
||||
app: FastAPI,
|
||||
mapping: dict[str, Any],
|
||||
*,
|
||||
delays: tuple[float, ...] | None = None,
|
||||
) -> asyncio.Task | None:
|
||||
"""Debounce one mapping's mutation and retry while WeKnora finishes Wiki work.
|
||||
|
||||
Document ingestion is asynchronous upstream, so a single immediate scan can
|
||||
legitimately observe the old Wiki snapshot. One coordinator per mapping
|
||||
performs a few inexpensive metadata-first scans. A newer write only bumps
|
||||
the generation and resets that coordinator; it never cancels an active
|
||||
sync. Database leases remain the cross-process source of truth.
|
||||
"""
|
||||
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
mapping_id = str(mapping.get("id") or "")
|
||||
if service is None or not mapping_id:
|
||||
return None
|
||||
retry_delays = tuple(max(0.0, float(value)) for value in (delays or _MUTATION_SYNC_DELAYS_SECONDS))
|
||||
if not retry_delays:
|
||||
return None
|
||||
|
||||
plans: dict[str, dict[str, Any]] = getattr(app.state, "llmwiki_mutation_sync_plans", None)
|
||||
if plans is None:
|
||||
plans = {}
|
||||
app.state.llmwiki_mutation_sync_plans = plans
|
||||
tasks: dict[str, asyncio.Task] = getattr(app.state, "llmwiki_mutation_sync_tasks", None)
|
||||
if tasks is None:
|
||||
tasks = {}
|
||||
app.state.llmwiki_mutation_sync_tasks = tasks
|
||||
|
||||
prior = plans.get(mapping_id) or {}
|
||||
wakeup = prior.get("wakeup")
|
||||
if not isinstance(wakeup, asyncio.Event):
|
||||
wakeup = asyncio.Event()
|
||||
plans[mapping_id] = {
|
||||
"generation": int(prior.get("generation") or 0) + 1,
|
||||
"mapping": dict(mapping),
|
||||
"wakeup": wakeup,
|
||||
}
|
||||
running = tasks.get(mapping_id)
|
||||
if running is not None and not running.done():
|
||||
# Interrupt a long backoff, but never cancel an in-flight sync. The
|
||||
# coordinator will observe the newer generation at its next await
|
||||
# boundary and restart the bounded settle sequence from one second.
|
||||
wakeup.set()
|
||||
return running
|
||||
|
||||
async def _remote_processing(current_mapping: dict[str, Any]) -> bool | None:
|
||||
"""Read the cheap KB status without listing every Wiki page.
|
||||
|
||||
``None`` deliberately means "unknown": a transient status-read error
|
||||
must not suppress the actual incremental scan.
|
||||
"""
|
||||
|
||||
client = getattr(service, "client", None)
|
||||
get_remote = getattr(client, "get_knowledge_base", None)
|
||||
remote_id = str(current_mapping.get("weknora_id") or "")
|
||||
if not callable(get_remote) or not remote_id:
|
||||
return None
|
||||
try:
|
||||
remote = await get_remote(remote_id)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not read WeKnora processing status mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
if not isinstance(remote, dict):
|
||||
return None
|
||||
return _knowledge_base_is_processing(remote)
|
||||
|
||||
def _sync_changed(result: Any) -> bool:
|
||||
if not isinstance(result, dict):
|
||||
return False
|
||||
if str(result.get("status") or "").lower() not in {"completed", "idle"}:
|
||||
return False
|
||||
return any(
|
||||
int(result.get(key) or 0) > 0
|
||||
for key in (
|
||||
"added_pages",
|
||||
"updated_pages",
|
||||
"deleted_pages",
|
||||
)
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
observed_generation = -1
|
||||
delay_index = 0
|
||||
quiet_scans = 0
|
||||
observed_processing = False
|
||||
post_processing_confirmations = 0
|
||||
deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS
|
||||
try:
|
||||
while True:
|
||||
plan = plans.get(mapping_id)
|
||||
if plan is None:
|
||||
return
|
||||
generation = int(plan["generation"])
|
||||
if generation != observed_generation:
|
||||
observed_generation = generation
|
||||
delay_index = 0
|
||||
quiet_scans = 0
|
||||
observed_processing = False
|
||||
post_processing_confirmations = 0
|
||||
deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS
|
||||
if not observed_processing and quiet_scans >= len(retry_delays):
|
||||
return
|
||||
remaining = deadline - monotonic()
|
||||
if remaining <= 0:
|
||||
return
|
||||
delay = min(retry_delays[min(delay_index, len(retry_delays) - 1)], remaining)
|
||||
wakeup = plan.get("wakeup")
|
||||
if isinstance(wakeup, asyncio.Event):
|
||||
wakeup.clear()
|
||||
# Re-read after clearing: a mutation scheduled from
|
||||
# another callback/thread between the earlier generation
|
||||
# read and ``clear()`` must not be hidden for a long
|
||||
# backoff interval.
|
||||
latest_after_clear = plans.get(mapping_id)
|
||||
if (
|
||||
latest_after_clear is None
|
||||
or int(latest_after_clear["generation"])
|
||||
!= observed_generation
|
||||
):
|
||||
continue
|
||||
try:
|
||||
await asyncio.wait_for(wakeup.wait(), timeout=delay)
|
||||
except TimeoutError:
|
||||
pass
|
||||
else:
|
||||
# A newer mutation resets the generation and the
|
||||
# backoff on the next loop iteration.
|
||||
continue
|
||||
else:
|
||||
await asyncio.sleep(delay)
|
||||
latest = plans.get(mapping_id)
|
||||
if latest is None:
|
||||
return
|
||||
if int(latest["generation"]) != observed_generation:
|
||||
continue
|
||||
current_mapping = dict(latest["mapping"])
|
||||
delay_index = min(delay_index + 1, len(retry_delays) - 1)
|
||||
|
||||
# Do not repeatedly enumerate the whole Wiki while WeKnora is
|
||||
# still parsing the uploaded document. Processing polls do
|
||||
# not consume one of the reserved quiet-scan opportunities,
|
||||
# so a final post-processing scan is always retained within
|
||||
# the bounded ten-minute settle window.
|
||||
if await _remote_processing(current_mapping) is True:
|
||||
observed_processing = True
|
||||
post_processing_confirmations = 0
|
||||
continue
|
||||
try:
|
||||
result = await service.sync_mapping(current_mapping, force=False)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Mutation-triggered local Wiki sync failed mapping_id=%s attempt=%s",
|
||||
mapping_id,
|
||||
quiet_scans + 1,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
# A periodic/manual worker may temporarily own the mapping.
|
||||
# That is not a completed settle scan; retry after its lease
|
||||
# releases instead of silently dropping this mutation.
|
||||
if isinstance(result, dict) and result.get("status") == "already_running":
|
||||
continue
|
||||
|
||||
post_processing = await _remote_processing(current_mapping)
|
||||
latest = plans.get(mapping_id)
|
||||
if latest is None:
|
||||
return
|
||||
if int(latest["generation"]) != observed_generation:
|
||||
continue
|
||||
if post_processing is True:
|
||||
observed_processing = True
|
||||
post_processing_confirmations = 0
|
||||
continue
|
||||
if isinstance(result, dict) and result.get("deferred_deletion"):
|
||||
# The service's destructive-phase fence observed a race
|
||||
# that the cheap pre-check missed (or could not verify).
|
||||
# Treat it exactly like an explicit processing window and
|
||||
# retain the post-processing confirmation scans.
|
||||
observed_processing = True
|
||||
post_processing_confirmations = 0
|
||||
delay_index = min(1, len(retry_delays) - 1)
|
||||
deadline = max(
|
||||
deadline,
|
||||
monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS,
|
||||
)
|
||||
continue
|
||||
|
||||
quiet_scans += 1
|
||||
# Once WeKnora has explicitly transitioned out of processing,
|
||||
# require one short, quiet confirmation when the first scan
|
||||
# sees no delta: some WeKnora builds clear ``is_processing``
|
||||
# just before publishing the final Wiki directory snapshot.
|
||||
# A synchronous edit or a post-processing delta can finish
|
||||
# immediately.
|
||||
if _sync_changed(result):
|
||||
return
|
||||
if observed_processing:
|
||||
post_processing_confirmations += 1
|
||||
if post_processing_confirmations >= 2:
|
||||
return
|
||||
delay_index = min(1, len(retry_delays) - 1)
|
||||
deadline = max(
|
||||
deadline,
|
||||
monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS,
|
||||
)
|
||||
finally:
|
||||
current = asyncio.current_task()
|
||||
if tasks.get(mapping_id) is current:
|
||||
tasks.pop(mapping_id, None)
|
||||
plan = plans.get(mapping_id)
|
||||
if plan is not None and int(plan.get("generation") or 0) == observed_generation:
|
||||
plans.pop(mapping_id, None)
|
||||
|
||||
task = spawn_local_wiki_index_job(app, _run())
|
||||
tasks[mapping_id] = task
|
||||
return task
|
||||
|
||||
|
||||
def _as_utc(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
else:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _mapping_is_due(
|
||||
mapping: dict[str, Any],
|
||||
state: dict[str, Any] | None,
|
||||
remote: dict[str, Any] | None,
|
||||
*,
|
||||
force_scan: bool,
|
||||
) -> bool:
|
||||
# Even the low-frequency reconciliation pass must not enumerate a Wiki
|
||||
# directory while WeKnora says it is transiently processing. ``force``
|
||||
# bypasses only the timestamp preflight; the destructive-phase fence in
|
||||
# the sync service remains the final race/data-loss guard.
|
||||
if remote is not None and _knowledge_base_is_processing(remote):
|
||||
return False
|
||||
if force_scan or state is None or int(state.get("index_revision") or 0) <= 0:
|
||||
return True
|
||||
if str(state.get("state") or "") in {"failed", "partial", "rebuild_required"}:
|
||||
return True
|
||||
if str(state.get("state") or "") in {"running", "rebuilding"}:
|
||||
return False
|
||||
if remote is None:
|
||||
# Preserve the local snapshot on a transient/incomplete directory read;
|
||||
# the low-frequency forced scan remains the deletion/error authority.
|
||||
return False
|
||||
remote_updated = _as_utc(remote.get("updated_at") or remote.get("updatedAt"))
|
||||
last_success = _as_utc(state.get("last_success_at"))
|
||||
if remote_updated is None or last_success is None:
|
||||
return True
|
||||
return remote_updated > last_success
|
||||
|
||||
|
||||
async def _refresh_local_wiki_config(app: FastAPI):
|
||||
from app.gateway.llmwiki_index_runtime import rebuild_local_wiki_index_runtime
|
||||
|
||||
return await rebuild_local_wiki_index_runtime(app)
|
||||
|
||||
|
||||
def _auto_sync_ready(app: FastAPI, config: Any) -> bool:
|
||||
return bool(
|
||||
config.enabled
|
||||
and config.auto_sync
|
||||
and getattr(app.state, "llmwiki_sync_service", None) is not None
|
||||
and getattr(app.state, "llmwiki_store", None) is not None
|
||||
)
|
||||
|
||||
|
||||
async def _sync_once(
|
||||
app: FastAPI,
|
||||
*,
|
||||
config: Any | None = None,
|
||||
force_scan: bool | None = None,
|
||||
) -> None:
|
||||
config = config or await _refresh_local_wiki_config(app)
|
||||
if force_scan is None:
|
||||
force_scan = bool(
|
||||
getattr(app.state, "llmwiki_force_full_scan_once", False)
|
||||
)
|
||||
service = getattr(app.state, "llmwiki_sync_service", None)
|
||||
mapping_store = getattr(app.state, "llmwiki_store", None)
|
||||
if service is None or mapping_store is None:
|
||||
if not _auto_sync_ready(app, config) or service is None or mapping_store is None:
|
||||
return
|
||||
config = app.state.config.llmwiki.local_wiki_index
|
||||
mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
|
||||
index_store = getattr(app.state, "llmwiki_index_store", None)
|
||||
states: dict[str, dict[str, Any]] = {}
|
||||
if index_store is not None:
|
||||
states = {
|
||||
str(row.get("knowledge_base_mapping_id") or ""): row
|
||||
for row in await index_store.list_index_status(
|
||||
[str(mapping["id"]) for mapping in mappings]
|
||||
)
|
||||
if isinstance(row, dict)
|
||||
}
|
||||
remote_by_id: dict[str, dict[str, Any]] = {}
|
||||
client = getattr(service, "client", None)
|
||||
list_remote = getattr(client, "list_knowledge_bases", None)
|
||||
if callable(list_remote):
|
||||
try:
|
||||
remote_rows = await list_remote(refresh=True)
|
||||
except TypeError:
|
||||
remote_rows = await list_remote()
|
||||
remote_by_id = {
|
||||
str(row.get("id") or ""): row
|
||||
for row in remote_rows
|
||||
if isinstance(row, dict) and row.get("id")
|
||||
}
|
||||
if remote_by_id:
|
||||
mappings = [
|
||||
mapping
|
||||
for mapping in mappings
|
||||
if _mapping_is_due(
|
||||
mapping,
|
||||
states.get(str(mapping["id"])),
|
||||
remote_by_id.get(str(mapping.get("weknora_id") or "")),
|
||||
force_scan=bool(force_scan),
|
||||
)
|
||||
]
|
||||
semaphore = asyncio.Semaphore(config.sync_concurrency)
|
||||
|
||||
async def run(mapping):
|
||||
@@ -47,16 +405,57 @@ async def _sync_once(app: FastAPI) -> None:
|
||||
|
||||
|
||||
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
|
||||
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
|
||||
logger.info(
|
||||
"Local Wiki auto-sync scheduler started; first scan is immediate, interval=%s seconds",
|
||||
interval,
|
||||
"Local Wiki auto-sync scheduler started; runtime config refresh interval=%s seconds",
|
||||
_CONFIG_REFRESH_SECONDS,
|
||||
)
|
||||
last_sync_started_at: float | None = None
|
||||
last_synced_runtime_signature: str | None = None
|
||||
last_full_scan_started_at: float | None = None
|
||||
while True:
|
||||
try:
|
||||
await _sync_once(app)
|
||||
# Rebuild checks both config.yaml mtime and persisted runtime
|
||||
# settings every poll, independently from the much longer sync
|
||||
# cadence. Provider/URL changes therefore take effect promptly.
|
||||
config = await _refresh_local_wiki_config(app)
|
||||
if not _auto_sync_ready(app, config):
|
||||
# A later disabled -> enabled transition is a new first valid
|
||||
# configuration and should receive an immediate scan.
|
||||
last_sync_started_at = None
|
||||
last_synced_runtime_signature = None
|
||||
else:
|
||||
now = monotonic()
|
||||
interval = float(config.sync_interval_seconds)
|
||||
runtime_signature = getattr(
|
||||
app.state,
|
||||
"llmwiki_runtime_signature",
|
||||
None,
|
||||
)
|
||||
if (
|
||||
last_sync_started_at is None
|
||||
or runtime_signature != last_synced_runtime_signature
|
||||
or now - last_sync_started_at >= interval
|
||||
):
|
||||
sync_started_at = now
|
||||
force_scan = bool(
|
||||
last_full_scan_started_at is None
|
||||
or runtime_signature != last_synced_runtime_signature
|
||||
or now - last_full_scan_started_at >= max(
|
||||
_FULL_SCAN_FALLBACK_SECONDS,
|
||||
interval,
|
||||
)
|
||||
)
|
||||
app.state.llmwiki_force_full_scan_once = force_scan
|
||||
try:
|
||||
await _sync_once(app, config=config)
|
||||
finally:
|
||||
app.state.llmwiki_force_full_scan_once = False
|
||||
last_sync_started_at = sync_started_at
|
||||
last_synced_runtime_signature = runtime_signature
|
||||
if force_scan:
|
||||
last_full_scan_started_at = sync_started_at
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(interval)
|
||||
await asyncio.sleep(_CONFIG_REFRESH_SECONDS)
|
||||
Reference in new issue
Block a user