sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -17,6 +17,193 @@ _MUTATION_SYNC_DELAYS_SECONDS = (1.0, 8.0, 30.0, 90.0, 300.0)
_MUTATION_SYNC_MAX_SECONDS = 10 * 60.0
_MUTATION_SYNC_CONFIRM_GRACE_SECONDS = 30.0
_FULL_SCAN_FALLBACK_SECONDS = 10 * 60.0
_WIKI_CONFIG_UPGRADE_RETRY_SECONDS = 5 * 60.0
def _has_focus_constraint(mapping: dict[str, Any]) -> bool:
"""Return whether this mapping needs strict focus reconciliation.
Both current ``wiki_directory_paths`` and legacy
``wiki_parent_directories`` values are prompt-only references. They never
trigger post-generation folder mutations.
"""
return bool(
str(mapping.get("wiki_focus_instructions") or "").strip()
)
def _wiki_config_schema_version(mapping: dict[str, Any]) -> int:
try:
return int(mapping.get("wiki_config_schema_version") or 0)
except (TypeError, ValueError):
return 0
async def _upgrade_wiki_generation_configs_once(
app: FastAPI,
*,
client: Any | None = None,
) -> dict[str, int]:
"""Replace legacy effective prompts with the current reference semantics.
Raw user inputs in the DeerFlow database are authoritative. A failed
provider write leaves the old schema version in place and records an error,
so a later scheduler pass retries safely. Each mapping is isolated: one bad
knowledge base never blocks application startup or the remaining upgrades.
"""
from app.gateway.wiki_directory_constraints import (
normalize_directory_paths,
paths_from_parent_directories,
)
from app.gateway.wiki_generation_config import (
WIKI_CONFIG_SCHEMA_VERSION,
compile_wiki_instructions,
normalize_focus_instructions,
)
mapping_store = getattr(app.state, "llmwiki_store", None)
list_visible = getattr(mapping_store, "list_visible", None)
update_mapping = getattr(mapping_store, "update_mapping", None)
if not callable(list_visible) or not callable(update_mapping):
return {"eligible": 0, "upgraded": 0, "failed": 0}
try:
rows = await list_visible("system", scope="all", is_admin=True)
except asyncio.CancelledError:
raise
except Exception:
logger.warning("Could not list Wiki mappings for prompt-schema upgrade", exc_info=True)
return {"eligible": 0, "upgraded": 0, "failed": 0}
targets = [
row
for row in rows
if isinstance(row, dict)
and row.get("wiki_generation_managed") is True
# Schema 2 is the first version that guarantees complete raw prompt
# fields. Older/unknown rows may only have provider-side instructions;
# recompiling those from missing raw values could erase user content.
and _wiki_config_schema_version(row) == 2
]
if not targets:
return {"eligible": 0, "upgraded": 0, "failed": 0}
remote_client = client
if remote_client is None:
service = getattr(app.state, "llmwiki_sync_service", None)
remote_client = getattr(service, "client", None)
if remote_client is None:
from deerflow.integrations.weknora.runtime import (
build_weknora_client,
get_resolved_llmwiki_runtime,
)
runtime = get_resolved_llmwiki_runtime(
getattr(app.state, "config", None)
)
if not runtime.weknora_enabled:
return {"eligible": len(targets), "upgraded": 0, "failed": 0}
try:
remote_client = build_weknora_client(runtime)
except Exception:
logger.warning("Wiki prompt-schema upgrade client is unavailable", exc_info=True)
return {"eligible": len(targets), "upgraded": 0, "failed": 0}
upgraded = 0
failed = 0
semaphore = asyncio.Semaphore(4)
async def run(mapping: dict[str, Any]) -> bool:
mapping_id = str(mapping.get("id") or "")
try:
remote_id = str(mapping.get("weknora_id") or "").strip()
if not mapping_id or not remote_id:
raise ValueError("Wiki mapping or WeKnora knowledge base ID is missing")
raw_paths = mapping.get("wiki_directory_paths")
directory_paths = (
normalize_directory_paths(raw_paths)
if raw_paths is not None
else paths_from_parent_directories(
mapping.get("wiki_parent_directories")
)
)
compiled = compile_wiki_instructions(
content_instructions=str(
mapping.get("wiki_content_instructions_raw") or ""
),
extraction_instructions=str(
mapping.get("wiki_extraction_instructions_raw") or ""
),
focus_instructions=normalize_focus_instructions(
str(mapping.get("wiki_focus_instructions") or "")
),
directory_paths=directory_paths,
)
async with semaphore:
await remote_client.update_knowledge_base(
remote_id,
{
"wiki_config": {
"content_instructions": compiled.content_instructions,
"extraction_instructions": compiled.extraction_instructions,
}
},
)
await update_mapping(
mapping_id,
wiki_config_schema_version=WIKI_CONFIG_SCHEMA_VERSION,
wiki_config_revision=int(
mapping.get("wiki_config_revision") or 0
)
+ 1,
wiki_config_apply_status="active",
wiki_config_apply_error=None,
)
return True
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning(
"Wiki prompt-schema upgrade failed mapping_id=%s",
mapping_id,
exc_info=True,
)
if mapping_id:
try:
await update_mapping(
mapping_id,
wiki_config_apply_status="error",
wiki_config_apply_error=str(exc)[:4000],
)
except asyncio.CancelledError:
raise
except Exception:
logger.warning(
"Could not record Wiki prompt-schema upgrade failure mapping_id=%s",
mapping_id,
exc_info=True,
)
return False
results = await asyncio.gather(*(run(mapping) for mapping in targets))
upgraded = sum(result is True for result in results)
failed = sum(result is False for result in results)
if upgraded or failed:
logger.info(
"Wiki prompt-schema upgrade finished eligible=%s upgraded=%s failed=%s",
len(targets),
upgraded,
failed,
)
return {"eligible": len(targets), "upgraded": upgraded, "failed": failed}
def _has_directory_constraints(mapping: dict[str, Any]) -> bool:
return bool(
mapping.get("wiki_directory_paths")
or mapping.get("wiki_parent_directories")
)
def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool:
@@ -35,6 +222,105 @@ def _knowledge_base_is_processing(remote: dict[str, Any]) -> bool:
return is_processing or processing_count > 0
def _document_is_processing(document: dict[str, Any]) -> bool:
for key in (
"pending_subtasks_count",
"pendingSubtasksCount",
"pending_subtasks",
):
try:
if int(document.get(key) or 0) > 0:
return True
except (TypeError, ValueError):
continue
active = {
"pending",
"queued",
"processing",
"running",
"finalizing",
"summarizing",
"extracting",
"generating",
"in_progress",
"in-progress",
}
return any(
str(document.get(key) or "").strip().casefold() in active
for key in (
"processing_status",
"processingStatus",
"parse_status",
"parseStatus",
"summary_status",
"summaryStatus",
"wiki_status",
"wikiStatus",
)
)
async def _remote_generation_is_processing(
client: Any,
mapping: dict[str, Any],
*,
remote: dict[str, Any] | None = None,
) -> bool | None:
"""Check document Wiki subtasks as well as the coarse KB parse flag.
WeKnora can clear ``knowledge_base.is_processing`` while Wiki ingest still
has pages in draft/finalizing. The per-document pending-subtask count is
the reliable completion signal for generation constraints.
"""
remote_id = str(mapping.get("weknora_id") or "")
if not remote_id:
return None
if remote is None:
get_remote = getattr(client, "get_knowledge_base", None)
if callable(get_remote):
try:
remote = await get_remote(remote_id)
except asyncio.CancelledError:
raise
except Exception:
return None
if isinstance(remote, dict) and _knowledge_base_is_processing(remote):
return True
list_documents = getattr(client, "list_documents", None)
if not callable(list_documents):
return False if isinstance(remote, dict) else None
page = 1
page_size = 100
try:
while True:
payload = await list_documents(
remote_id,
page=page,
page_size=page_size,
)
if not isinstance(payload, dict):
return None
rows = payload.get("items")
if not isinstance(rows, list):
return None
if any(
_document_is_processing(row)
for row in rows
if isinstance(row, dict)
):
return True
total = int(payload.get("total") or len(rows))
if page * page_size >= total or len(rows) < page_size:
return False
page += 1
except asyncio.CancelledError:
raise
except Exception:
return None
def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
"""Keep fire-and-forget index jobs strongly referenced until completion."""
@@ -53,8 +339,10 @@ def schedule_local_wiki_mutation_sync(
mapping: dict[str, Any],
*,
delays: tuple[float, ...] | None = None,
client: Any | None = None,
sync_local: bool = True,
) -> asyncio.Task | None:
"""Debounce one mapping's mutation and retry while WeKnora finishes Wiki work.
"""Debounce one mapping's mutation until WeKnora finishes Wiki work.
Document ingestion is asynchronous upstream, so a single immediate scan can
legitimately observe the old Wiki snapshot. One coordinator per mapping
@@ -64,8 +352,10 @@ def schedule_local_wiki_mutation_sync(
"""
service = getattr(app.state, "llmwiki_sync_service", None)
constrained = _has_focus_constraint(mapping)
local_sync_enabled = bool(sync_local and service is not None)
mapping_id = str(mapping.get("id") or "")
if service is None or not mapping_id:
if not mapping_id:
return None
retry_delays = tuple(max(0.0, float(value)) for value in (delays or _MUTATION_SYNC_DELAYS_SECONDS))
if not retry_delays:
@@ -81,6 +371,32 @@ def schedule_local_wiki_mutation_sync(
app.state.llmwiki_mutation_sync_tasks = tasks
prior = plans.get(mapping_id) or {}
remote_client = (
client
or getattr(service, "client", None)
or prior.get("client")
)
running = tasks.get(mapping_id)
# Clearing a focus must supersede an already-running settle plan even when
# the optional local index is disabled. With no prior plan there is no work
# to schedule.
if not local_sync_enabled and not constrained:
if running is None or running.done() or not prior:
return None
wakeup = prior.get("wakeup")
if not isinstance(wakeup, asyncio.Event):
wakeup = asyncio.Event()
plans[mapping_id] = {
"generation": int(prior.get("generation") or 0) + 1,
"mapping": dict(mapping),
"wakeup": wakeup,
"client": remote_client,
"sync_local": False,
}
wakeup.set()
return running
if constrained and remote_client is None:
return None
wakeup = prior.get("wakeup")
if not isinstance(wakeup, asyncio.Event):
wakeup = asyncio.Event()
@@ -88,8 +404,9 @@ def schedule_local_wiki_mutation_sync(
"generation": int(prior.get("generation") or 0) + 1,
"mapping": dict(mapping),
"wakeup": wakeup,
"client": remote_client,
"sync_local": local_sync_enabled,
}
running = tasks.get(mapping_id)
if running is not None and not running.done():
# Interrupt a long backoff, but never cancel an in-flight sync. The
# coordinator will observe the newer generation at its next await
@@ -97,32 +414,26 @@ def schedule_local_wiki_mutation_sync(
wakeup.set()
return running
async def _remote_processing(current_mapping: dict[str, Any]) -> bool | None:
"""Read the cheap KB status without listing every Wiki page.
async def _remote_processing(
current_mapping: dict[str, Any],
current_client: Any,
) -> bool | None:
"""Read KB and per-document generation status.
``None`` deliberately means "unknown": a transient status-read error
must not suppress the actual incremental scan.
must not be mistaken for a completed Wiki generation.
"""
client = getattr(service, "client", None)
get_remote = getattr(client, "get_knowledge_base", None)
remote_id = str(current_mapping.get("weknora_id") or "")
if not callable(get_remote) or not remote_id:
return None
try:
remote = await get_remote(remote_id)
except asyncio.CancelledError:
raise
except Exception:
status = await _remote_generation_is_processing(
current_client,
current_mapping,
)
if status is None:
logger.debug(
"Could not read WeKnora processing status mapping_id=%s",
"Could not read WeKnora generation status mapping_id=%s",
mapping_id,
exc_info=True,
)
return None
if not isinstance(remote, dict):
return None
return _knowledge_base_is_processing(remote)
return status
def _sync_changed(result: Any) -> bool:
if not isinstance(result, dict):
@@ -194,6 +505,8 @@ def schedule_local_wiki_mutation_sync(
if int(latest["generation"]) != observed_generation:
continue
current_mapping = dict(latest["mapping"])
current_client = latest.get("client")
current_sync_local = bool(latest.get("sync_local"))
delay_index = min(delay_index + 1, len(retry_delays) - 1)
# Do not repeatedly enumerate the whole Wiki while WeKnora is
@@ -201,11 +514,84 @@ def schedule_local_wiki_mutation_sync(
# not consume one of the reserved quiet-scan opportunities,
# so a final post-processing scan is always retained within
# the bounded ten-minute settle window.
if await _remote_processing(current_mapping) is True:
processing = await _remote_processing(
current_mapping,
current_client,
)
if processing is True:
observed_processing = True
post_processing_confirmations = 0
# Upstream Wiki generation can legitimately take longer
# than the bounded quiet-snapshot window. Keep the
# coordinator alive while the provider explicitly reports
# active work, and start the timeout from the latest
# positive processing observation instead of the upload.
deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS
continue
if processing is None and _has_focus_constraint(current_mapping):
observed_processing = True
post_processing_confirmations = 0
continue
if (
observed_processing
and _has_focus_constraint(current_mapping)
and post_processing_confirmations == 0
):
# Focus reconciliation can delete out-of-scope generated
# pages. Require a second consecutive completed status
# before that destructive phase.
post_processing_confirmations = 1
delay_index = min(1, len(retry_delays) - 1)
deadline = max(
deadline,
monotonic() + _MUTATION_SYNC_CONFIRM_GRACE_SECONDS,
)
continue
try:
if _has_focus_constraint(current_mapping):
from app.gateway.wiki_directory_constraints import (
reconcile_fixed_parent_directories,
)
await reconcile_fixed_parent_directories(
current_client,
current_mapping,
)
if not current_sync_local or service is None:
# Strict focus filtering remains useful even when the
# optional DeerFlow-local vector mirror is disabled.
latest = plans.get(mapping_id)
if (
latest is not None
and int(latest["generation"])
!= observed_generation
):
continue
if observed_processing:
post_processing = await _remote_processing(
current_mapping,
current_client,
)
latest = plans.get(mapping_id)
if (
latest is not None
and int(latest["generation"])
!= observed_generation
):
continue
if post_processing is not False:
post_processing_confirmations = 0
continue
post_processing_confirmations += 1
if post_processing_confirmations < 2:
delay_index = min(1, len(retry_delays) - 1)
deadline = max(
deadline,
monotonic()
+ _MUTATION_SYNC_CONFIRM_GRACE_SECONDS,
)
continue
return
result = await service.sync_mapping(current_mapping, force=False)
except asyncio.CancelledError:
raise
@@ -223,14 +609,30 @@ def schedule_local_wiki_mutation_sync(
# releases instead of silently dropping this mutation.
if isinstance(result, dict) and result.get("status") == "already_running":
continue
# The sync service already owns the provider-wide exponential
# breaker. A mutation settle loop must not burn all of its
# delayed attempts while that breaker is open.
if isinstance(result, dict) and result.get("status") == "provider_backoff":
latest = plans.get(mapping_id)
if (
latest is not None
and int(latest["generation"]) != observed_generation
):
continue
return
post_processing = await _remote_processing(current_mapping)
post_processing = await _remote_processing(current_mapping, current_client)
latest = plans.get(mapping_id)
if latest is None:
return
if int(latest["generation"]) != observed_generation:
continue
if post_processing is True:
observed_processing = True
post_processing_confirmations = 0
deadline = monotonic() + _MUTATION_SYNC_MAX_SECONDS
continue
if post_processing is None and _has_focus_constraint(current_mapping):
observed_processing = True
post_processing_confirmations = 0
continue
@@ -355,7 +757,16 @@ async def _sync_once(
mapping_store = getattr(app.state, "llmwiki_store", None)
if not _auto_sync_ready(app, config) or service is None or mapping_store is None:
return
mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
mappings = [
row
for row in await mapping_store.list_visible(
"system",
scope="all",
is_admin=True,
)
if row.get("wiki_index_enabled", True)
or _has_focus_constraint(row)
]
index_store = getattr(app.state, "llmwiki_index_store", None)
states: dict[str, dict[str, Any]] = {}
if index_store is not None:
@@ -380,22 +791,55 @@ async def _sync_once(
if isinstance(row, dict) and row.get("id")
}
if remote_by_id:
def should_run(mapping: dict[str, Any]) -> bool:
remote = remote_by_id.get(str(mapping.get("weknora_id") or ""))
if not mapping.get("wiki_index_enabled", True):
return bool(
force_scan
and _has_focus_constraint(mapping)
and remote is not None
and not _knowledge_base_is_processing(remote)
)
return _mapping_is_due(
mapping,
states.get(str(mapping["id"])),
remote,
force_scan=bool(force_scan),
)
mappings = [
mapping
for mapping in mappings
if _mapping_is_due(
mapping,
states.get(str(mapping["id"])),
remote_by_id.get(str(mapping.get("weknora_id") or "")),
force_scan=bool(force_scan),
)
if should_run(mapping)
]
elif not force_scan:
mappings = [
mapping
for mapping in mappings
if mapping.get("wiki_index_enabled", True)
]
semaphore = asyncio.Semaphore(config.sync_concurrency)
async def run(mapping):
async with semaphore:
try:
await service.sync_mapping(mapping)
if _has_focus_constraint(mapping):
processing = await _remote_generation_is_processing(
client,
mapping,
remote=remote_by_id.get(
str(mapping.get("weknora_id") or "")
),
)
if processing is not False:
return
from app.gateway.wiki_directory_constraints import (
reconcile_fixed_parent_directories,
)
await reconcile_fixed_parent_directories(client, mapping)
if mapping.get("wiki_index_enabled", True):
await service.sync_mapping(mapping)
except asyncio.CancelledError:
raise
except Exception:
@@ -406,25 +850,36 @@ async def _sync_once(
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
logger.info(
"Local Wiki auto-sync scheduler started; runtime config refresh interval=%s seconds",
"Wiki background coordinator started; runtime config refresh interval=%s seconds",
_CONFIG_REFRESH_SECONDS,
)
last_sync_started_at: float | None = None
last_synced_runtime_signature: str | None = None
last_full_scan_started_at: float | None = None
last_prompt_schema_upgrade_started_at: float | None = None
while True:
try:
# Rebuild checks both config.yaml mtime and persisted runtime
# settings every poll, independently from the much longer sync
# cadence. Provider/URL changes therefore take effect promptly.
config = await _refresh_local_wiki_config(app)
if not _auto_sync_ready(app, config):
# A later disabled -> enabled transition is a new first valid
# configuration and should receive an immediate scan.
now = monotonic()
auto_sync_ready = _auto_sync_ready(app, config)
if not auto_sync_ready:
# With automatic sync off, the scheduler must remain local-only:
# no periodic WeKnora page scan, prompt rewrite, or directory
# reconciliation. Explicit user/API calls remain responsible
# for remote refreshes.
last_sync_started_at = None
last_synced_runtime_signature = None
else:
now = monotonic()
if (
last_prompt_schema_upgrade_started_at is None
or now - last_prompt_schema_upgrade_started_at
>= _WIKI_CONFIG_UPGRADE_RETRY_SECONDS
):
last_prompt_schema_upgrade_started_at = now
await _upgrade_wiki_generation_configs_once(app)
interval = float(config.sync_interval_seconds)
runtime_signature = getattr(
app.state,