初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,267 @@
"""Conditional LLMWiki provider configuration.
DeerFlow owns only the connection to WeKnora. Database, Redis, document
parsing, storage and model configuration stay entirely in WeKnora's own
deployment files. An effective non-empty API URL is the sole provider switch.
"""
from __future__ import annotations
from dataclasses import dataclass
from urllib.parse import urlsplit, urlunsplit
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
def normalize_service_url(value: str | None) -> str:
"""Normalize an optional HTTP(S) service URL and reject embedded secrets."""
raw = str(value or "").strip()
if not raw:
return ""
parsed = urlsplit(raw)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("WeKnora addresses must be absolute http(s) URLs")
if parsed.username is not None or parsed.password is not None:
raise ValueError("WeKnora addresses must not contain credentials")
if parsed.query or parsed.fragment:
raise ValueError("WeKnora addresses must not contain query strings or fragments")
path = parsed.path.rstrip("/")
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
class WeKnoraConnectionConfig(BaseModel):
"""The complete DeerFlow-owned connection surface for WeKnora."""
api_base_url: str = Field(default="", description="WeKnora app API base URL; non-empty enables WeKnora mode")
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
admin_email: str = Field(
default="",
description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
admin_password: str = Field(
default="",
repr=False,
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class WeKnoraDirectGraphConfig(BaseModel):
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
direct_write_enabled: bool = False
neo4j_http_url: str = ""
username: str = ""
password: str = Field(default="", repr=False)
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
timeout_seconds: float = Field(default=30, gt=0, le=300)
model_config = ConfigDict(extra="ignore")
@field_validator("neo4j_http_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
model: str = "bge-embedding-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
local_model_path: str = ""
local_cache_dir: str = ""
local_process_isolation: bool = True
local_threads: int = Field(default=2, ge=1, le=32)
dimensions: int | None = Field(
default=None,
ge=1,
le=65536,
description="Expected response vector dimensions; not sent to the embedding endpoint.",
)
batch_size: int = Field(default=32, ge=1, le=512)
timeout_seconds: float = Field(default=30, gt=0, le=300)
max_retries: int = Field(default=3, ge=0, le=10)
model_config = ConfigDict(extra="ignore")
@field_validator("provider")
@classmethod
def _validate_provider(cls, value: str) -> str:
normalized = value.strip().lower()
if normalized not in {"openai", "openai_compatible", "local"}:
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
return normalized
@field_validator("base_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiExternalApiConfig(BaseModel):
enabled: bool = False
api_key: str = Field(default="", repr=False)
previous_api_key: str = Field(default="", repr=False)
max_query_chars: int = Field(default=1000, ge=1, le=10000)
max_knowledge_bases: int = Field(default=20, ge=1, le=100)
max_results: int = Field(default=50, ge=1, le=100)
default_results: int = Field(default=10, ge=1, le=100)
include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
requests_per_minute: int = Field(default=60, ge=1, le=100000)
model_config = ConfigDict(extra="ignore")
@model_validator(mode="after")
def _validate_enabled_key(self):
if self.enabled and not self.api_key.strip():
raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
if self.default_results > self.max_results:
raise ValueError("external_api.default_results must not exceed max_results")
return self
class LocalWikiIndexConfig(BaseModel):
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
enabled: bool = False
frontend_base_url: str = Field(
default="http://127.0.0.1:5174",
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
)
auto_sync: bool = True
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32)
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
sync_max_retries: int = Field(default=3, ge=0, le=10)
sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
chunk_min_chars: int = Field(default=80, ge=1, le=10000)
title_prefix_enabled: bool = True
embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
engine: str = "numpy_exact"
strict_vector: bool = True
top_k_sections: int = Field(default=30, ge=1, le=1000)
top_k_pages: int = Field(default=8, ge=1, le=100)
max_sections_per_page: int = Field(default=2, ge=1, le=20)
min_similarity: float = Field(default=0.35, ge=-1, le=1)
keyword_boost_enabled: bool = False
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
database_batch_size: int = Field(default=50, ge=1, le=500)
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
model_config = ConfigDict(extra="ignore")
@field_validator("frontend_base_url")
@classmethod
def _validate_frontend_base_url(cls, value: str) -> str:
normalized = normalize_service_url(value)
if not normalized:
raise ValueError("local Wiki index frontend_base_url is required")
return normalized
@model_validator(mode="before")
@classmethod
def _migrate_legacy_flags(cls, value):
if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value:
value = {**value, "auto_sync": value.get("sync_enabled")}
return value
@property
def sync_enabled(self) -> bool:
"""Backward-compatible status name; automatic work is one explicit switch."""
return self.auto_sync
@property
def internal_search_enabled(self) -> bool:
return self.enabled
@property
def reference_ui_enabled(self) -> bool:
return self.enabled
@model_validator(mode="after")
def _validate_index(self):
if self.chunk_overlap_chars >= self.chunk_max_chars:
raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
if self.chunk_min_chars > self.chunk_max_chars:
raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
if self.top_k_pages > self.top_k_sections:
raise ValueError("top_k_pages must not exceed top_k_sections")
if self.engine != "numpy_exact":
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
if self.enabled and not self.embedding.enabled:
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
return self
class LlmWikiConfig(BaseModel):
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
model_config = ConfigDict(extra="ignore")
class LlmWikiRuntimeOverride(BaseModel):
"""Administrator-written URL override stored in system settings.
``enabled=False`` means "use config.yaml". When enabled, an empty API URL
deliberately forces legacy mode even if the source file has an address.
Secrets are never stored here.
"""
enabled: bool = False
api_base_url: str = ""
web_base_url: str = ""
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
@dataclass(frozen=True)
class ResolvedLlmWikiRuntime:
provider: str
api_base_url: str
web_base_url: str
admin_email: str
admin_password: str
override_enabled: bool
@property
def weknora_enabled(self) -> bool:
return self.provider == "weknora"
def resolve_llmwiki_runtime(
config: LlmWikiConfig,
override: LlmWikiRuntimeOverride | None = None,
) -> ResolvedLlmWikiRuntime:
"""Resolve the deterministic provider from file config plus URL override."""
use_override = bool(override and override.enabled)
api_base_url = override.api_base_url if use_override and override else config.weknora.api_base_url
web_base_url = override.web_base_url if use_override and override else config.weknora.web_base_url
return ResolvedLlmWikiRuntime(
provider="weknora" if api_base_url else "legacy",
api_base_url=api_base_url,
web_base_url=web_base_url,
admin_email=config.weknora.admin_email,
admin_password=config.weknora.admin_password,
override_enabled=use_override,
)