初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,267 @@
|
||||
"""Conditional LLMWiki provider configuration.
|
||||
|
||||
DeerFlow owns only the connection to WeKnora. Database, Redis, document
|
||||
parsing, storage and model configuration stay entirely in WeKnora's own
|
||||
deployment files. An effective non-empty API URL is the sole provider switch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def normalize_service_url(value: str | None) -> str:
|
||||
"""Normalize an optional HTTP(S) service URL and reject embedded secrets."""
|
||||
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("WeKnora addresses must be absolute http(s) URLs")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError("WeKnora addresses must not contain credentials")
|
||||
if parsed.query or parsed.fragment:
|
||||
raise ValueError("WeKnora addresses must not contain query strings or fragments")
|
||||
path = parsed.path.rstrip("/")
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
|
||||
|
||||
|
||||
class WeKnoraConnectionConfig(BaseModel):
|
||||
"""The complete DeerFlow-owned connection surface for WeKnora."""
|
||||
|
||||
api_base_url: str = Field(default="", description="WeKnora app API base URL; non-empty enables WeKnora mode")
|
||||
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
|
||||
admin_email: str = Field(
|
||||
default="",
|
||||
description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
||||
)
|
||||
admin_password: str = Field(
|
||||
default="",
|
||||
repr=False,
|
||||
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
||||
)
|
||||
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("api_base_url", "web_base_url")
|
||||
@classmethod
|
||||
def _validate_url(cls, value: str) -> str:
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class WeKnoraDirectGraphConfig(BaseModel):
|
||||
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
|
||||
|
||||
direct_write_enabled: bool = False
|
||||
neo4j_http_url: str = ""
|
||||
username: str = ""
|
||||
password: str = Field(default="", repr=False)
|
||||
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
|
||||
timeout_seconds: float = Field(default=30, gt=0, le=300)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("neo4j_http_url")
|
||||
@classmethod
|
||||
def _validate_url(cls, value: str) -> str:
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class LocalWikiEmbeddingConfig(BaseModel):
|
||||
enabled: bool = True
|
||||
provider: str = "openai"
|
||||
model: str = "bge-embedding-m3"
|
||||
base_url: str = ""
|
||||
api_key: str = Field(default="", repr=False)
|
||||
local_model_path: str = ""
|
||||
local_cache_dir: str = ""
|
||||
local_process_isolation: bool = True
|
||||
local_threads: int = Field(default=2, ge=1, le=32)
|
||||
dimensions: int | None = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
le=65536,
|
||||
description="Expected response vector dimensions; not sent to the embedding endpoint.",
|
||||
)
|
||||
batch_size: int = Field(default=32, ge=1, le=512)
|
||||
timeout_seconds: float = Field(default=30, gt=0, le=300)
|
||||
max_retries: int = Field(default=3, ge=0, le=10)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("provider")
|
||||
@classmethod
|
||||
def _validate_provider(cls, value: str) -> str:
|
||||
normalized = value.strip().lower()
|
||||
if normalized not in {"openai", "openai_compatible", "local"}:
|
||||
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
|
||||
return normalized
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def _validate_url(cls, value: str) -> str:
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class LocalWikiExternalApiConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
api_key: str = Field(default="", repr=False)
|
||||
previous_api_key: str = Field(default="", repr=False)
|
||||
max_query_chars: int = Field(default=1000, ge=1, le=10000)
|
||||
max_knowledge_bases: int = Field(default=20, ge=1, le=100)
|
||||
max_results: int = Field(default=50, ge=1, le=100)
|
||||
default_results: int = Field(default=10, ge=1, le=100)
|
||||
include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
|
||||
requests_per_minute: int = Field(default=60, ge=1, le=100000)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_enabled_key(self):
|
||||
if self.enabled and not self.api_key.strip():
|
||||
raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
|
||||
if self.default_results > self.max_results:
|
||||
raise ValueError("external_api.default_results must not exceed max_results")
|
||||
return self
|
||||
|
||||
|
||||
class LocalWikiIndexConfig(BaseModel):
|
||||
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
|
||||
|
||||
enabled: bool = False
|
||||
frontend_base_url: str = Field(
|
||||
default="http://127.0.0.1:5174",
|
||||
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
|
||||
)
|
||||
auto_sync: bool = True
|
||||
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
|
||||
sync_page_size: int = Field(default=100, ge=1, le=500)
|
||||
sync_concurrency: int = Field(default=3, ge=1, le=32)
|
||||
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
|
||||
sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
|
||||
sync_max_retries: int = Field(default=3, ge=0, le=10)
|
||||
sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
|
||||
chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
|
||||
chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
|
||||
chunk_min_chars: int = Field(default=80, ge=1, le=10000)
|
||||
title_prefix_enabled: bool = True
|
||||
embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
|
||||
engine: str = "numpy_exact"
|
||||
strict_vector: bool = True
|
||||
top_k_sections: int = Field(default=30, ge=1, le=1000)
|
||||
top_k_pages: int = Field(default=8, ge=1, le=100)
|
||||
max_sections_per_page: int = Field(default=2, ge=1, le=20)
|
||||
min_similarity: float = Field(default=0.35, ge=-1, le=1)
|
||||
keyword_boost_enabled: bool = False
|
||||
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
|
||||
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
|
||||
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
|
||||
database_batch_size: int = Field(default=50, ge=1, le=500)
|
||||
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("frontend_base_url")
|
||||
@classmethod
|
||||
def _validate_frontend_base_url(cls, value: str) -> str:
|
||||
normalized = normalize_service_url(value)
|
||||
if not normalized:
|
||||
raise ValueError("local Wiki index frontend_base_url is required")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _migrate_legacy_flags(cls, value):
|
||||
if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value:
|
||||
value = {**value, "auto_sync": value.get("sync_enabled")}
|
||||
return value
|
||||
|
||||
@property
|
||||
def sync_enabled(self) -> bool:
|
||||
"""Backward-compatible status name; automatic work is one explicit switch."""
|
||||
|
||||
return self.auto_sync
|
||||
|
||||
@property
|
||||
def internal_search_enabled(self) -> bool:
|
||||
return self.enabled
|
||||
|
||||
@property
|
||||
def reference_ui_enabled(self) -> bool:
|
||||
return self.enabled
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_index(self):
|
||||
if self.chunk_overlap_chars >= self.chunk_max_chars:
|
||||
raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
|
||||
if self.chunk_min_chars > self.chunk_max_chars:
|
||||
raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
|
||||
if self.top_k_pages > self.top_k_sections:
|
||||
raise ValueError("top_k_pages must not exceed top_k_sections")
|
||||
if self.engine != "numpy_exact":
|
||||
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
|
||||
if self.enabled and not self.embedding.enabled:
|
||||
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
|
||||
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
|
||||
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
|
||||
return self
|
||||
|
||||
|
||||
class LlmWikiConfig(BaseModel):
|
||||
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
|
||||
|
||||
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
|
||||
local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class LlmWikiRuntimeOverride(BaseModel):
|
||||
"""Administrator-written URL override stored in system settings.
|
||||
|
||||
``enabled=False`` means "use config.yaml". When enabled, an empty API URL
|
||||
deliberately forces legacy mode even if the source file has an address.
|
||||
Secrets are never stored here.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
api_base_url: str = ""
|
||||
web_base_url: str = ""
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("api_base_url", "web_base_url")
|
||||
@classmethod
|
||||
def _validate_url(cls, value: str) -> str:
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedLlmWikiRuntime:
|
||||
provider: str
|
||||
api_base_url: str
|
||||
web_base_url: str
|
||||
admin_email: str
|
||||
admin_password: str
|
||||
override_enabled: bool
|
||||
|
||||
@property
|
||||
def weknora_enabled(self) -> bool:
|
||||
return self.provider == "weknora"
|
||||
|
||||
|
||||
def resolve_llmwiki_runtime(
|
||||
config: LlmWikiConfig,
|
||||
override: LlmWikiRuntimeOverride | None = None,
|
||||
) -> ResolvedLlmWikiRuntime:
|
||||
"""Resolve the deterministic provider from file config plus URL override."""
|
||||
|
||||
use_override = bool(override and override.enabled)
|
||||
api_base_url = override.api_base_url if use_override and override else config.weknora.api_base_url
|
||||
web_base_url = override.web_base_url if use_override and override else config.weknora.web_base_url
|
||||
return ResolvedLlmWikiRuntime(
|
||||
provider="weknora" if api_base_url else "legacy",
|
||||
api_base_url=api_base_url,
|
||||
web_base_url=web_base_url,
|
||||
admin_email=config.weknora.admin_email,
|
||||
admin_password=config.weknora.admin_password,
|
||||
override_enabled=use_override,
|
||||
)
|
||||
Reference in new issue
Block a user