feat: add local LLMWiki wiki vector index

offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 from urllib.parse import urlsplit, urlunsplit
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 
 def normalize_service_url(value: str | None) -> str:
@@ -37,18 +37,12 @@ class WeKnoraConnectionConfig(BaseModel):
     web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
     admin_email: str = Field(
         default="",
-        description=(
-            "Optional WeKnora administrator email used by DeerFlow's server-side "
-            "WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
-        ),
+        description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
     )
     admin_password: str = Field(
         default="",
         repr=False,
-        description=(
-            "Optional WeKnora administrator password used by DeerFlow's server-side "
-            "WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
-        ),
+        description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
     )
     model_config = ConfigDict(extra="ignore")
 
@@ -58,10 +52,107 @@ class WeKnoraConnectionConfig(BaseModel):
         return normalize_service_url(value)
 
 
+class LocalWikiEmbeddingConfig(BaseModel):
+    enabled: bool = True
+    provider: str = "openai"
+    model: str = "bge-m3"
+    base_url: str = ""
+    api_key: str = Field(default="", repr=False)
+    dimensions: int | None = Field(default=1024, ge=1, le=65536)
+    batch_size: int = Field(default=32, ge=1, le=512)
+    timeout_seconds: float = Field(default=30, gt=0, le=300)
+    max_retries: int = Field(default=3, ge=0, le=10)
+    model_config = ConfigDict(extra="ignore")
+
+    @field_validator("provider")
+    @classmethod
+    def _validate_provider(cls, value: str) -> str:
+        normalized = value.strip().lower()
+        if normalized not in {"openai", "openai_compatible"}:
+            raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
+        return normalized
+
+    @field_validator("base_url")
+    @classmethod
+    def _validate_url(cls, value: str) -> str:
+        return normalize_service_url(value)
+
+
+class LocalWikiExternalApiConfig(BaseModel):
+    enabled: bool = False
+    api_key: str = Field(default="", repr=False)
+    previous_api_key: str = Field(default="", repr=False)
+    max_query_chars: int = Field(default=1000, ge=1, le=10000)
+    max_knowledge_bases: int = Field(default=20, ge=1, le=100)
+    max_results: int = Field(default=50, ge=1, le=100)
+    default_results: int = Field(default=10, ge=1, le=100)
+    include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
+    requests_per_minute: int = Field(default=60, ge=1, le=100000)
+    model_config = ConfigDict(extra="ignore")
+
+    @model_validator(mode="after")
+    def _validate_enabled_key(self):
+        if self.enabled and not self.api_key.strip():
+            raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
+        if self.default_results > self.max_results:
+            raise ValueError("external_api.default_results must not exceed max_results")
+        return self
+
+
+class LocalWikiIndexConfig(BaseModel):
+    """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
+
+    enabled: bool = False
+    sync_enabled: bool = False
+    internal_search_enabled: bool = False
+    reference_ui_enabled: bool = False
+    sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
+    sync_page_size: int = Field(default=100, ge=1, le=500)
+    sync_concurrency: int = Field(default=3, ge=1, le=32)
+    sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
+    sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
+    sync_max_retries: int = Field(default=3, ge=0, le=10)
+    sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
+    chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
+    chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
+    chunk_min_chars: int = Field(default=80, ge=1, le=10000)
+    title_prefix_enabled: bool = True
+    embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
+    engine: str = "numpy_exact"
+    strict_vector: bool = True
+    top_k_sections: int = Field(default=30, ge=1, le=1000)
+    top_k_pages: int = Field(default=8, ge=1, le=100)
+    max_sections_per_page: int = Field(default=2, ge=1, le=20)
+    min_similarity: float = Field(default=0.35, ge=-1, le=1)
+    keyword_boost_enabled: bool = False
+    keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
+    cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
+    cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
+    external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
+    model_config = ConfigDict(extra="ignore")
+
+    @model_validator(mode="after")
+    def _validate_index(self):
+        if self.chunk_overlap_chars >= self.chunk_max_chars:
+            raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
+        if self.chunk_min_chars > self.chunk_max_chars:
+            raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
+        if self.top_k_pages > self.top_k_sections:
+            raise ValueError("top_k_pages must not exceed top_k_sections")
+        if self.engine != "numpy_exact":
+            raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
+        if self.enabled and not self.embedding.enabled:
+            raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
+        if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
+            raise ValueError("local Wiki index embedding base_url and model are required when enabled")
+        return self
+
+
 class LlmWikiConfig(BaseModel):
     """Top-level ``config.yaml`` section for conditional WeKnora mode."""
 
     weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
+    local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
     model_config = ConfigDict(extra="ignore")