AdministratorandClaude Opus 4.8
683cc6d76e
- embeddings.py：OpenAI 兼容 embedding 客户端，未配置时关键词兜底
- chunking.py + knowledge_embeddings 表：Markdown 分块与向量索引（Python 余弦）
- search.py：keyword/vector/hybrid 三种模式，向量不可用自动降级
- 笔记创建/更新自动重建向量索引；新增 /reindex 维护接口
- KnowledgeRagMiddleware：按对话级开关 knowledge_rag_enabled 注入 <knowledge-context>，接入 lead agent 中间件链（默认随 config.knowledge.rag_enabled）
- service.build_rag_context（wiki-query 风格分层召回）
- 笔记版本快照（knowledge_note_versions）+ GET /notes/{id}/versions
- 前端：聊天页「知识库」对话级 RAG 开关 + 类型字段
- 新增 entities/relations 表（阶段四预置）
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
80 lines
3.1 KiB
Python
80 lines
3.1 KiB
Python
"""Embedding generation for vector RAG (phase 2).
|
|
|
|
Talks to an OpenAI-compatible ``/embeddings`` endpoint configured under
|
|
``knowledge.embedding``. Everything here is **best-effort**: if embeddings are
|
|
disabled, unconfigured, or the endpoint errors, callers get an empty result and
|
|
the search layer falls back to keyword scoring — so vector RAG never breaks a
|
|
deployment that has no embedding model.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import math
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from deerflow.config.knowledge_config import KnowledgeEmbeddingConfig
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def cosine_similarity(a: list[float], b: list[float]) -> float:
|
|
"""Cosine similarity of two equal-length vectors (0 when degenerate)."""
|
|
if not a or not b or len(a) != len(b):
|
|
return 0.0
|
|
dot = sum(x * y for x, y in zip(a, b, strict=False))
|
|
na = math.sqrt(sum(x * x for x in a))
|
|
nb = math.sqrt(sum(y * y for y in b))
|
|
if na == 0 or nb == 0:
|
|
return 0.0
|
|
return dot / (na * nb)
|
|
|
|
|
|
class EmbeddingClient:
|
|
"""Thin async client over an OpenAI-compatible embeddings endpoint."""
|
|
|
|
def __init__(self, config: KnowledgeEmbeddingConfig) -> None:
|
|
self.config = config
|
|
|
|
def is_enabled(self) -> bool:
|
|
return bool(self.config.enabled and self.config.base_url and self.config.model)
|
|
|
|
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
|
"""Return one vector per input text; ``[]`` on any failure/disabled."""
|
|
if not self.is_enabled() or not texts:
|
|
return []
|
|
try:
|
|
import httpx
|
|
except Exception: # pragma: no cover - httpx is a core dep
|
|
logger.warning("httpx unavailable; knowledge embeddings disabled")
|
|
return []
|
|
|
|
url = self.config.base_url.rstrip("/") + "/embeddings"
|
|
headers = {"Content-Type": "application/json"}
|
|
if self.config.api_key:
|
|
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
|
|
|
vectors: list[list[float]] = []
|
|
batch = max(1, self.config.batch_size)
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
|
|
for start in range(0, len(texts), batch):
|
|
chunk = texts[start : start + batch]
|
|
payload: dict = {"model": self.config.model, "input": chunk}
|
|
if self.config.dimensions:
|
|
payload["dimensions"] = self.config.dimensions
|
|
resp = await client.post(url, headers=headers, json=payload)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
items = sorted(data.get("data", []), key=lambda d: d.get("index", 0))
|
|
vectors.extend([item.get("embedding", []) for item in items])
|
|
except Exception:
|
|
logger.exception("Knowledge embedding request failed; falling back to keyword search")
|
|
return []
|
|
return vectors
|
|
|
|
async def embed_query(self, text: str) -> list[float]:
|
|
vecs = await self.embed_texts([text])
|
|
return vecs[0] if vecs else []
|