feat(knowledge): 阶段二 向量 RAG 与召回注入
- embeddings.py:OpenAI 兼容 embedding 客户端,未配置时关键词兜底
- chunking.py + knowledge_embeddings 表:Markdown 分块与向量索引(Python 余弦)
- search.py:keyword/vector/hybrid 三种模式,向量不可用自动降级
- 笔记创建/更新自动重建向量索引;新增 /reindex 维护接口
- KnowledgeRagMiddleware:按对话级开关 knowledge_rag_enabled 注入 <knowledge-context>,接入 lead agent 中间件链(默认随 config.knowledge.rag_enabled)
- service.build_rag_context(wiki-query 风格分层召回)
- 笔记版本快照(knowledge_note_versions)+ GET /notes/{id}/versions
- 前端:聊天页「知识库」对话级 RAG 开关 + 类型字段
- 新增 entities/relations 表(阶段四预置)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8463289f75
commit
683cc6d76e
14 files changed
+738
-40
No files matched your search
@@ -0,0 +1,60 @@
|
||||
"""Markdown chunking for embedding/index (phase 2).
|
||||
|
||||
Splits a note body into overlapping chunks along Markdown structure (headings →
|
||||
paragraphs) so each embedded chunk is semantically coherent and bounded in
|
||||
size. Frontmatter is assumed already stripped by the caller.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
||||
|
||||
|
||||
def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
|
||||
"""Return a list of text chunks for ``content``.
|
||||
|
||||
Sections are cut at Markdown headings; long sections are further split into
|
||||
~``max_chars`` windows with ``overlap`` characters of carry-over.
|
||||
"""
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Split into heading-delimited sections, keeping the heading with its body.
|
||||
sections: list[str] = []
|
||||
last = 0
|
||||
for m in _HEADING_RE.finditer(text):
|
||||
if m.start() > last:
|
||||
sections.append(text[last : m.start()].strip())
|
||||
last = m.start()
|
||||
sections.append(text[last:].strip())
|
||||
sections = [s for s in sections if s]
|
||||
|
||||
chunks: list[str] = []
|
||||
for section in sections:
|
||||
if len(section) <= max_chars:
|
||||
chunks.append(section)
|
||||
continue
|
||||
# Pack paragraphs into windows, then hard-split anything still too long.
|
||||
buf = ""
|
||||
for para in _split_paragraphs(section):
|
||||
if len(buf) + len(para) + 2 <= max_chars:
|
||||
buf = f"{buf}\n\n{para}".strip()
|
||||
else:
|
||||
if buf:
|
||||
chunks.append(buf)
|
||||
if len(para) <= max_chars:
|
||||
buf = para
|
||||
else:
|
||||
for i in range(0, len(para), max_chars - overlap):
|
||||
chunks.append(para[i : i + max_chars])
|
||||
buf = ""
|
||||
if buf:
|
||||
chunks.append(buf)
|
||||
return [c for c in chunks if c.strip()]
|
||||
Reference in new issue
Block a user