AdministratorandClaude Opus 4.8
683cc6d76e
- embeddings.py：OpenAI 兼容 embedding 客户端，未配置时关键词兜底
- chunking.py + knowledge_embeddings 表：Markdown 分块与向量索引（Python 余弦）
- search.py：keyword/vector/hybrid 三种模式，向量不可用自动降级
- 笔记创建/更新自动重建向量索引；新增 /reindex 维护接口
- KnowledgeRagMiddleware：按对话级开关 knowledge_rag_enabled 注入 <knowledge-context>，接入 lead agent 中间件链（默认随 config.knowledge.rag_enabled）
- service.build_rag_context（wiki-query 风格分层召回）
- 笔记版本快照（knowledge_note_versions）+ GET /notes/{id}/versions
- 前端：聊天页「知识库」对话级 RAG 开关 + 类型字段
- 新增 entities/relations 表（阶段四预置）
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
61 lines
2.0 KiB
Python
61 lines
2.0 KiB
Python
"""Markdown chunking for embedding/index (phase 2).
|
|
|
|
Splits a note body into overlapping chunks along Markdown structure (headings →
|
|
paragraphs) so each embedded chunk is semantically coherent and bounded in
|
|
size. Frontmatter is assumed already stripped by the caller.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
_HEADING_RE = re.compile(r"^#{1,6}\s+", re.MULTILINE)
|
|
|
|
|
|
def _split_paragraphs(text: str) -> list[str]:
|
|
return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
|
|
|
|
|
def chunk_markdown(content: str, *, max_chars: int = 800, overlap: int = 120) -> list[str]:
|
|
"""Return a list of text chunks for ``content``.
|
|
|
|
Sections are cut at Markdown headings; long sections are further split into
|
|
~``max_chars`` windows with ``overlap`` characters of carry-over.
|
|
"""
|
|
text = (content or "").strip()
|
|
if not text:
|
|
return []
|
|
|
|
# Split into heading-delimited sections, keeping the heading with its body.
|
|
sections: list[str] = []
|
|
last = 0
|
|
for m in _HEADING_RE.finditer(text):
|
|
if m.start() > last:
|
|
sections.append(text[last : m.start()].strip())
|
|
last = m.start()
|
|
sections.append(text[last:].strip())
|
|
sections = [s for s in sections if s]
|
|
|
|
chunks: list[str] = []
|
|
for section in sections:
|
|
if len(section) <= max_chars:
|
|
chunks.append(section)
|
|
continue
|
|
# Pack paragraphs into windows, then hard-split anything still too long.
|
|
buf = ""
|
|
for para in _split_paragraphs(section):
|
|
if len(buf) + len(para) + 2 <= max_chars:
|
|
buf = f"{buf}\n\n{para}".strip()
|
|
else:
|
|
if buf:
|
|
chunks.append(buf)
|
|
if len(para) <= max_chars:
|
|
buf = para
|
|
else:
|
|
for i in range(0, len(para), max_chars - overlap):
|
|
chunks.append(para[i : i + max_chars])
|
|
buf = ""
|
|
if buf:
|
|
chunks.append(buf)
|
|
return [c for c in chunks if c.strip()]
|