sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -26,6 +26,49 @@ PACKAGE_RECORD_TYPES = {
"graph",
}
_PACKAGE_SNIFF_BYTES = 256 * 1024
def looks_like_knowledge_package(content: bytes) -> bool:
"""Recognize a DeerFlow Wiki-vector JSONL package from a bounded prefix.
The same check is used at ordinary-document upload boundaries so an
exported package can never be handed to a document parser (and therefore
to a model) by mistake. Source metadata may be split across consecutive
``source`` records for compatibility with packages produced by older test
helpers and releases.
"""
source: dict[str, Any] = {}
text = content[:_PACKAGE_SNIFF_BYTES].decode("utf-8-sig", errors="ignore")
for raw_line in text.splitlines():
# In a multipart upload the file's BOM is not the request body's BOM,
# so strip it again at the beginning of each candidate line.
line = raw_line.strip().lstrip("\ufeff").strip()
if not line.startswith("{"):
continue
try:
record = json.loads(line)
except (TypeError, ValueError):
continue
if not isinstance(record, dict):
continue
if record.get("record_type") != "source":
# A real package starts with one or more source records. Do not
# scan arbitrary JSONL prose for a package-shaped example later.
return False
payload = record.get("payload")
if not isinstance(payload, dict):
continue
source.update(payload)
provider = source.get("provider") or source.get("type")
if (
source.get("package_version") == PACKAGE_VERSION
and provider in {"weknora", "assistant"}
):
return True
return False
def package_path(root: Path, job_id: str) -> Path:
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})