sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+43
@@ -26,6 +26,49 @@ PACKAGE_RECORD_TYPES = {
|
||||
"graph",
|
||||
}
|
||||
|
||||
_PACKAGE_SNIFF_BYTES = 256 * 1024
|
||||
|
||||
|
||||
def looks_like_knowledge_package(content: bytes) -> bool:
|
||||
"""Recognize a DeerFlow Wiki-vector JSONL package from a bounded prefix.
|
||||
|
||||
The same check is used at ordinary-document upload boundaries so an
|
||||
exported package can never be handed to a document parser (and therefore
|
||||
to a model) by mistake. Source metadata may be split across consecutive
|
||||
``source`` records for compatibility with packages produced by older test
|
||||
helpers and releases.
|
||||
"""
|
||||
|
||||
source: dict[str, Any] = {}
|
||||
text = content[:_PACKAGE_SNIFF_BYTES].decode("utf-8-sig", errors="ignore")
|
||||
for raw_line in text.splitlines():
|
||||
# In a multipart upload the file's BOM is not the request body's BOM,
|
||||
# so strip it again at the beginning of each candidate line.
|
||||
line = raw_line.strip().lstrip("\ufeff").strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
if record.get("record_type") != "source":
|
||||
# A real package starts with one or more source records. Do not
|
||||
# scan arbitrary JSONL prose for a package-shaped example later.
|
||||
return False
|
||||
payload = record.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
source.update(payload)
|
||||
provider = source.get("provider") or source.get("type")
|
||||
if (
|
||||
source.get("package_version") == PACKAGE_VERSION
|
||||
and provider in {"weknora", "assistant"}
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def package_path(root: Path, job_id: str) -> Path:
|
||||
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
|
||||
|
||||
Reference in new issue
Block a user