sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -1,9 +1,10 @@
import fnmatch
import os
import re
import os
import re
import time
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
IGNORE_PATTERNS = [
".git",
".svn",
@@ -56,8 +57,10 @@ IGNORE_PATTERNS = [
".ruff_cache",
]
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
DEFAULT_LINE_SUMMARY_LENGTH = 200
DEFAULT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024
DEFAULT_MAX_TOTAL_SCAN_BYTES = 64 * 1024 * 1024
DEFAULT_MAX_SCAN_SECONDS = 5.0
DEFAULT_LINE_SUMMARY_LENGTH = 200
@dataclass(frozen=True)
@@ -148,7 +151,9 @@ def find_grep_matches(
literal: bool = False,
case_sensitive: bool = False,
max_results: int = 100,
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
max_file_size: int | None = None,
max_total_scan_bytes: int = DEFAULT_MAX_TOTAL_SCAN_BYTES,
max_scan_seconds: float = DEFAULT_MAX_SCAN_SECONDS,
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
) -> tuple[list[GrepMatch], bool]:
matches: list[GrepMatch] = []
@@ -162,16 +167,31 @@ def find_grep_matches(
regex_source = re.escape(pattern) if literal else pattern
flags = 0 if case_sensitive else re.IGNORECASE
regex = re.compile(regex_source, flags)
# Skip lines longer than this to prevent ReDoS on minified / no-newline files.
_max_line_chars = line_summary_length * 10
regex = re.compile(regex_source, flags)
if max_file_size is None:
try:
from deerflow.uploads.policy import get_runtime_upload_policy
groups = (get_runtime_upload_policy().get("groups") or {}).values()
configured = max((int(group.get("max_bytes") or 0) for group in groups), default=DEFAULT_MAX_FILE_SIZE_BYTES)
max_file_size = min(200 * 1024 * 1024, max(DEFAULT_MAX_FILE_SIZE_BYTES, configured))
except Exception:
max_file_size = DEFAULT_MAX_FILE_SIZE_BYTES
# Read lines in bounded fragments to prevent a minified / no-newline file
# from allocating its entire body before the length check can run.
_max_line_chars = line_summary_length * 10
deadline = time.monotonic() + max(0.1, max_scan_seconds)
scanned_bytes = 0
for current_root, dirs, files in os.walk(root):
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
rel_dir = Path(current_root).relative_to(root)
for name in files:
for name in files:
if time.monotonic() >= deadline:
return matches, True
if should_ignore_name(name):
continue
@@ -187,13 +207,32 @@ def find_grep_matches(
file_path = candidate_path.resolve()
if not file_path.is_relative_to(root):
continue
if file_path.stat().st_size > max_file_size or is_binary_file(file_path):
continue
with file_path.open(encoding="utf-8", errors="replace") as handle:
for line_number, line in enumerate(handle, start=1):
if len(line) > _max_line_chars:
continue
if regex.search(line):
file_size = file_path.stat().st_size
if file_size > max_file_size:
truncated = True
continue
if scanned_bytes + file_size > max_total_scan_bytes:
return matches, True
scanned_bytes += file_size
if is_binary_file(file_path):
continue
with file_path.open(encoding="utf-8", errors="replace") as handle:
line_number = 0
while True:
if time.monotonic() >= deadline:
return matches, True
line = handle.readline(_max_line_chars + 1)
if not line:
break
line_number += 1
if len(line) > _max_line_chars and not line.endswith(("\n", "\r")):
# Drain the rest of this physical line in bounded
# fragments, but never run an attacker-controlled
# regex over it.
while line and not line.endswith(("\n", "\r")):
line = handle.readline(_max_line_chars + 1)
continue
if regex.search(line):
matches.append(
GrepMatch(
path=str(file_path),