sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -1,9 +1,10 @@
|
||||
import fnmatch
|
||||
import os
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
".git",
|
||||
".svn",
|
||||
@@ -56,8 +57,10 @@ IGNORE_PATTERNS = [
|
||||
".ruff_cache",
|
||||
]
|
||||
|
||||
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
|
||||
DEFAULT_LINE_SUMMARY_LENGTH = 200
|
||||
DEFAULT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024
|
||||
DEFAULT_MAX_TOTAL_SCAN_BYTES = 64 * 1024 * 1024
|
||||
DEFAULT_MAX_SCAN_SECONDS = 5.0
|
||||
DEFAULT_LINE_SUMMARY_LENGTH = 200
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -148,7 +151,9 @@ def find_grep_matches(
|
||||
literal: bool = False,
|
||||
case_sensitive: bool = False,
|
||||
max_results: int = 100,
|
||||
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
|
||||
max_file_size: int | None = None,
|
||||
max_total_scan_bytes: int = DEFAULT_MAX_TOTAL_SCAN_BYTES,
|
||||
max_scan_seconds: float = DEFAULT_MAX_SCAN_SECONDS,
|
||||
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
|
||||
) -> tuple[list[GrepMatch], bool]:
|
||||
matches: list[GrepMatch] = []
|
||||
@@ -162,16 +167,31 @@ def find_grep_matches(
|
||||
|
||||
regex_source = re.escape(pattern) if literal else pattern
|
||||
flags = 0 if case_sensitive else re.IGNORECASE
|
||||
regex = re.compile(regex_source, flags)
|
||||
|
||||
# Skip lines longer than this to prevent ReDoS on minified / no-newline files.
|
||||
_max_line_chars = line_summary_length * 10
|
||||
regex = re.compile(regex_source, flags)
|
||||
|
||||
if max_file_size is None:
|
||||
try:
|
||||
from deerflow.uploads.policy import get_runtime_upload_policy
|
||||
|
||||
groups = (get_runtime_upload_policy().get("groups") or {}).values()
|
||||
configured = max((int(group.get("max_bytes") or 0) for group in groups), default=DEFAULT_MAX_FILE_SIZE_BYTES)
|
||||
max_file_size = min(200 * 1024 * 1024, max(DEFAULT_MAX_FILE_SIZE_BYTES, configured))
|
||||
except Exception:
|
||||
max_file_size = DEFAULT_MAX_FILE_SIZE_BYTES
|
||||
|
||||
# Read lines in bounded fragments to prevent a minified / no-newline file
|
||||
# from allocating its entire body before the length check can run.
|
||||
_max_line_chars = line_summary_length * 10
|
||||
deadline = time.monotonic() + max(0.1, max_scan_seconds)
|
||||
scanned_bytes = 0
|
||||
|
||||
for current_root, dirs, files in os.walk(root):
|
||||
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
|
||||
rel_dir = Path(current_root).relative_to(root)
|
||||
|
||||
for name in files:
|
||||
for name in files:
|
||||
if time.monotonic() >= deadline:
|
||||
return matches, True
|
||||
if should_ignore_name(name):
|
||||
continue
|
||||
|
||||
@@ -187,13 +207,32 @@ def find_grep_matches(
|
||||
file_path = candidate_path.resolve()
|
||||
if not file_path.is_relative_to(root):
|
||||
continue
|
||||
if file_path.stat().st_size > max_file_size or is_binary_file(file_path):
|
||||
continue
|
||||
with file_path.open(encoding="utf-8", errors="replace") as handle:
|
||||
for line_number, line in enumerate(handle, start=1):
|
||||
if len(line) > _max_line_chars:
|
||||
continue
|
||||
if regex.search(line):
|
||||
file_size = file_path.stat().st_size
|
||||
if file_size > max_file_size:
|
||||
truncated = True
|
||||
continue
|
||||
if scanned_bytes + file_size > max_total_scan_bytes:
|
||||
return matches, True
|
||||
scanned_bytes += file_size
|
||||
if is_binary_file(file_path):
|
||||
continue
|
||||
with file_path.open(encoding="utf-8", errors="replace") as handle:
|
||||
line_number = 0
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
return matches, True
|
||||
line = handle.readline(_max_line_chars + 1)
|
||||
if not line:
|
||||
break
|
||||
line_number += 1
|
||||
if len(line) > _max_line_chars and not line.endswith(("\n", "\r")):
|
||||
# Drain the rest of this physical line in bounded
|
||||
# fragments, but never run an attacker-controlled
|
||||
# regex over it.
|
||||
while line and not line.endswith(("\n", "\r")):
|
||||
line = handle.readline(_max_line_chars + 1)
|
||||
continue
|
||||
if regex.search(line):
|
||||
matches.append(
|
||||
GrepMatch(
|
||||
path=str(file_path),
|
||||
|
||||
Reference in new issue
Block a user