scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

fix(rag): tighten shebang and ASCII-magic content classification

Browse source 
Reject shebangs even when preceded by a UTF-8 BOM or leading whitespace,
and stop misclassifying UTF-8 text that happens to start with the
ASCII-printable magics BZh (bzip2) or dex\\n (Android DEX) as archives
or executables by applying the same UTF-8 carve-out already used for MZ
This commit is contained in:
Cursor Agent 2026-08-24 20:17:36 +00:00
parent ac29505f3d
commit 6afe0d73c8
No known key found for this signature in database
1 changed files with 16 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
litellm/proxy/rag_endpoints/upload_security.py
View file

@ -28,7 +28,6 @@ _ARCHIVE_MAGIC_PREFIXES: Final[tuple[bytes, ...]] = (
b"PK\x05\x06",
b"PK\x07\x08",
b"\x1f\x8b",
b"BZh",
b"\xfd7zXZ\x00",
b"7z\xbc\xaf\x27\x1c",
b"Rar!\x1a\x07\x00",
@ -37,6 +36,8 @@ _ARCHIVE_MAGIC_PREFIXES: Final[tuple[bytes, ...]] = (
b"\x28\xb5\x2f\xfd",
)
_ARCHIVE_MAGIC_PREFIXES_ASCII_AMBIGUOUS: Final[tuple[bytes, ...]] = (b"BZh",)
_EXECUTABLE_MAGIC_PREFIXES: Final[tuple[bytes, ...]] = (
b"\x7fELF",
b"\xca\xfe\xba\xbe",
@ -45,12 +46,15 @@ _EXECUTABLE_MAGIC_PREFIXES: Final[tuple[bytes, ...]] = (
b"\xce\xfa\xed\xfe",
b"\xcf\xfa\xed\xfe",
b"\x00asm",
b"dex\n",
)
_EXECUTABLE_MAGIC_PREFIXES_ASCII_AMBIGUOUS: Final[tuple[bytes, ...]] = (b"MZ", b"dex\n")
_TAR_USTAR_MAGIC: Final = b"ustar"
_TAR_USTAR_OFFSET: Final = 257
_UTF8_BOM: Final = b"\xef\xbb\xbf"
class DetectedFormat(str, Enum):
PDF = "pdf"
@ -158,7 +162,9 @@ def _is_archive(content: bytes) -> bool:
if _starts_with_any(content, _ARCHIVE_MAGIC_PREFIXES):
return True
tar_magic_end: Final = _TAR_USTAR_OFFSET + len(_TAR_USTAR_MAGIC)
return len(content) >= tar_magic_end and content[_TAR_USTAR_OFFSET:tar_magic_end] == _TAR_USTAR_MAGIC
if len(content) >= tar_magic_end and content[_TAR_USTAR_OFFSET:tar_magic_end] == _TAR_USTAR_MAGIC:
return True
return _starts_with_any(content, _ARCHIVE_MAGIC_PREFIXES_ASCII_AMBIGUOUS) and not _is_utf8_text(content)
def _is_utf8_text(content: bytes) -> bool:
@ -174,11 +180,16 @@ def _is_utf8_text(content: bytes) -> bool:
def _is_executable_binary(content: bytes) -> bool:
if _starts_with_any(content, _EXECUTABLE_MAGIC_PREFIXES):
return True
return content.startswith(b"MZ") and not _is_utf8_text(content)
return _starts_with_any(content, _EXECUTABLE_MAGIC_PREFIXES_ASCII_AMBIGUOUS) and not _is_utf8_text(content)
def _looks_like_shebang(content: bytes) -> bool:
body: Final = content.removeprefix(_UTF8_BOM).lstrip()
return body.startswith(b"#!")
def inspect_content(content: bytes) -> ContentInspection:
if content.startswith(b"#!"):
if _looks_like_shebang(content):
return DisallowedContent(DisallowedKind.EXECUTABLE)
if content.startswith(b"%PDF-"):
return AllowedContent(DetectedFormat.PDF)