mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-19 00:01:29 +00:00
ci(aws-partition): count allowlisted literal occurrences so duplicates in allowed files fail
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
302394edff
commit
6a9ae2bba2
1 changed files with 34 additions and 15 deletions
|
|
@ -13,11 +13,12 @@ literal parts of f-strings and the strings inside `.format()` calls and
|
|||
concatenations. Docstrings and comments are not, since they never reach a request.
|
||||
`amazonaws.com.cn` passes because it is already the China partition.
|
||||
|
||||
`ALLOWED` holds the (file, token) pairs that are text rather than a request target:
|
||||
a hosted logo, an IAM service principal, and hostnames quoted as examples inside
|
||||
error messages and field descriptions. An entry only covers that exact token in that
|
||||
exact file, so a second literal in an allowed file is still caught, and an entry
|
||||
whose token is gone fails the check so the set only shrinks.
|
||||
`ALLOWED` holds the (file, token, count) triples that are text rather than a request
|
||||
target: a hosted logo, an IAM service principal, and hostnames quoted as examples
|
||||
inside error messages and field descriptions. An entry only covers that many
|
||||
occurrences of that exact token in that exact file, so a second copy of an allowed
|
||||
literal is still caught, and an entry whose token is gone or whose count has changed
|
||||
fails the check so the set only shrinks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -25,7 +26,9 @@ from __future__ import annotations
|
|||
import ast
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NamedTuple
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
|
|
@ -38,25 +41,29 @@ COMMERCIAL_TOKEN: Final = re.compile(r"[A-Za-z0-9.-]*amazonaws\.com(?!\.cn)|arn:
|
|||
class Allowance(NamedTuple):
|
||||
file: str
|
||||
token: str
|
||||
occurrences: int
|
||||
|
||||
|
||||
ALLOWED: Final = frozenset(
|
||||
{
|
||||
Allowance("litellm/integrations/email_alerting.py", "litellm-listing.s3.amazonaws.com"),
|
||||
Allowance("litellm/types/integrations/slack_alerting.py", "litellm-listing.s3.amazonaws.com"),
|
||||
Allowance("litellm/rag/ingestion/bedrock_ingestion.py", "bedrock.amazonaws.com"),
|
||||
Allowance("litellm/integrations/email_alerting.py", "litellm-listing.s3.amazonaws.com", 1),
|
||||
Allowance("litellm/types/integrations/slack_alerting.py", "litellm-listing.s3.amazonaws.com", 1),
|
||||
Allowance("litellm/rag/ingestion/bedrock_ingestion.py", "bedrock.amazonaws.com", 1),
|
||||
Allowance(
|
||||
"litellm/llms/bedrock/chat/agentcore/transformation.py",
|
||||
"arn:aws:bedrock-agentcore:region:account:runtime/runtime_id",
|
||||
1,
|
||||
),
|
||||
Allowance("litellm/llms/bedrock/search/transformation.py", ".amazonaws.com"),
|
||||
Allowance("litellm/llms/bedrock/search/transformation.py", ".amazonaws.com", 1),
|
||||
Allowance(
|
||||
"litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py",
|
||||
"bucket.s3.amazonaws.com",
|
||||
1,
|
||||
),
|
||||
Allowance("litellm/types/proxy/claude_code_endpoints.py", "bucket.s3.amazonaws.com"),
|
||||
Allowance("litellm/types/proxy/claude_code_endpoints.py", "bucket.s3.amazonaws.com", 1),
|
||||
}
|
||||
)
|
||||
ALLOWED_COUNTS: Final = MappingProxyType({(entry.file, entry.token): entry.occurrences for entry in ALLOWED})
|
||||
|
||||
|
||||
class Hit(NamedTuple):
|
||||
|
|
@ -98,14 +105,26 @@ def find_hits(scan_root: Path) -> tuple[Hit, ...]:
|
|||
)
|
||||
|
||||
|
||||
def _violation_message(hit: Hit, found: int) -> str:
|
||||
allowed: Final = ALLOWED_COUNTS.get((hit.file, hit.token))
|
||||
if allowed is None:
|
||||
return f"{hit.file}:{hit.line}: hardcoded commercial AWS partition literal {hit.token!r}"
|
||||
return (
|
||||
f"{hit.file}:{hit.line}: {hit.token!r} appears {found} times but ALLOWED covers {allowed}; "
|
||||
"build it from the region helper or update the count"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
hits: Final = find_hits(SCAN_ROOT)
|
||||
seen: Final = frozenset(Allowance(hit.file, hit.token) for hit in hits)
|
||||
violations: Final = tuple(hit for hit in hits if Allowance(hit.file, hit.token) not in ALLOWED)
|
||||
stale: Final = ALLOWED - seen
|
||||
counts: Final = MappingProxyType(Counter((hit.file, hit.token) for hit in hits))
|
||||
violations: Final = tuple(
|
||||
sorted(hit for hit in hits if Allowance(hit.file, hit.token, counts[hit.file, hit.token]) not in ALLOWED)
|
||||
)
|
||||
stale: Final = tuple(entry for entry in sorted(ALLOWED) if (entry.file, entry.token) not in counts)
|
||||
for hit in violations:
|
||||
print(f"{hit.file}:{hit.line}: hardcoded commercial AWS partition literal {hit.token!r}")
|
||||
for allowance in sorted(stale):
|
||||
print(_violation_message(hit, counts[hit.file, hit.token]))
|
||||
for allowance in stale:
|
||||
print(f"{allowance.file}: ALLOWED entry {allowance.token!r} no longer matches anything, remove it")
|
||||
if violations or stale:
|
||||
print(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue