diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 3bf2c88a848..31ba7ca9379 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -1,20 +1,19 @@ import os import re +from collections.abc import Iterator # Define the base directory for the litellm repository and documentation path repo_base = "./litellm" # Change this to your actual path -# Regular expressions to capture the keys used in os.getenv() and litellm.get_secret() -getenv_pattern = re.compile(r'os\.getenv\(\s*[\'"]([^\'"]+)[\'"]\s*(?:,\s*[^)]*)?\)') -get_secret_pattern = re.compile( - r'litellm\.get_secret\(\s*[\'"]([^\'"]+)[\'"]\s*(?:,\s*[^)]*|,\s*default_value=[^)]*)?\)' -) -get_secret_str_pattern = re.compile( - r'litellm\.get_secret_str\(\s*[\'"]([^\'"]+)[\'"]\s*(?:,\s*[^)]*|,\s*default_value=[^)]*)?\)' -) +_GETENV_ARGS = r"""\(\s*['"]([^'"]+)['"]\s*(?:,\s*[^)]*)?\)""" +_GET_SECRET_ARGS = r"""\(\s*['"]([^'"]+)['"]\s*(?:,\s*[^)]*|,\s*default_value=[^)]*)?\)""" -# Set to store unique keys from the code -env_keys = set() +ENV_KEY_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"os\.getenv" + _GETENV_ARGS), + re.compile(r"litellm\.get_secret" + _GET_SECRET_ARGS), + re.compile(r"litellm\.get_secret_str" + _GET_SECRET_ARGS), + re.compile(r"(? frozenset[str]: + """Return every documentable env var name read by the given Python source.""" + return frozenset( + match for pattern in ENV_KEY_PATTERNS for match in pattern.findall(source) if match not in EXCLUDED_KEYS ) -print(f"documented_keys: {documented_keys}") -# Compare and find undocumented keys -undocumented_keys = env_keys - documented_keys +def collect_env_keys(base_dir: str) -> frozenset[str]: + """Return every documentable env var name read anywhere under ``base_dir``.""" + return frozenset(key for file_path in _python_files(base_dir) for key in extract_env_keys(_read_text(file_path))) -# Print results -print("Keys expected in 'environment settings' (found in code):") -for key in sorted(env_keys): - print(key) -if undocumented_keys: - raise Exception( - f"\nKeys not documented in 'environment settings - Reference': {undocumented_keys}" +def _python_files(base_dir: str) -> Iterator[str]: + for root, dirs, files in os.walk(base_dir): + # Skip dependency/venv directories - prevents picking up env vars from installed packages + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + yield from (os.path.join(root, name) for name in files if name.endswith(".py")) + + +def _read_text(file_path: str) -> str: + with open(file_path, "r", encoding="utf-8") as f: + return f.read() + + +def extract_documented_keys(docs_content: str) -> frozenset[str]: + """Return the key names listed in the 'environment variables - Reference' table.""" + section = re.search( + r"### environment variables - Reference(.*?)(?=\n###|\Z)", + docs_content, + re.DOTALL | re.MULTILINE, ) -else: - print( - "\nAll keys are documented in 'environment settings - Reference'. - {}".format( - env_keys - ) + if section is None: + return frozenset() + # Match | KEY_NAME | description | - capture first column only + return frozenset( + match.group(1).strip() + for match in (re.match(r"^\|\s*([A-Z_][A-Z0-9_]*)\s*\|", line) for line in section.group(1).split("\n")) + if match is not None ) + + +def main() -> None: + env_keys = collect_env_keys(repo_base) + print(env_keys) + + docs_path = "./docs/my-website/docs/proxy/config_settings.md" # Path to the documentation + try: + documented_keys = extract_documented_keys(_read_text(docs_path)) + except Exception as e: + raise Exception(f"Error reading documentation: {e}, \n repo base - {os.listdir('./')}") + + print(f"documented_keys: {documented_keys}") + undocumented_keys = env_keys - documented_keys + + print("Keys expected in 'environment settings' (found in code):") + for key in sorted(env_keys): + print(key) + + if undocumented_keys: + raise Exception(f"\nKeys not documented in 'environment settings - Reference': {sorted(undocumented_keys)}") + print(f"\nAll keys are documented in 'environment settings - Reference'. - {env_keys}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/test_env_key_doc_gate.py b/tests/test_litellm/test_env_key_doc_gate.py new file mode 100644 index 00000000000..aabda09a441 --- /dev/null +++ b/tests/test_litellm/test_env_key_doc_gate.py @@ -0,0 +1,103 @@ +"""Tests for the env-var extraction used by tests/documentation_tests/test_env_keys.py. + +That script is the CI gate that fails when a user-facing environment variable read +under litellm/ has no row in the docs reference table. It only sees a key if one of its +patterns matches the call, so a call shape the patterns miss silently bypasses the gate. +Each supported shape is asserted here, along with the shapes that must not be treated as +env var reads, so narrowing a pattern makes a test fail instead of quietly reopening the +hole. +""" + +import importlib.util +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_MODULE_PATH = _REPO_ROOT / "tests" / "documentation_tests" / "test_env_keys.py" +_spec = importlib.util.spec_from_file_location("documentation_test_env_keys", _MODULE_PATH) +assert _spec is not None and _spec.loader is not None +gate = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = gate +_spec.loader.exec_module(gate) + + +def test_bare_get_secret_bool_is_captured() -> None: + assert gate.extract_env_keys('flag = get_secret_bool("QSTASH_FLUSH_ON_BOOT")') == {"QSTASH_FLUSH_ON_BOOT"} + + +def test_get_secret_bool_with_default_is_captured() -> None: + assert gate.extract_env_keys('if get_secret_bool("QSTASH_FLUSH_ON_BOOT", False) is not True:') == { + "QSTASH_FLUSH_ON_BOOT" + } + + +def test_get_secret_bool_with_keyword_default_is_captured() -> None: + assert gate.extract_env_keys('get_secret_bool("QSTASH_FLUSH_ON_BOOT", default_value=False)') == { + "QSTASH_FLUSH_ON_BOOT" + } + + +def test_litellm_prefixed_get_secret_bool_is_captured() -> None: + assert gate.extract_env_keys('litellm.get_secret_bool("QSTASH_FLUSH_ON_BOOT")') == {"QSTASH_FLUSH_ON_BOOT"} + + +def test_previously_supported_call_shapes_are_still_captured() -> None: + source = "\n".join( + ( + 'os.getenv("QSTASH_ALPHA")', + 'os.getenv("QSTASH_BRAVO", "fallback")', + 'litellm.get_secret("QSTASH_CHARLIE")', + 'litellm.get_secret_str("QSTASH_DELTA", default_value=None)', + ) + ) + assert gate.extract_env_keys(source) == {"QSTASH_ALPHA", "QSTASH_BRAVO", "QSTASH_CHARLIE", "QSTASH_DELTA"} + + +def test_get_secret_calls_on_unrelated_objects_are_not_env_reads() -> None: + source = "\n".join( + ( + 'vault_client.get_secret("QSTASH_ALPHA")', + 'self.get_secret_str("QSTASH_BRAVO")', + 'provider.get_secret_bool("QSTASH_CHARLIE")', + ) + ) + assert gate.extract_env_keys(source) == frozenset() + + +def test_similarly_named_helpers_are_not_env_reads() -> None: + assert gate.extract_env_keys('get_secret_bundle("QSTASH_ALPHA")') == frozenset() + + +def test_non_literal_arguments_are_not_env_reads() -> None: + assert gate.extract_env_keys("get_secret_bool(flag_name)") == frozenset() + + +def test_excluded_keys_are_filtered_for_every_call_shape() -> None: + source = "\n".join( + ( + 'os.getenv("TERM_PROGRAM")', + 'get_secret_bool("LITELLM_RUST")', + 'litellm.get_secret_str("MAVVRIK_FOCUS_FREQUENCY")', + ) + ) + assert gate.extract_env_keys(source) == frozenset() + + +def test_documented_keys_are_read_from_the_reference_table_only() -> None: + docs = "\n".join( + ( + "### general_settings - Reference", + "| BEFORE_THE_TABLE | not the env var table", + "", + "### environment variables - Reference", + "", + "| Name | Description |", + "|------|-------------|", + "| QSTASH_ALPHA | first key", + "| QSTASH_BRAVO | second key", + "", + "### another section - Reference", + "| AFTER_THE_TABLE | also not the env var table", + ) + ) + assert gate.extract_documented_keys(docs) == {"QSTASH_ALPHA", "QSTASH_BRAVO"}