diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 31ba7ca9379..b91c404b2eb 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -10,11 +10,14 @@ _GET_SECRET_ARGS = r"""\(\s*['"]([^'"]+)['"]\s*(?:,\s*[^)]*|,\s*default_value=[^ ENV_KEY_PATTERNS: tuple[re.Pattern[str], ...] = ( re.compile(r"os\.getenv" + _GETENV_ARGS), - re.compile(r"litellm\.get_secret" + _GET_SECRET_ARGS), - re.compile(r"litellm\.get_secret_str" + _GET_SECRET_ARGS), - re.compile(r"(? frozenset[str]: def collect_env_keys(base_dir: str) -> frozenset[str]: """Return every documentable env var name read anywhere under ``base_dir``.""" - return frozenset(key for file_path in _python_files(base_dir) for key in extract_env_keys(_read_text(file_path))) + return frozenset( + key for file_path in _files_with_suffix(base_dir, (".py",)) for key in extract_env_keys(_read_text(file_path)) + ) -def _python_files(base_dir: str) -> Iterator[str]: +def _files_with_suffix(base_dir: str, suffixes: tuple[str, ...]) -> Iterator[str]: for root, dirs, files in os.walk(base_dir): # Skip dependency/venv directories - prevents picking up env vars from installed packages dirs[:] = [d for d in dirs if d not in SKIP_DIRS] - yield from (os.path.join(root, name) for name in files if name.endswith(".py")) + yield from (os.path.join(root, name) for name in files if name.endswith(suffixes)) def _read_text(file_path: str) -> str: @@ -88,42 +93,37 @@ def _read_text(file_path: str) -> str: def extract_documented_keys(docs_content: str) -> frozenset[str]: - """Return the key names listed in the 'environment variables - Reference' table.""" - section = re.search( - r"### environment variables - Reference(.*?)(?=\n###|\Z)", - docs_content, - re.DOTALL | re.MULTILINE, - ) - if section is None: - return frozenset() - # Match | KEY_NAME | description | - capture first column only + """Return every env-var-shaped name mentioned anywhere in a documentation page.""" + return frozenset(DOCUMENTED_KEY_PATTERN.findall(docs_content)) + + +def collect_documented_keys(docs_dir: str) -> frozenset[str]: + """Return every env-var-shaped name mentioned on any page under ``docs_dir``.""" return frozenset( - match.group(1).strip() - for match in (re.match(r"^\|\s*([A-Z_][A-Z0-9_]*)\s*\|", line) for line in section.group(1).split("\n")) - if match is not None + key + for file_path in _files_with_suffix(docs_dir, DOCS_SUFFIXES) + for key in extract_documented_keys(_read_text(file_path)) ) +def undocumented_env_keys(base_dir: str, docs_dir: str) -> frozenset[str]: + """Return the env vars read under ``base_dir`` that no page under ``docs_dir`` mentions.""" + return collect_env_keys(base_dir) - collect_documented_keys(docs_dir) + + def main() -> None: - env_keys = collect_env_keys(repo_base) - print(env_keys) - - docs_path = "./docs/my-website/docs/proxy/config_settings.md" # Path to the documentation - try: - documented_keys = extract_documented_keys(_read_text(docs_path)) - except Exception as e: - raise Exception(f"Error reading documentation: {e}, \n repo base - {os.listdir('./')}") - - print(f"documented_keys: {documented_keys}") - undocumented_keys = env_keys - documented_keys - - print("Keys expected in 'environment settings' (found in code):") - for key in sorted(env_keys): - print(key) + if not os.path.isdir(DOCS_BASE): + raise Exception(f"No documentation found at {DOCS_BASE}; check out BerriAI/litellm-docs into docs/my-website") + undocumented_keys = undocumented_env_keys(repo_base, DOCS_BASE) if undocumented_keys: - raise Exception(f"\nKeys not documented in 'environment settings - Reference': {sorted(undocumented_keys)}") - print(f"\nAll keys are documented in 'environment settings - Reference'. - {env_keys}") + raise Exception( + f"Environment variables read under {repo_base} but mentioned nowhere in the docs: " + f"{sorted(undocumented_keys)}" + f"\nDocument each one, either on the relevant provider page or as a row in the " + f"'environment variables - Reference' table in {REFERENCE_TABLE_PATH}" + ) + print(f"Every environment variable read under {repo_base} is documented") if __name__ == "__main__": diff --git a/tests/test_litellm/test_env_key_doc_gate.py b/tests/test_litellm/test_env_key_doc_gate.py index aabda09a441..efc615a6ea7 100644 --- a/tests/test_litellm/test_env_key_doc_gate.py +++ b/tests/test_litellm/test_env_key_doc_gate.py @@ -1,11 +1,12 @@ """Tests for the env-var extraction used by tests/documentation_tests/test_env_keys.py. That script is the CI gate that fails when a user-facing environment variable read -under litellm/ has no row in the docs reference table. It only sees a key if one of its +under litellm/ is mentioned nowhere on the docs site. It only sees a key if one of its patterns matches the call, so a call shape the patterns miss silently bypasses the gate. Each supported shape is asserted here, along with the shapes that must not be treated as env var reads, so narrowing a pattern makes a test fail instead of quietly reopening the -hole. +hole. The docs side is asserted too, since a key documented on a provider page rather +than in the central reference table still counts as documented. """ import importlib.util @@ -41,6 +42,36 @@ def test_litellm_prefixed_get_secret_bool_is_captured() -> None: assert gate.extract_env_keys('litellm.get_secret_bool("QSTASH_FLUSH_ON_BOOT")') == {"QSTASH_FLUSH_ON_BOOT"} +def test_bare_get_secret_is_captured() -> None: + assert gate.extract_env_keys('key = get_secret("QSTASH_ALPHA")') == {"QSTASH_ALPHA"} + + +def test_bare_get_secret_with_default_is_captured() -> None: + assert gate.extract_env_keys('key = get_secret("QSTASH_ALPHA", "fallback")') == {"QSTASH_ALPHA"} + + +def test_bare_get_secret_str_is_captured() -> None: + assert gate.extract_env_keys('key = get_secret_str("QSTASH_BRAVO")') == {"QSTASH_BRAVO"} + + +def test_bare_get_secret_str_with_keyword_default_is_captured() -> None: + assert gate.extract_env_keys('get_secret_str("QSTASH_BRAVO", default_value=None)') == {"QSTASH_BRAVO"} + + +def test_get_secret_reached_through_the_utils_module_is_captured() -> None: + assert gate.extract_env_keys('litellm.utils.get_secret("QSTASH_ALPHA")') == {"QSTASH_ALPHA"} + + +def test_get_secret_on_an_unrelated_utils_attribute_is_not_an_env_read() -> None: + source = "\n".join( + ( + 'vault.utils.get_secret("QSTASH_ALPHA")', + 'self.utils.get_secret_str("QSTASH_BRAVO")', + ) + ) + assert gate.extract_env_keys(source) == frozenset() + + def test_previously_supported_call_shapes_are_still_captured() -> None: source = "\n".join( ( @@ -83,21 +114,77 @@ def test_excluded_keys_are_filtered_for_every_call_shape() -> None: assert gate.extract_env_keys(source) == frozenset() -def test_documented_keys_are_read_from_the_reference_table_only() -> None: +def test_a_key_mentioned_outside_the_reference_table_counts_as_documented() -> None: docs = "\n".join( ( - "### general_settings - Reference", - "| BEFORE_THE_TABLE | not the env var table", + "# Qstash", "", - "### environment variables - Reference", + "Set `QSTASH_ALPHA` to your endpoint before calling the provider.", "", - "| Name | Description |", - "|------|-------------|", - "| QSTASH_ALPHA | first key", - "| QSTASH_BRAVO | second key", - "", - "### another section - Reference", - "| AFTER_THE_TABLE | also not the env var table", + "```bash", + 'export QSTASH_BRAVO="sk-..."', + "```", ) ) - assert gate.extract_documented_keys(docs) == {"QSTASH_ALPHA", "QSTASH_BRAVO"} + documented = gate.extract_documented_keys(docs) + assert "QSTASH_ALPHA" in documented + assert "QSTASH_BRAVO" in documented + + +def test_a_name_glued_to_surrounding_text_is_not_a_mention() -> None: + documented = gate.extract_documented_keys("the useQSTASH_ALPHA helper reads it") + assert "QSTASH_ALPHA" not in documented + + +def test_a_longer_name_does_not_document_the_key_it_ends_with() -> None: + documented = gate.extract_documented_keys("Set AZURE_QSTASH_ALPHA in your environment") + assert "AZURE_QSTASH_ALPHA" in documented + assert "QSTASH_ALPHA" not in documented + + +def test_lowercase_mentions_are_not_treated_as_env_var_names() -> None: + assert gate.extract_documented_keys("pass qstash_alpha as a config key") == frozenset() + + +def test_documented_keys_are_collected_from_every_page_of_the_docs_site(tmp_path: Path) -> None: + (tmp_path / "providers").mkdir() + (tmp_path / "providers" / "qstash.md").write_text("Set `QSTASH_ALPHA` to your endpoint.\n", encoding="utf-8") + (tmp_path / "providers" / "qstash_batches.mdx").write_text("| QSTASH_BRAVO | second key |\n", encoding="utf-8") + documented = gate.collect_documented_keys(str(tmp_path)) + assert "QSTASH_ALPHA" in documented + assert "QSTASH_BRAVO" in documented + assert "QSTASH_CHARLIE" not in documented + + +def _write_tree(tmp_path: Path, source: str, docs_page: str) -> tuple[str, str]: + source_dir = tmp_path / "litellm" + docs_dir = tmp_path / "docs" / "providers" + source_dir.mkdir() + docs_dir.mkdir(parents=True) + (source_dir / "qstash.py").write_text(source, encoding="utf-8") + (docs_dir / "qstash.md").write_text(docs_page, encoding="utf-8") + return str(source_dir), str(tmp_path / "docs") + + +def test_a_key_documented_only_on_a_provider_page_satisfies_the_gate(tmp_path: Path) -> None: + source_dir, docs_dir = _write_tree( + tmp_path, + 'api_key = get_secret_str("QSTASH_ALPHA")\n', + "# Qstash\n\nSet `QSTASH_ALPHA` to your API key.\n", + ) + assert gate.undocumented_env_keys(source_dir, docs_dir) == frozenset() + + +def test_a_key_documented_on_no_page_at_all_fails_the_gate(tmp_path: Path) -> None: + source_dir, docs_dir = _write_tree( + tmp_path, + 'api_key = get_secret_str("QSTASH_ALPHA")\n', + "# Qstash\n\nThis provider needs an API key.\n", + ) + assert gate.undocumented_env_keys(source_dir, docs_dir) == {"QSTASH_ALPHA"} + + +def test_only_documentation_pages_are_scanned_for_mentions(tmp_path: Path) -> None: + (tmp_path / "notes.txt").write_text("QSTASH_ALPHA\n", encoding="utf-8") + (tmp_path / "example.py").write_text('get_secret("QSTASH_BRAVO")\n', encoding="utf-8") + assert gate.collect_documented_keys(str(tmp_path)) == frozenset()