fix(ci): make the env-key doc gate see bare get_secret and get_secret_str reads (#35996)

The gate required a litellm. prefix on get_secret and get_secret_str, so any
module importing either function directly bypassed it: 335 environment variables
read under litellm/ were invisible to it. The three patterns collapse into one
with the prefix optional, a negative lookbehind so attribute calls on unrelated
objects cannot match, and litellm.utils. accepted since four call sites reach
get_secret that way.

Widening the patterns alone would demand about 320 new rows in the central
reference table, most of them provider credentials that are already documented
on their own provider pages. So the gate now looks across every page of the docs
site rather than only that one table, which leaves 143 keys genuinely
undocumented instead of 322.

tests/documentation_tests/test_env_keys.py

@@ -10,11 +10,14 @@ _GET_SECRET_ARGS = r"""\(\s*['"]([^'"]+)['"]\s*(?:,\s*[^)]*|,\s*default_value=[^
 
 ENV_KEY_PATTERNS: tuple[re.Pattern[str], ...] = (
     re.compile(r"os\.getenv" + _GETENV_ARGS),
-    re.compile(r"litellm\.get_secret" + _GET_SECRET_ARGS),
-    re.compile(r"litellm\.get_secret_str" + _GET_SECRET_ARGS),
-    re.compile(r"(?<![\w.])(?:litellm\.)?get_secret_bool" + _GET_SECRET_ARGS),
+    re.compile(r"(?<![\w.])(?:litellm\.(?:utils\.)?)?get_secret(?:_str|_bool)?" + _GET_SECRET_ARGS),
 )
 
+DOCS_BASE = "./docs/my-website/docs"
+REFERENCE_TABLE_PATH = f"{DOCS_BASE}/proxy/config_settings.md"
+DOCS_SUFFIXES = (".md", ".mdx")
+DOCUMENTED_KEY_PATTERN = re.compile(r"\b[A-Z][A-Z0-9_]*\b")
+
 # Terminal/environment detection variables that should not be documented
 # These are internal variables used for terminal detection, not user-configurable settings
 # Guard-only env vars: read solely to raise on invalid values; the only valid
@@ -72,14 +75,16 @@ def extract_env_keys(source: str) -> frozenset[str]:
 
 def collect_env_keys(base_dir: str) -> frozenset[str]:
     """Return every documentable env var name read anywhere under ``base_dir``."""
-    return frozenset(key for file_path in _python_files(base_dir) for key in extract_env_keys(_read_text(file_path)))
+    return frozenset(
+        key for file_path in _files_with_suffix(base_dir, (".py",)) for key in extract_env_keys(_read_text(file_path))
+    )
 
 
-def _python_files(base_dir: str) -> Iterator[str]:
+def _files_with_suffix(base_dir: str, suffixes: tuple[str, ...]) -> Iterator[str]:
     for root, dirs, files in os.walk(base_dir):
         # Skip dependency/venv directories - prevents picking up env vars from installed packages
         dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
-        yield from (os.path.join(root, name) for name in files if name.endswith(".py"))
+        yield from (os.path.join(root, name) for name in files if name.endswith(suffixes))
 
 
 def _read_text(file_path: str) -> str:
@@ -88,42 +93,37 @@ def _read_text(file_path: str) -> str:
 
 
 def extract_documented_keys(docs_content: str) -> frozenset[str]:
-    """Return the key names listed in the 'environment variables - Reference' table."""
-    section = re.search(
-        r"### environment variables - Reference(.*?)(?=\n###|\Z)",
-        docs_content,
-        re.DOTALL | re.MULTILINE,
-    )
-    if section is None:
-        return frozenset()
-    # Match | KEY_NAME | description | - capture first column only
+    """Return every env-var-shaped name mentioned anywhere in a documentation page."""
+    return frozenset(DOCUMENTED_KEY_PATTERN.findall(docs_content))
+
+
+def collect_documented_keys(docs_dir: str) -> frozenset[str]:
+    """Return every env-var-shaped name mentioned on any page under ``docs_dir``."""
     return frozenset(
-        match.group(1).strip()
-        for match in (re.match(r"^\|\s*([A-Z_][A-Z0-9_]*)\s*\|", line) for line in section.group(1).split("\n"))
-        if match is not None
+        key
+        for file_path in _files_with_suffix(docs_dir, DOCS_SUFFIXES)
+        for key in extract_documented_keys(_read_text(file_path))
     )
 
 
+def undocumented_env_keys(base_dir: str, docs_dir: str) -> frozenset[str]:
+    """Return the env vars read under ``base_dir`` that no page under ``docs_dir`` mentions."""
+    return collect_env_keys(base_dir) - collect_documented_keys(docs_dir)
+
+
 def main() -> None:
-    env_keys = collect_env_keys(repo_base)
-    print(env_keys)
-
-    docs_path = "./docs/my-website/docs/proxy/config_settings.md"  # Path to the documentation
-    try:
-        documented_keys = extract_documented_keys(_read_text(docs_path))
-    except Exception as e:
-        raise Exception(f"Error reading documentation: {e}, \n repo base - {os.listdir('./')}")
-
-    print(f"documented_keys: {documented_keys}")
-    undocumented_keys = env_keys - documented_keys
-
-    print("Keys expected in 'environment settings' (found in code):")
-    for key in sorted(env_keys):
-        print(key)
+    if not os.path.isdir(DOCS_BASE):
+        raise Exception(f"No documentation found at {DOCS_BASE}; check out BerriAI/litellm-docs into docs/my-website")
 
+    undocumented_keys = undocumented_env_keys(repo_base, DOCS_BASE)
     if undocumented_keys:
-        raise Exception(f"\nKeys not documented in 'environment settings - Reference': {sorted(undocumented_keys)}")
-    print(f"\nAll keys are documented in 'environment settings - Reference'. - {env_keys}")
+        raise Exception(
+            f"Environment variables read under {repo_base} but mentioned nowhere in the docs: "
+            f"{sorted(undocumented_keys)}"
+            f"\nDocument each one, either on the relevant provider page or as a row in the "
+            f"'environment variables - Reference' table in {REFERENCE_TABLE_PATH}"
+        )
+    print(f"Every environment variable read under {repo_base} is documented")
 
 
 if __name__ == "__main__":

tests/test_litellm/test_env_key_doc_gate.py

@@ -1,11 +1,12 @@
 """Tests for the env-var extraction used by tests/documentation_tests/test_env_keys.py.
 
 That script is the CI gate that fails when a user-facing environment variable read
-under litellm/ has no row in the docs reference table. It only sees a key if one of its
+under litellm/ is mentioned nowhere on the docs site. It only sees a key if one of its
 patterns matches the call, so a call shape the patterns miss silently bypasses the gate.
 Each supported shape is asserted here, along with the shapes that must not be treated as
 env var reads, so narrowing a pattern makes a test fail instead of quietly reopening the
-hole.
+hole. The docs side is asserted too, since a key documented on a provider page rather
+than in the central reference table still counts as documented.
 """
 
 import importlib.util
@@ -41,6 +42,36 @@ def test_litellm_prefixed_get_secret_bool_is_captured() -> None:
     assert gate.extract_env_keys('litellm.get_secret_bool("QSTASH_FLUSH_ON_BOOT")') == {"QSTASH_FLUSH_ON_BOOT"}
 
 
+def test_bare_get_secret_is_captured() -> None:
+    assert gate.extract_env_keys('key = get_secret("QSTASH_ALPHA")') == {"QSTASH_ALPHA"}
+
+
+def test_bare_get_secret_with_default_is_captured() -> None:
+    assert gate.extract_env_keys('key = get_secret("QSTASH_ALPHA", "fallback")') == {"QSTASH_ALPHA"}
+
+
+def test_bare_get_secret_str_is_captured() -> None:
+    assert gate.extract_env_keys('key = get_secret_str("QSTASH_BRAVO")') == {"QSTASH_BRAVO"}
+
+
+def test_bare_get_secret_str_with_keyword_default_is_captured() -> None:
+    assert gate.extract_env_keys('get_secret_str("QSTASH_BRAVO", default_value=None)') == {"QSTASH_BRAVO"}
+
+
+def test_get_secret_reached_through_the_utils_module_is_captured() -> None:
+    assert gate.extract_env_keys('litellm.utils.get_secret("QSTASH_ALPHA")') == {"QSTASH_ALPHA"}
+
+
+def test_get_secret_on_an_unrelated_utils_attribute_is_not_an_env_read() -> None:
+    source = "\n".join(
+        (
+            'vault.utils.get_secret("QSTASH_ALPHA")',
+            'self.utils.get_secret_str("QSTASH_BRAVO")',
+        )
+    )
+    assert gate.extract_env_keys(source) == frozenset()
+
+
 def test_previously_supported_call_shapes_are_still_captured() -> None:
     source = "\n".join(
         (
@@ -83,21 +114,77 @@ def test_excluded_keys_are_filtered_for_every_call_shape() -> None:
     assert gate.extract_env_keys(source) == frozenset()
 
 
-def test_documented_keys_are_read_from_the_reference_table_only() -> None:
+def test_a_key_mentioned_outside_the_reference_table_counts_as_documented() -> None:
     docs = "\n".join(
         (
-            "### general_settings - Reference",
-            "| BEFORE_THE_TABLE | not the env var table",
+            "# Qstash",
             "",
-            "### environment variables - Reference",
+            "Set `QSTASH_ALPHA` to your endpoint before calling the provider.",
             "",
-            "| Name | Description |",
-            "|------|-------------|",
-            "| QSTASH_ALPHA | first key",
-            "| QSTASH_BRAVO | second key",
-            "",
-            "### another section - Reference",
-            "| AFTER_THE_TABLE | also not the env var table",
+            "```bash",
+            'export QSTASH_BRAVO="sk-..."',
+            "```",
         )
     )
-    assert gate.extract_documented_keys(docs) == {"QSTASH_ALPHA", "QSTASH_BRAVO"}
+    documented = gate.extract_documented_keys(docs)
+    assert "QSTASH_ALPHA" in documented
+    assert "QSTASH_BRAVO" in documented
+
+
+def test_a_name_glued_to_surrounding_text_is_not_a_mention() -> None:
+    documented = gate.extract_documented_keys("the useQSTASH_ALPHA helper reads it")
+    assert "QSTASH_ALPHA" not in documented
+
+
+def test_a_longer_name_does_not_document_the_key_it_ends_with() -> None:
+    documented = gate.extract_documented_keys("Set AZURE_QSTASH_ALPHA in your environment")
+    assert "AZURE_QSTASH_ALPHA" in documented
+    assert "QSTASH_ALPHA" not in documented
+
+
+def test_lowercase_mentions_are_not_treated_as_env_var_names() -> None:
+    assert gate.extract_documented_keys("pass qstash_alpha as a config key") == frozenset()
+
+
+def test_documented_keys_are_collected_from_every_page_of_the_docs_site(tmp_path: Path) -> None:
+    (tmp_path / "providers").mkdir()
+    (tmp_path / "providers" / "qstash.md").write_text("Set `QSTASH_ALPHA` to your endpoint.\n", encoding="utf-8")
+    (tmp_path / "providers" / "qstash_batches.mdx").write_text("| QSTASH_BRAVO | second key |\n", encoding="utf-8")
+    documented = gate.collect_documented_keys(str(tmp_path))
+    assert "QSTASH_ALPHA" in documented
+    assert "QSTASH_BRAVO" in documented
+    assert "QSTASH_CHARLIE" not in documented
+
+
+def _write_tree(tmp_path: Path, source: str, docs_page: str) -> tuple[str, str]:
+    source_dir = tmp_path / "litellm"
+    docs_dir = tmp_path / "docs" / "providers"
+    source_dir.mkdir()
+    docs_dir.mkdir(parents=True)
+    (source_dir / "qstash.py").write_text(source, encoding="utf-8")
+    (docs_dir / "qstash.md").write_text(docs_page, encoding="utf-8")
+    return str(source_dir), str(tmp_path / "docs")
+
+
+def test_a_key_documented_only_on_a_provider_page_satisfies_the_gate(tmp_path: Path) -> None:
+    source_dir, docs_dir = _write_tree(
+        tmp_path,
+        'api_key = get_secret_str("QSTASH_ALPHA")\n',
+        "# Qstash\n\nSet `QSTASH_ALPHA` to your API key.\n",
+    )
+    assert gate.undocumented_env_keys(source_dir, docs_dir) == frozenset()
+
+
+def test_a_key_documented_on_no_page_at_all_fails_the_gate(tmp_path: Path) -> None:
+    source_dir, docs_dir = _write_tree(
+        tmp_path,
+        'api_key = get_secret_str("QSTASH_ALPHA")\n',
+        "# Qstash\n\nThis provider needs an API key.\n",
+    )
+    assert gate.undocumented_env_keys(source_dir, docs_dir) == {"QSTASH_ALPHA"}
+
+
+def test_only_documentation_pages_are_scanned_for_mentions(tmp_path: Path) -> None:
+    (tmp_path / "notes.txt").write_text("QSTASH_ALPHA\n", encoding="utf-8")
+    (tmp_path / "example.py").write_text('get_secret("QSTASH_BRAVO")\n', encoding="utf-8")
+    assert gate.collect_documented_keys(str(tmp_path)) == frozenset()