Merge pull request #42143 from BerriAI/litellm_e2e_changed_keep_pytest_log

ci(e2e): fix the stage-mirror batch reds and keep a redacted pytest log

.github/e2e-stack/redact_output.py

@@ -0,0 +1,83 @@
+import argparse
+import os
+import sys
+from functools import reduce
+from pathlib import Path
+from typing import Final
+from xml.sax.saxutils import escape
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from secrets_to_env import MIN_MASKED_LENGTH
+
+REDACTED: Final = "***"
+json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+
+
+def string_leaves(node: JsonValue) -> tuple[str, ...]:
+    match node:
+        case str():
+            return (node,)
+        case list():
+            return tuple(leaf for child in node for leaf in string_leaves(child))
+        case dict():
+            return tuple(leaf for child in node.values() for leaf in string_leaves(child))
+    return ()
+
+
+def field_lines(value: str) -> tuple[str, ...]:
+    try:
+        return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
+    except ValidationError:
+        return ()
+
+
+def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
+    values: Final = frozenset(
+        line.split("=", 1)[1].strip().strip("'")
+        for path in values_files
+        for line in path.read_text().splitlines()
+        if "=" in line
+    )
+    texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
+    renderings: Final = frozenset(
+        rendering
+        for text in texts
+        if len(text) >= MIN_MASKED_LENGTH
+        for rendering in (text, escape(text), escape(text, {'"': """}))
+    )
+    return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
+
+
+def redact(text: str, values: tuple[str, ...]) -> str:
+    return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
+
+
+def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
+    target: Final = out_dir / source.name
+    with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
+        _ = handle.write(redact(source.read_text(errors="replace"), values))
+
+
+def main() -> int:
+    parser: Final = argparse.ArgumentParser()
+    _ = parser.add_argument("--values", action="append", type=Path, required=True)
+    _ = parser.add_argument("--out", type=Path, required=True)
+    _ = parser.add_argument("files", nargs="*", type=Path)
+    args: Final = parser.parse_args()
+    values_files: Final = tuple(args.values)
+    out_dir: Final[Path] = args.out
+    sources: Final = tuple(args.files)
+    try:
+        values: Final = masked_values(values_files)
+        out_dir.mkdir(mode=0o700, exist_ok=True)
+        for source in sources:
+            write_redacted(source, out_dir, values)
+    except OSError as error:
+        _ = sys.stderr.write(f"could not redact {error.filename}\n")
+        return 1
+    _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/up.sh

@@ -143,7 +143,7 @@ env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/m
 
 start_server() {
   local name="$1"; shift
-  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
   echo $! > "${PIDS_DIR}/${name}.pid"
 }
 

.github/workflows/test-e2e-changed.yml

@@ -209,6 +209,24 @@ jobs:
             echo "pass ${pass} of 3 passed"
           done
 
+      - name: Redact the pytest output
+        if: always() && steps.boot.outcome == 'success'
+        run: |
+          umask 077
+          shopt -s nullglob
+          uv run --no-sync python .github/e2e-stack/redact_output.py \
+            --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
+            --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+
+      - name: Keep the redacted pytest output
+        if: always() && steps.boot.outcome == 'success'
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-changed-pytest-output-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/e2e-redacted
+          retention-days: 14
+          if-no-files-found: ignore
+
       - name: Stop the stack
         if: always() && steps.boot.outcome != 'skipped'
         run: bash .github/e2e-stack/down.sh
@@ -217,7 +235,7 @@ jobs:
         if: always()
         run: |
           rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
-          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
 
   gate:
     name: e2e-changed-tests

tests/code_coverage_tests/test_e2e_changed_gate.py

@@ -10,6 +10,7 @@ import pytest
 GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
 SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py")
 SELECT_TESTS: Final = GATE.with_name("select_tests.py")
+REDACT_OUTPUT: Final = GATE.with_name("redact_output.py")
 CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 
@@ -116,6 +117,81 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat
     assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n"
 
 
+def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]:
+    env_path: Final = tmp_path / ".env"
+    _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values)))
+    stack_env: Final = tmp_path / "stack.env"
+    _ = stack_env.write_text("LITELLM_MASTER_KEY=sk-e2e-master0123\nREDIS_PORT=6379\n")
+    log: Final = tmp_path / "e2e-pass-1.log"
+    _ = log.write_text(text)
+    out_dir: Final = tmp_path / "redacted"
+    result: Final = subprocess.run(  # test-quality-ok: standalone script that imports its sibling by script directory
+        [
+            sys.executable,
+            str(REDACT_OUTPUT),
+            "--values",
+            str(env_path),
+            "--values",
+            str(stack_env),
+            "--out",
+            str(out_dir),
+            str(log),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    return result, out_dir / log.name
+
+
+def test_redacted_output_hides_every_masked_value_and_keeps_the_rest(tmp_path: Path) -> None:
+    text: Final = (
+        "FAILED key=sk-0123456789abcdef master=sk-e2e-master0123 flag=1 port=6379 message=Missing credentials\n"
+    )
+
+    result, redacted = redact_output(tmp_path, ("sk-0123456789abcdef", "1"), text)
+
+    assert result.returncode == 0, result.stderr
+    assert redacted.read_text() == "FAILED key=*** master=*** flag=1 port=6379 message=Missing credentials\n"
+    assert (redacted.stat().st_mode & 0o777) == 0o600
+    assert (tmp_path / "e2e-pass-1.log").read_text() == text
+    assert "sk-" not in result.stdout + result.stderr
+
+
+def test_a_masked_value_that_prefixes_a_longer_one_leaves_no_tail(tmp_path: Path) -> None:
+    result, redacted = redact_output(tmp_path, ("sk-0123456789", "sk-0123456789abcdef"), "token sk-0123456789abcdef\n")
+
+    assert result.returncode == 0, result.stderr
+    assert redacted.read_text() == "token ***\n"
+
+
+def test_a_json_secret_is_hidden_field_by_field_however_it_is_escaped(tmp_path: Path) -> None:
+    credentials: Final = (
+        '{"type": "service_account", "signing_key": "MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\n'
+        'c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n", "client_id": "104857600000000000001"}'
+    )
+    text: Final = (
+        "decoded MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\n"
+        "c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n"
+        "escaped MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n\n"
+        "twice MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n"
+        "client 104857600000000000001 status 403\n"
+    )
+
+    result, redacted = redact_output(tmp_path, (credentials,), text)
+
+    assert result.returncode == 0, result.stderr
+    assert redacted.read_text() == "decoded ***\n***\nescaped ***\\n***\\n\ntwice ***\\\\n***\nclient *** status 403\n"
+
+
+def test_a_secret_with_xml_special_characters_is_hidden_in_the_junit_file(tmp_path: Path) -> None:
+    text: Final = '<failure message="got p&amp;ss&lt;w&quot;rd-1">body p&amp;ss&lt;w"rd-1</failure>\n'
+
+    result, redacted = redact_output(tmp_path, ('p&ss<w"rd-1',), text)
+
+    assert result.returncode == 0, result.stderr
+    assert redacted.read_text() == '<failure message="got ***">body ***</failure>\n'
+
+
 def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
     result: Final = subprocess.run(
         [sys.executable, str(SELECT_TESTS), *CANARY],

tests/e2e/gateway/stage_mirror_ci_config.yml

@@ -64,6 +64,23 @@ model_list:
       model: openai/text-embedding-3-small
       api_key: os.environ/OPENAI_API_KEY
 
+files_settings:
+  - custom_llm_provider: openai
+    api_key: os.environ/OPENAI_API_KEY
+  - custom_llm_provider: azure
+    api_base: os.environ/AZURE_API_BASE
+    api_key: os.environ/AZURE_API_KEY
+    api_version: 2025-04-01-preview
+  - custom_llm_provider: vertex_ai
+    vertex_project: os.environ/VERTEXAI_PROJECT
+    vertex_location: us-central1
+    vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
+    bucket_name: os.environ/GCS_BUCKET_NAME
+
+finetune_settings:
+  - custom_llm_provider: openai
+    api_key: os.environ/OPENAI_API_KEY
+
 mcp_servers:
   devin:
     url: "https://mcp.devin.ai/mcp"