diff --git a/.github/e2e-stack/redact_output.py b/.github/e2e-stack/redact_output.py
new file mode 100644
index 00000000000..233a8be3e2d
--- /dev/null
+++ b/.github/e2e-stack/redact_output.py
@@ -0,0 +1,83 @@
+import argparse
+import os
+import sys
+from functools import reduce
+from pathlib import Path
+from typing import Final
+from xml.sax.saxutils import escape
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from secrets_to_env import MIN_MASKED_LENGTH
+
+REDACTED: Final = "***"
+json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+
+
+def string_leaves(node: JsonValue) -> tuple[str, ...]:
+ match node:
+ case str():
+ return (node,)
+ case list():
+ return tuple(leaf for child in node for leaf in string_leaves(child))
+ case dict():
+ return tuple(leaf for child in node.values() for leaf in string_leaves(child))
+ return ()
+
+
+def field_lines(value: str) -> tuple[str, ...]:
+ try:
+ return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
+ except ValidationError:
+ return ()
+
+
+def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
+ values: Final = frozenset(
+ line.split("=", 1)[1].strip().strip("'")
+ for path in values_files
+ for line in path.read_text().splitlines()
+ if "=" in line
+ )
+ texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
+ renderings: Final = frozenset(
+ rendering
+ for text in texts
+ if len(text) >= MIN_MASKED_LENGTH
+ for rendering in (text, escape(text), escape(text, {'"': """}))
+ )
+ return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
+
+
+def redact(text: str, values: tuple[str, ...]) -> str:
+ return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
+
+
+def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
+ target: Final = out_dir / source.name
+ with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
+ _ = handle.write(redact(source.read_text(errors="replace"), values))
+
+
+def main() -> int:
+ parser: Final = argparse.ArgumentParser()
+ _ = parser.add_argument("--values", action="append", type=Path, required=True)
+ _ = parser.add_argument("--out", type=Path, required=True)
+ _ = parser.add_argument("files", nargs="*", type=Path)
+ args: Final = parser.parse_args()
+ values_files: Final = tuple(args.values)
+ out_dir: Final[Path] = args.out
+ sources: Final = tuple(args.files)
+ try:
+ values: Final = masked_values(values_files)
+ out_dir.mkdir(mode=0o700, exist_ok=True)
+ for source in sources:
+ write_redacted(source, out_dir, values)
+ except OSError as error:
+ _ = sys.stderr.write(f"could not redact {error.filename}\n")
+ return 1
+ _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh
index a789a570483..928b58e93bb 100755
--- a/.github/e2e-stack/up.sh
+++ b/.github/e2e-stack/up.sh
@@ -143,7 +143,7 @@ env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/m
start_server() {
local name="$1"; shift
- env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+ env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
echo $! > "${PIDS_DIR}/${name}.pid"
}
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index 6da16a33ea3..8e03a902383 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -209,6 +209,24 @@ jobs:
echo "pass ${pass} of 3 passed"
done
+ - name: Redact the pytest output
+ if: always() && steps.boot.outcome == 'success'
+ run: |
+ umask 077
+ shopt -s nullglob
+ uv run --no-sync python .github/e2e-stack/redact_output.py \
+ --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
+ --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+
+ - name: Keep the redacted pytest output
+ if: always() && steps.boot.outcome == 'success'
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: e2e-changed-pytest-output-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/e2e-redacted
+ retention-days: 14
+ if-no-files-found: ignore
+
- name: Stop the stack
if: always() && steps.boot.outcome != 'skipped'
run: bash .github/e2e-stack/down.sh
@@ -217,7 +235,7 @@ jobs:
if: always()
run: |
rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
- rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+ rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
gate:
name: e2e-changed-tests
diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py
index 9519145570c..78e6562a4a8 100644
--- a/tests/code_coverage_tests/test_e2e_changed_gate.py
+++ b/tests/code_coverage_tests/test_e2e_changed_gate.py
@@ -10,6 +10,7 @@ import pytest
GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py")
SELECT_TESTS: Final = GATE.with_name("select_tests.py")
+REDACT_OUTPUT: Final = GATE.with_name("redact_output.py")
CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
@@ -116,6 +117,81 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat
assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n"
+def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]:
+ env_path: Final = tmp_path / ".env"
+ _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values)))
+ stack_env: Final = tmp_path / "stack.env"
+ _ = stack_env.write_text("LITELLM_MASTER_KEY=sk-e2e-master0123\nREDIS_PORT=6379\n")
+ log: Final = tmp_path / "e2e-pass-1.log"
+ _ = log.write_text(text)
+ out_dir: Final = tmp_path / "redacted"
+ result: Final = subprocess.run( # test-quality-ok: standalone script that imports its sibling by script directory
+ [
+ sys.executable,
+ str(REDACT_OUTPUT),
+ "--values",
+ str(env_path),
+ "--values",
+ str(stack_env),
+ "--out",
+ str(out_dir),
+ str(log),
+ ],
+ capture_output=True,
+ text=True,
+ )
+ return result, out_dir / log.name
+
+
+def test_redacted_output_hides_every_masked_value_and_keeps_the_rest(tmp_path: Path) -> None:
+ text: Final = (
+ "FAILED key=sk-0123456789abcdef master=sk-e2e-master0123 flag=1 port=6379 message=Missing credentials\n"
+ )
+
+ result, redacted = redact_output(tmp_path, ("sk-0123456789abcdef", "1"), text)
+
+ assert result.returncode == 0, result.stderr
+ assert redacted.read_text() == "FAILED key=*** master=*** flag=1 port=6379 message=Missing credentials\n"
+ assert (redacted.stat().st_mode & 0o777) == 0o600
+ assert (tmp_path / "e2e-pass-1.log").read_text() == text
+ assert "sk-" not in result.stdout + result.stderr
+
+
+def test_a_masked_value_that_prefixes_a_longer_one_leaves_no_tail(tmp_path: Path) -> None:
+ result, redacted = redact_output(tmp_path, ("sk-0123456789", "sk-0123456789abcdef"), "token sk-0123456789abcdef\n")
+
+ assert result.returncode == 0, result.stderr
+ assert redacted.read_text() == "token ***\n"
+
+
+def test_a_json_secret_is_hidden_field_by_field_however_it_is_escaped(tmp_path: Path) -> None:
+ credentials: Final = (
+ '{"type": "service_account", "signing_key": "MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\n'
+ 'c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n", "client_id": "104857600000000000001"}'
+ )
+ text: Final = (
+ "decoded MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\n"
+ "c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n"
+ "escaped MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n\n"
+ "twice MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n"
+ "client 104857600000000000001 status 403\n"
+ )
+
+ result, redacted = redact_output(tmp_path, (credentials,), text)
+
+ assert result.returncode == 0, result.stderr
+ assert redacted.read_text() == "decoded ***\n***\nescaped ***\\n***\\n\ntwice ***\\\\n***\nclient *** status 403\n"
+
+
+def test_a_secret_with_xml_special_characters_is_hidden_in_the_junit_file(tmp_path: Path) -> None:
+ text: Final = 'body p&ss<w"rd-1\n'
+
+ result, redacted = redact_output(tmp_path, ('p&ssbody ***\n'
+
+
def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
result: Final = subprocess.run(
[sys.executable, str(SELECT_TESTS), *CANARY],
diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml
index 8c8e64443cb..352caddf588 100644
--- a/tests/e2e/gateway/stage_mirror_ci_config.yml
+++ b/tests/e2e/gateway/stage_mirror_ci_config.yml
@@ -64,6 +64,23 @@ model_list:
model: openai/text-embedding-3-small
api_key: os.environ/OPENAI_API_KEY
+files_settings:
+ - custom_llm_provider: openai
+ api_key: os.environ/OPENAI_API_KEY
+ - custom_llm_provider: azure
+ api_base: os.environ/AZURE_API_BASE
+ api_key: os.environ/AZURE_API_KEY
+ api_version: 2025-04-01-preview
+ - custom_llm_provider: vertex_ai
+ vertex_project: os.environ/VERTEXAI_PROJECT
+ vertex_location: us-central1
+ vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
+ bucket_name: os.environ/GCS_BUCKET_NAME
+
+finetune_settings:
+ - custom_llm_provider: openai
+ api_key: os.environ/OPENAI_API_KEY
+
mcp_servers:
devin:
url: "https://mcp.devin.ai/mcp"