diff --git a/.github/e2e-stack/redact_output.py b/.github/e2e-stack/redact_output.py new file mode 100644 index 00000000000..233a8be3e2d --- /dev/null +++ b/.github/e2e-stack/redact_output.py @@ -0,0 +1,83 @@ +import argparse +import os +import sys +from functools import reduce +from pathlib import Path +from typing import Final +from xml.sax.saxutils import escape + +from pydantic import JsonValue, TypeAdapter, ValidationError +from secrets_to_env import MIN_MASKED_LENGTH + +REDACTED: Final = "***" +json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def string_leaves(node: JsonValue) -> tuple[str, ...]: + match node: + case str(): + return (node,) + case list(): + return tuple(leaf for child in node for leaf in string_leaves(child)) + case dict(): + return tuple(leaf for child in node.values() for leaf in string_leaves(child)) + return () + + +def field_lines(value: str) -> tuple[str, ...]: + try: + return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines()) + except ValidationError: + return () + + +def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]: + values: Final = frozenset( + line.split("=", 1)[1].strip().strip("'") + for path in values_files + for line in path.read_text().splitlines() + if "=" in line + ) + texts: Final = frozenset(text for value in values for text in (value, *field_lines(value))) + renderings: Final = frozenset( + rendering + for text in texts + if len(text) >= MIN_MASKED_LENGTH + for rendering in (text, escape(text), escape(text, {'"': """})) + ) + return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering))) + + +def redact(text: str, values: tuple[str, ...]) -> str: + return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text) + + +def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None: + target: Final = out_dir / source.name + with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle: + _ = handle.write(redact(source.read_text(errors="replace"), values)) + + +def main() -> int: + parser: Final = argparse.ArgumentParser() + _ = parser.add_argument("--values", action="append", type=Path, required=True) + _ = parser.add_argument("--out", type=Path, required=True) + _ = parser.add_argument("files", nargs="*", type=Path) + args: Final = parser.parse_args() + values_files: Final = tuple(args.values) + out_dir: Final[Path] = args.out + sources: Final = tuple(args.files) + try: + values: Final = masked_values(values_files) + out_dir.mkdir(mode=0o700, exist_ok=True) + for source in sources: + write_redacted(source, out_dir, values) + except OSError as error: + _ = sys.stderr.write(f"could not redact {error.filename}\n") + return 1 + _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index a789a570483..928b58e93bb 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -143,7 +143,7 @@ env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/m start_server() { local name="$1"; shift - env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 & + env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 & echo $! > "${PIDS_DIR}/${name}.pid" } diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 6da16a33ea3..8e03a902383 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -209,6 +209,24 @@ jobs: echo "pass ${pass} of 3 passed" done + - name: Redact the pytest output + if: always() && steps.boot.outcome == 'success' + run: | + umask 077 + shopt -s nullglob + uv run --no-sync python .github/e2e-stack/redact_output.py \ + --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \ + --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml + + - name: Keep the redacted pytest output + if: always() && steps.boot.outcome == 'success' + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: e2e-changed-pytest-output-${{ github.run_attempt }} + path: ${{ runner.temp }}/e2e-redacted + retention-days: 14 + if-no-files-found: ignore + - name: Stop the stack if: always() && steps.boot.outcome != 'skipped' run: bash .github/e2e-stack/down.sh @@ -217,7 +235,7 @@ jobs: if: always() run: | rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml - rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" + rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted" gate: name: e2e-changed-tests diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 9519145570c..78e6562a4a8 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -10,6 +10,7 @@ import pytest GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py" SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py") SELECT_TESTS: Final = GATE.with_name("select_tests.py") +REDACT_OUTPUT: Final = GATE.with_name("redact_output.py") CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") @@ -116,6 +117,81 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" +def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]: + env_path: Final = tmp_path / ".env" + _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values))) + stack_env: Final = tmp_path / "stack.env" + _ = stack_env.write_text("LITELLM_MASTER_KEY=sk-e2e-master0123\nREDIS_PORT=6379\n") + log: Final = tmp_path / "e2e-pass-1.log" + _ = log.write_text(text) + out_dir: Final = tmp_path / "redacted" + result: Final = subprocess.run( # test-quality-ok: standalone script that imports its sibling by script directory + [ + sys.executable, + str(REDACT_OUTPUT), + "--values", + str(env_path), + "--values", + str(stack_env), + "--out", + str(out_dir), + str(log), + ], + capture_output=True, + text=True, + ) + return result, out_dir / log.name + + +def test_redacted_output_hides_every_masked_value_and_keeps_the_rest(tmp_path: Path) -> None: + text: Final = ( + "FAILED key=sk-0123456789abcdef master=sk-e2e-master0123 flag=1 port=6379 message=Missing credentials\n" + ) + + result, redacted = redact_output(tmp_path, ("sk-0123456789abcdef", "1"), text) + + assert result.returncode == 0, result.stderr + assert redacted.read_text() == "FAILED key=*** master=*** flag=1 port=6379 message=Missing credentials\n" + assert (redacted.stat().st_mode & 0o777) == 0o600 + assert (tmp_path / "e2e-pass-1.log").read_text() == text + assert "sk-" not in result.stdout + result.stderr + + +def test_a_masked_value_that_prefixes_a_longer_one_leaves_no_tail(tmp_path: Path) -> None: + result, redacted = redact_output(tmp_path, ("sk-0123456789", "sk-0123456789abcdef"), "token sk-0123456789abcdef\n") + + assert result.returncode == 0, result.stderr + assert redacted.read_text() == "token ***\n" + + +def test_a_json_secret_is_hidden_field_by_field_however_it_is_escaped(tmp_path: Path) -> None: + credentials: Final = ( + '{"type": "service_account", "signing_key": "MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\n' + 'c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n", "client_id": "104857600000000000001"}' + ) + text: Final = ( + "decoded MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\n" + "c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n" + "escaped MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n\n" + "twice MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n" + "client 104857600000000000001 status 403\n" + ) + + result, redacted = redact_output(tmp_path, (credentials,), text) + + assert result.returncode == 0, result.stderr + assert redacted.read_text() == "decoded ***\n***\nescaped ***\\n***\\n\ntwice ***\\\\n***\nclient *** status 403\n" + + +def test_a_secret_with_xml_special_characters_is_hidden_in_the_junit_file(tmp_path: Path) -> None: + text: Final = 'body p&ss<w"rd-1\n' + + result, redacted = redact_output(tmp_path, ('p&ssbody ***\n' + + def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: result: Final = subprocess.run( [sys.executable, str(SELECT_TESTS), *CANARY], diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 8c8e64443cb..352caddf588 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -64,6 +64,23 @@ model_list: model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY +files_settings: + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + - custom_llm_provider: azure + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: 2025-04-01-preview + - custom_llm_provider: vertex_ai + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: us-central1 + vertex_credentials: os.environ/VERTEXAI_CREDENTIALS + bucket_name: os.environ/GCS_BUCKET_NAME + +finetune_settings: + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + mcp_servers: devin: url: "https://mcp.devin.ai/mcp"