diff --git a/cookbook/prompt_portability_check/README.md b/cookbook/prompt_portability_check/README.md new file mode 100644 index 00000000000..4c63410f9a2 --- /dev/null +++ b/cookbook/prompt_portability_check/README.md @@ -0,0 +1,31 @@ +# Lint a request for cross-provider portability before a LiteLLM provider swap + +[LiteLLM](https://github.com/BerriAI/litellm) standardizes the *call* -- one +`completion()` signature across OpenAI, Anthropic, Gemini, Bedrock, and more. +It does not check whether the *content* of your messages/params (a hardcoded +"as an OpenAI assistant" system prompt, a provider-specific stop-sequence +limit, a temperature outside the target provider's accepted range, a leaked +chat-template token) will still behave correctly once you point `model=` at +a different provider. + +[`prompt-portability`](https://github.com/nac7/prompt-portability) is an +open-source CLI/library that lints exactly that gap. This example lints a +request before handing it to `litellm.completion()`. + +## Run it + +```bash +pip install prompt-portability litellm +python check_before_provider_swap.py +``` + +## What it does + +1. Builds a request payload written and tested against OpenAI. +2. Runs `prompt-portability`'s linter against it, surfacing portability + issues that would only otherwise show up once the request actually hits + a different provider. +3. Calls `litellm.completion()` with the same payload (mocked, so no API key + is needed to run this example) to show LiteLLM's own call succeeds + regardless -- confirming these are issues LiteLLM's translation layer + does not catch on its own. diff --git a/cookbook/prompt_portability_check/check_before_provider_swap.py b/cookbook/prompt_portability_check/check_before_provider_swap.py new file mode 100644 index 00000000000..fa24f6168dd --- /dev/null +++ b/cookbook/prompt_portability_check/check_before_provider_swap.py @@ -0,0 +1,70 @@ +""" +Lint a request payload for cross-provider portability *before* handing it to +LiteLLM. + +LiteLLM standardizes the *call* -- one `completion()` signature that routes +to OpenAI, Anthropic, Gemini, Bedrock, etc. It does not, however, check +whether the *content* of your messages/params is actually safe to send to +every provider behind that call. Things like: + + - a system prompt that hardcodes "As an OpenAI language model..." + - a stop_sequences list with 5 entries (OpenAI caps this at 4) + - a temperature of 2.5 (outside Anthropic/Gemini's accepted range) + - a chat-template special token leaked from a different model family + +...will pass straight through LiteLLM's translation layer and fail (or +silently misbehave) only once they hit the target provider's API. +`prompt-portability` catches these before the call is made, so a provider +swap in your LiteLLM `model=` string doesn't surface a portability bug at +runtime. + +Install: + pip install prompt-portability litellm + +Run: + python check_before_provider_swap.py +""" + +from __future__ import annotations + +import litellm +from llm_prompt_lint.linter import lint +from llm_prompt_lint.parsers import detect_and_parse + +request = { + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant. As an OpenAI assistant, " + "always answer in a formal tone.", + }, + {"role": "user", "content": "Summarize this quarter's RAG index drift report."}, + ], + "temperature": 2.5, + "stop": ["\n\n", "END", "###", "STOP", "<|end|>"], +} + +doc = detect_and_parse(request) +report = lint(doc) + +print(f"prompt-portability found {len(report.findings)} portability issue(s):\n") +for finding in report.findings: + print(f" [{finding.rule_id}] {finding.message}") + +if report.findings: + print( + "\nFix these before swapping providers -- e.g. via LiteLLM's " + "`model=\"anthropic/claude-...\"` -- to avoid a runtime surprise on " + "the new provider." + ) + +response = litellm.completion( + model="gpt-4o", + messages=request["messages"], + temperature=request["temperature"], + stop=request["stop"], + mock_response="This call succeeds even though the payload above has " + "portability issues LiteLLM doesn't check.", +) +print(f"\nLiteLLM call (mocked) still went through: {response.choices[0].message.content!r}")