From 8caa6e7b1b8cead130829587868a90415817d61e Mon Sep 17 00:00:00 2001 From: nac7 <29607512+nac7@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:10:06 -0500 Subject: [PATCH 1/2] docs: add prompt-portability cookbook example (lint before provider swap) --- cookbook/prompt_portability_check/README.md | 31 ++++++++ .../check_before_provider_swap.py | 74 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 cookbook/prompt_portability_check/README.md create mode 100644 cookbook/prompt_portability_check/check_before_provider_swap.py diff --git a/cookbook/prompt_portability_check/README.md b/cookbook/prompt_portability_check/README.md new file mode 100644 index 00000000000..4c63410f9a2 --- /dev/null +++ b/cookbook/prompt_portability_check/README.md @@ -0,0 +1,31 @@ +# Lint a request for cross-provider portability before a LiteLLM provider swap + +[LiteLLM](https://github.com/BerriAI/litellm) standardizes the *call* -- one +`completion()` signature across OpenAI, Anthropic, Gemini, Bedrock, and more. +It does not check whether the *content* of your messages/params (a hardcoded +"as an OpenAI assistant" system prompt, a provider-specific stop-sequence +limit, a temperature outside the target provider's accepted range, a leaked +chat-template token) will still behave correctly once you point `model=` at +a different provider. + +[`prompt-portability`](https://github.com/nac7/prompt-portability) is an +open-source CLI/library that lints exactly that gap. This example lints a +request before handing it to `litellm.completion()`. + +## Run it + +```bash +pip install prompt-portability litellm +python check_before_provider_swap.py +``` + +## What it does + +1. Builds a request payload written and tested against OpenAI. +2. Runs `prompt-portability`'s linter against it, surfacing portability + issues that would only otherwise show up once the request actually hits + a different provider. +3. Calls `litellm.completion()` with the same payload (mocked, so no API key + is needed to run this example) to show LiteLLM's own call succeeds + regardless -- confirming these are issues LiteLLM's translation layer + does not catch on its own. diff --git a/cookbook/prompt_portability_check/check_before_provider_swap.py b/cookbook/prompt_portability_check/check_before_provider_swap.py new file mode 100644 index 00000000000..813e9bda531 --- /dev/null +++ b/cookbook/prompt_portability_check/check_before_provider_swap.py @@ -0,0 +1,74 @@ +""" +Lint a request payload for cross-provider portability *before* handing it to +LiteLLM. + +LiteLLM standardizes the *call* -- one `completion()` signature that routes +to OpenAI, Anthropic, Gemini, Bedrock, etc. It does not, however, check +whether the *content* of your messages/params is actually safe to send to +every provider behind that call. Things like: + + - a system prompt that hardcodes "As an OpenAI language model..." + - a stop_sequences list with 5 entries (OpenAI caps this at 4) + - a temperature of 2.5 (outside Anthropic/Gemini's accepted range) + - a chat-template special token leaked from a different model family + +...will pass straight through LiteLLM's translation layer and fail (or +silently misbehave) only once they hit the target provider's API. +`prompt-portability` catches these before the call is made, so a provider +swap in your LiteLLM `model=` string doesn't surface a portability bug at +runtime. + +Install: + pip install prompt-portability litellm + +Run: + python prompt_portability_litellm_check.py +""" + +from __future__ import annotations + +import litellm +from llm_prompt_lint.linter import lint +from llm_prompt_lint.parsers import detect_and_parse + +# A request written and tested against OpenAI, about to be pointed at a +# different provider via LiteLLM's `model=` switch. +request = { + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant. As an OpenAI assistant, " + "always answer in a formal tone.", + }, + {"role": "user", "content": "Summarize this quarter's RAG index drift report."}, + ], + "temperature": 2.5, + "stop": ["\n\n", "END", "###", "STOP", "<|end|>"], +} + +doc = detect_and_parse(request) +report = lint(doc) + +print(f"prompt-portability found {len(report.findings)} portability issue(s):\n") +for finding in report.findings: + print(f" [{finding.rule_id}] {finding.message}") + +if report.findings: + print( + "\nFix these before swapping providers -- e.g. via LiteLLM's " + "`model=\"anthropic/claude-...\"` -- to avoid a runtime surprise on " + "the new provider." + ) + +# LiteLLM's own call succeeds regardless -- it standardizes the transport, +# not the content, which is exactly the gap prompt-portability closes. +response = litellm.completion( + model="gpt-4o", + messages=request["messages"], + temperature=request["temperature"], + stop=request["stop"], + mock_response="This call succeeds even though the payload above has " + "portability issues LiteLLM doesn't check.", +) +print(f"\nLiteLLM call (mocked) still went through: {response.choices[0].message.content!r}") From e1f48d2d97a46917ee9ca817da07b177832153a8 Mon Sep 17 00:00:00 2001 From: nac7 <29607512+nac7@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:19:41 -0500 Subject: [PATCH 2/2] docs: fix stale filename in docstring, drop redundant comments (greptile) --- .../prompt_portability_check/check_before_provider_swap.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cookbook/prompt_portability_check/check_before_provider_swap.py b/cookbook/prompt_portability_check/check_before_provider_swap.py index 813e9bda531..fa24f6168dd 100644 --- a/cookbook/prompt_portability_check/check_before_provider_swap.py +++ b/cookbook/prompt_portability_check/check_before_provider_swap.py @@ -22,7 +22,7 @@ Install: pip install prompt-portability litellm Run: - python prompt_portability_litellm_check.py + python check_before_provider_swap.py """ from __future__ import annotations @@ -31,8 +31,6 @@ import litellm from llm_prompt_lint.linter import lint from llm_prompt_lint.parsers import detect_and_parse -# A request written and tested against OpenAI, about to be pointed at a -# different provider via LiteLLM's `model=` switch. request = { "model": "gpt-4o", "messages": [ @@ -61,8 +59,6 @@ if report.findings: "the new provider." ) -# LiteLLM's own call succeeds regardless -- it standardizes the transport, -# not the content, which is exactly the gap prompt-portability closes. response = litellm.completion( model="gpt-4o", messages=request["messages"],