mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
test(ocr): record provider fixtures in the migration harness
This commit is contained in:
parent
2c30fe16b0
commit
8c180afd4b
146 changed files with 54309 additions and 27 deletions
|
|
@ -23,7 +23,7 @@ def _positive_int(value: str) -> int:
|
|||
|
||||
def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs:
|
||||
parser: Final = argparse.ArgumentParser()
|
||||
parser.add_argument("--concurrency", type=_positive_int, default=4)
|
||||
parser.add_argument("--concurrency", type=_positive_int, default=2)
|
||||
parser.add_argument("--examples", type=_positive_int, default=4)
|
||||
parser.add_argument("--fixture-dir", type=Path)
|
||||
namespace: Final = parser.parse_args(argv)
|
||||
|
|
|
|||
|
|
@ -132,6 +132,9 @@ def _record_job(job: RecordingJob[InputT], case_type: type[CaseT]) -> RecordedFi
|
|||
job.case_input,
|
||||
job.invocation.execute,
|
||||
)
|
||||
status: Final = interactions[-1].response.status_code
|
||||
if status in {408, 429} or status >= 500:
|
||||
raise RuntimeError(f"Upstream returned transient HTTP {status}; rerun recording to retry")
|
||||
case: Final = case_type.model_validate(
|
||||
{
|
||||
"litellm_input": job.case_input,
|
||||
|
|
|
|||
|
|
@ -44,8 +44,9 @@ class _ParityCase(BaseModel):
|
|||
class _Upstream(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, status: int = 200) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _UpstreamHandler)
|
||||
self.response_status: Final = status
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
|
|
@ -59,7 +60,9 @@ class _UpstreamHandler(BaseHTTPRequestHandler):
|
|||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
self.rfile.read(length)
|
||||
body: Final = b"{}"
|
||||
self.send_response(200)
|
||||
server: Final = self.server
|
||||
assert isinstance(server, _Upstream)
|
||||
self.send_response(server.response_status)
|
||||
self.send_header("content-type", "application/json")
|
||||
self.send_header("content-length", str(len(body)))
|
||||
self.end_headers()
|
||||
|
|
@ -70,8 +73,8 @@ class _UpstreamHandler(BaseHTTPRequestHandler):
|
|||
|
||||
|
||||
@contextmanager
|
||||
def _upstream() -> Generator[_Upstream]:
|
||||
server: Final = _Upstream()
|
||||
def _upstream(status: int = 200) -> Generator[_Upstream]:
|
||||
server: Final = _Upstream(status)
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
|
|
@ -189,3 +192,26 @@ def test_failure_does_not_stop_independent_recordings(tmp_path: Path) -> None:
|
|||
assert len(summary.failed) == 1
|
||||
assert summary.failed[0].target_name == "stale"
|
||||
assert summary.exit_code == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", (408, 429, 500, 503))
|
||||
def test_transient_response_is_not_cached_and_can_be_retried(tmp_path: Path, status: int) -> None:
|
||||
case_input: Final = _FixtureInput(identifier="retry")
|
||||
with _upstream(status) as upstream:
|
||||
target: Final = _target("retry", upstream.url, case_input, _Invocation())
|
||||
failed: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase)
|
||||
assert failed.exit_code == 1
|
||||
assert not fixture_path(tmp_path / "retry", case_input).exists()
|
||||
with _upstream() as healthy_upstream:
|
||||
healthy_target: Final = _target("retry", healthy_upstream.url, case_input, _Invocation())
|
||||
retried: Final = record_fixtures((healthy_target,), tmp_path, 1, 1, _ParityCase)
|
||||
assert retried.exit_code == 0
|
||||
assert len(retried.recorded) == 1
|
||||
|
||||
|
||||
def test_provider_rejected_response_can_be_recorded(tmp_path: Path) -> None:
|
||||
with _upstream(400) as upstream:
|
||||
target: Final = _target("rejected", upstream.url, _FixtureInput(identifier="invalid"), _Invocation())
|
||||
summary: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase)
|
||||
assert summary.exit_code == 0
|
||||
assert len(summary.recorded) == 1
|
||||
|
|
|
|||
|
|
@ -10,14 +10,32 @@ The recording command runs four stages:
|
|||
Run it with:
|
||||
|
||||
```shell
|
||||
uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --examples 4 --concurrency 4
|
||||
uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --examples 1000
|
||||
```
|
||||
|
||||
`--concurrency` caps provider calls across all targets. Independent jobs finish after a failure, then the command exits
|
||||
nonzero if any job failed
|
||||
`--concurrency` defaults to 2 and caps active recording jobs across all targets. Increase it explicitly when provider
|
||||
quotas permit. Concurrency limits do not guarantee a request-per-minute quota; HTTP 408, 429, and 5xx responses fail
|
||||
recording without being saved. Rerunning retries missing fixtures and reuses successful recordings
|
||||
|
||||
New recordings are VCR YAML cassettes. The committed corpus contains 31 migrated cassettes: 18 Mistral, 9 Reducto v3,
|
||||
and 4 Reducto legacy. Their original response bytes, statuses, headers, and recording timestamps are preserved
|
||||
New recordings are VCR YAML cassettes. The corpus retains the original 31 migrated cassettes and adds live recordings
|
||||
for all four providers. Original response bytes, statuses, headers, and recording timestamps are preserved
|
||||
|
||||
For Vertex, authenticate and select a project once:
|
||||
|
||||
```shell
|
||||
gcloud auth login
|
||||
gcloud config set project YOUR_PROJECT_ID
|
||||
```
|
||||
|
||||
The recording command reads the project from `VERTEXAI_PROJECT`, `VERTEX_PROJECT`, or the active gcloud configuration,
|
||||
then gets an OAuth access token with `gcloud auth print-access-token`. Tokens stay in memory and are removed from
|
||||
recorded headers. `VERTEX_AI_ACCESS_TOKEN` or the legacy `VERTEX_AI_API_KEY` can override token lookup. The
|
||||
`VERTEXT_API_KEY` express-mode key is not used as a Bearer token. Mistral defaults to `us-central1`; DeepSeek defaults
|
||||
to the global host and `global` location. `VERTEX_DEEPSEEK_LOCATION` and `VERTEX_DEEPSEEK_API_BASE` override the latter
|
||||
|
||||
Azure accepts `AZURE_KEY` and `AZURE_ENDPOINT` as fallbacks for both Azure OCR contracts. Provider-specific variables
|
||||
take precedence. Set `AZURE_DEPLOYMENT_NAME=mistral-ocr-4-0` to record that deployment instead of the default
|
||||
`mistral-document-ai-2512`. Mistral and Reducto use `MISTRAL_API_KEY` and `REDUCTO_API_KEY`
|
||||
|
||||
To migrate an existing JSON fixture directory locally:
|
||||
|
||||
|
|
@ -34,11 +52,18 @@ OCR strategies generate public `litellm.ocr()` and `litellm.aocr()` inputs. Ever
|
|||
document, optional provider override, and LiteLLM keyword arguments. The fixture-only `contract` literal selects the
|
||||
input schema and is removed before calling the SDK. Strategies never build provider wire payloads
|
||||
|
||||
Each contract has a required corpus containing a baseline and cases for its supported top-level OCR parameters. The
|
||||
Each contract's strategy contains baselines and cases for its supported top-level OCR parameters. The
|
||||
contracts are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, Vertex DeepSeek,
|
||||
Reducto v3, and Reducto legacy. Credentials and endpoints only control target discovery, so a machine records the
|
||||
contracts it has configured and skips the rest
|
||||
|
||||
`--examples 1000` exhausts the current finite strategies: 32 Mistral, 15 Azure Mistral, 17 Azure Document Intelligence,
|
||||
16 Vertex Mistral, 2 Vertex DeepSeek, 55 Reducto v3, and 3 Reducto legacy cases, including fixed rejected inputs.
|
||||
This covers the defined strategy choices, not every possible value accepted by the schemas. The live run recorded
|
||||
139 of these 140 cases. Vertex Mistral's standalone `document_annotation_format` case repeatedly returned HTTP 500
|
||||
and remains pending. Its bounding-box annotation, annotation-prompt, and confidence cases record upstream 404/422
|
||||
rejections; schema acceptance does not imply support by the hosted model
|
||||
|
||||
Reducto fixtures record upload and parse responses. Their parity cases remain non-strict expected failures until the
|
||||
Rust OCR bridge supports Reducto. Azure and Vertex generation paths are unit-tested without credentials in CI, so the
|
||||
committed corpus does not need live recordings for every target
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from typing import Final, Literal, cast
|
|||
|
||||
from hypothesis import strategies as st
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
from pydantic import StrictInt, StrictStr, field_validator
|
||||
from pydantic import StrictInt, StrictStr, TypeAdapter, field_validator
|
||||
|
||||
from ......shared.parity.fixtures.recording import UpstreamEndpoint
|
||||
from .base import OcrDocument, OcrSdkInputBase
|
||||
|
|
@ -21,7 +21,7 @@ from .mistral import (
|
|||
mistral_input_values_strategy,
|
||||
)
|
||||
|
||||
AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512",]
|
||||
AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512", "azure_ai/mistral-ocr-4-0"]
|
||||
AzureMistralFixtureModel = AzureMistralModel | Literal["azure_ai/invalid-ocr-model-for-parity"]
|
||||
AzureDocumentIntelligenceModel = Literal[
|
||||
"azure_ai/doc-intelligence/prebuilt-read",
|
||||
|
|
@ -87,13 +87,16 @@ def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) ->
|
|||
return AzureMistralOcrSdkInput.model_validate({**values, "model": model})
|
||||
|
||||
|
||||
def azure_mistral_input_strategy(inline_image_data_uri: str) -> SearchStrategy[AzureMistralOcrSdkInput]:
|
||||
def azure_mistral_input_strategy(
|
||||
inline_image_data_uri: str,
|
||||
models: tuple[AzureMistralModel, ...] = AZURE_MISTRAL_MODELS,
|
||||
) -> SearchStrategy[AzureMistralOcrSdkInput]:
|
||||
# Foundry's active gateway schema rejects 2512-only controls and
|
||||
# document_annotation_prompt, even though native Mistral accepts them.
|
||||
return st.builds(
|
||||
_azure_mistral_input,
|
||||
values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False),
|
||||
model=st.sampled_from(AZURE_MISTRAL_MODELS),
|
||||
model=st.sampled_from(models),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -171,17 +174,23 @@ def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocument
|
|||
def azure_mistral_recording_targets(
|
||||
environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str
|
||||
) -> tuple[OcrRecordingTarget, ...]:
|
||||
api_key: Final = environ.get("AZURE_AI_API_KEY")
|
||||
base_url: Final = environ.get("AZURE_AI_API_BASE")
|
||||
api_key: Final = environ.get("AZURE_AI_API_KEY") or environ.get("AZURE_KEY")
|
||||
base_url: Final = environ.get("AZURE_AI_API_BASE") or environ.get("AZURE_ENDPOINT")
|
||||
if not api_key or not base_url:
|
||||
return ()
|
||||
deployment: Final = environ.get("AZURE_DEPLOYMENT_NAME")
|
||||
models: Final = (
|
||||
(TypeAdapter(AzureMistralModel).validate_python(f"azure_ai/{deployment.removeprefix('azure_ai/')}"),)
|
||||
if deployment
|
||||
else AZURE_MISTRAL_MODELS
|
||||
)
|
||||
return (
|
||||
OcrRecordingTarget(
|
||||
name="azure-mistral",
|
||||
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
|
||||
strategy=cast(
|
||||
SearchStrategy[OcrSdkInputBase],
|
||||
azure_mistral_input_strategy(inline_image_data_uri),
|
||||
azure_mistral_input_strategy(inline_image_data_uri, models),
|
||||
),
|
||||
invocation=invoke_with_api_key(client, api_key),
|
||||
required_inputs=AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS,
|
||||
|
|
@ -192,8 +201,8 @@ def azure_mistral_recording_targets(
|
|||
def azure_document_intelligence_recording_targets(
|
||||
environ: Mapping[str, str], client: OcrFixtureClient
|
||||
) -> tuple[OcrRecordingTarget, ...]:
|
||||
api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY")
|
||||
base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
|
||||
api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") or environ.get("AZURE_KEY")
|
||||
base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") or environ.get("AZURE_ENDPOINT")
|
||||
if not api_key or not base_url:
|
||||
return ()
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from collections.abc import Callable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -8,6 +10,35 @@ FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
|||
DEFAULT_FIXTURE_DIRECTORY: Final = Path(__file__).with_name("data")
|
||||
|
||||
|
||||
def read_gcloud(arguments: tuple[str, ...]) -> str:
|
||||
try:
|
||||
result: Final = subprocess.run(("gcloud", *arguments), capture_output=True, text=True, timeout=45, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return ""
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def recording_environment(
|
||||
environ: Mapping[str, str],
|
||||
command_reader: Callable[[tuple[str, ...]], str] = read_gcloud,
|
||||
) -> Mapping[str, str]:
|
||||
project: Final = (
|
||||
environ.get("VERTEXAI_PROJECT")
|
||||
or environ.get("VERTEX_PROJECT")
|
||||
or command_reader(("config", "get-value", "project"))
|
||||
)
|
||||
if not project or project == "(unset)":
|
||||
return environ
|
||||
token: Final = (
|
||||
environ.get("VERTEX_AI_ACCESS_TOKEN")
|
||||
or environ.get("VERTEX_AI_API_KEY")
|
||||
or command_reader(("auth", "print-access-token"))
|
||||
)
|
||||
if not token:
|
||||
raise SystemExit("Vertex OCR needs OAuth credentials. Run `gcloud auth login` or set VERTEX_AI_ACCESS_TOKEN")
|
||||
return {**environ, "VERTEXAI_PROJECT": project, "VERTEX_AI_API_KEY": token}
|
||||
|
||||
|
||||
def configured_fixture_directory() -> Path:
|
||||
configured: Final = os.environ.get(FIXTURE_DIR_ENV)
|
||||
return Path(configured).expanduser() if configured is not None else DEFAULT_FIXTURE_DIRECTORY
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,116 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: '{"urlSource":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: POST
|
||||
uri: http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30
|
||||
response:
|
||||
body:
|
||||
string: ''
|
||||
headers:
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:36:01 GMT
|
||||
Operation-Location:
|
||||
- http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout/analyzeResults/42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a?api-version=2024-11-30
|
||||
Server:
|
||||
- istio-envoy
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
apim-request-id:
|
||||
- 42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a
|
||||
x-envoy-upstream-service-time:
|
||||
- '670'
|
||||
x-ms-region:
|
||||
- West US 3
|
||||
status:
|
||||
code: 202
|
||||
message: ''
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: GET
|
||||
uri: http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout/analyzeResults/42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a?api-version=2024-11-30
|
||||
response:
|
||||
body:
|
||||
string: '{"status":"running","createdDateTime":"2026-09-02T22:36:01Z","lastUpdatedDateTime":"2026-09-02T22:36:01Z"}'
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json; charset=utf-8
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:36:01 GMT
|
||||
Retry-After:
|
||||
- '4'
|
||||
Server:
|
||||
- istio-envoy
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
apim-request-id:
|
||||
- 011f699b-8d28-4ecb-917b-24cd42ffee29
|
||||
x-envoy-upstream-service-time:
|
||||
- '12'
|
||||
x-ms-region:
|
||||
- West US 3
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: GET
|
||||
uri: http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout/analyzeResults/42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a?api-version=2024-11-30
|
||||
response:
|
||||
body:
|
||||
string: '{"status":"succeeded","createdDateTime":"2026-09-02T22:36:01Z","lastUpdatedDateTime":"2026-09-02T22:36:02Z","analyzeResult":{"apiVersion":"2024-11-30","modelId":"prebuilt-layout","stringIndexType":"textElements","content":"invoice
|
||||
123","pages":[{"pageNumber":1,"angle":-0.12780000269412994,"width":800,"height":300,"unit":"pixel","words":[{"content":"invoice","polygon":[338,138,418,139,418,160,338,160],"confidence":0.994,"span":{"offset":0,"length":7}},{"content":"123","polygon":[424,139,461,138,461,160,424,160],"confidence":0.999,"span":{"offset":8,"length":3}}],"lines":[{"content":"invoice
|
||||
123","polygon":[338,138,460,138,461,160,338,160],"spans":[{"offset":0,"length":11}]}],"spans":[{"offset":0,"length":11}]}],"tables":[],"paragraphs":[{"spans":[{"offset":0,"length":11}],"boundingRegions":[{"pageNumber":1,"polygon":[338,138,460,138,461,160,338,160]}],"role":"title","content":"invoice
|
||||
123"}],"styles":[],"contentFormat":"text","sections":[{"spans":[{"offset":0,"length":11}],"elements":["/paragraphs/0"]}]}}'
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json; charset=utf-8
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:36:06 GMT
|
||||
Server:
|
||||
- istio-envoy
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
apim-request-id:
|
||||
- 8940c1a9-9c86-4188-86bf-8f34bb8fb537
|
||||
x-envoy-upstream-service-time:
|
||||
- '32'
|
||||
x-ms-region:
|
||||
- West US 3
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
recorded_at: '2026-09-02T22:36:06.585674+00:00'
|
||||
ttl_seconds: 0
|
||||
version: 1
|
||||
x-litellm:
|
||||
case:
|
||||
litellm_input:
|
||||
document:
|
||||
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
|
||||
type: image_url
|
||||
model: azure_ai/doc-intelligence/prebuilt-layout
|
||||
request_source: recorded
|
||||
schema_version: 1
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,92 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: '{"model":"mistral-ocr-4-0","document":{"type":"image_url","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC"}}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: POST
|
||||
uri: http://parity-provider.invalid/providers/mistral/azure/ocr
|
||||
response:
|
||||
body:
|
||||
string: '{"pages":[{"index":0,"markdown":"$$\\text{DOC } 123$$","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":80,"width":320},"confidence_scores":null,"blocks":[{"top_left_x":0,"top_left_y":0,"bottom_right_x":320,"bottom_right_y":80,"content":"$$\\text{DOC
|
||||
} 123$$","type":"equation"}]}],"model":"mistral-ocr-4-0","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":438}}'
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:38:07 GMT
|
||||
Server:
|
||||
- istio-envoy
|
||||
Strict-Transport-Security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
apim-request-id:
|
||||
- 3795070f-a4a7-4ad6-b31d-fe51fdcf29d4
|
||||
azureai-fe-is-streaming:
|
||||
- 'False'
|
||||
azureai-fe-offer-type:
|
||||
- paygo
|
||||
azureai-processed-tier:
|
||||
- default
|
||||
azureai-requested-tier:
|
||||
- default
|
||||
azureml-destination-deployment:
|
||||
- d20260831221917-7d0e5d7f
|
||||
azureml-destination-endpoint:
|
||||
- mo4-use-h1-oai-oe-nvisoc
|
||||
azureml-destination-model-group:
|
||||
- aoai-balanced
|
||||
azureml-destination-region:
|
||||
- eastus
|
||||
azureml-model-deployment:
|
||||
- d20260831221917-7d0e5d7f
|
||||
azureml-model-group:
|
||||
- aoai-balanced
|
||||
azureml-model-session:
|
||||
- d20260831221917-7d0e5d7f
|
||||
azureml-ms-model-error-comprehensive:
|
||||
- --
|
||||
azureml-served-by-cluster:
|
||||
- hyena-westus3-01
|
||||
x-ms-client-request-id:
|
||||
- 3795070f-a4a7-4ad6-b31d-fe51fdcf29d4
|
||||
x-ms-is-spilled-over:
|
||||
- 'false'
|
||||
x-ms-region:
|
||||
- West US 3
|
||||
x-pages-processed:
|
||||
- '1'
|
||||
x-ratelimit-abusepenalty-active:
|
||||
- 'False'
|
||||
x-ratelimit-key:
|
||||
- mistral-ocr-4-0
|
||||
x-ratelimit-limit-requests:
|
||||
- '10'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '8'
|
||||
x-ratelimit-renewalperiod-requests:
|
||||
- '60'
|
||||
x-ratelimit-reset-requests:
|
||||
- '14'
|
||||
x-request-id:
|
||||
- 7a885df6-7464-4f05-b091-1a154c180886
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
recorded_at: '2026-09-02T22:38:08.503737+00:00'
|
||||
ttl_seconds: 0
|
||||
version: 1
|
||||
x-litellm:
|
||||
case:
|
||||
litellm_input:
|
||||
document:
|
||||
image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC
|
||||
type: image_url
|
||||
model: azure_ai/mistral-ocr-4-0
|
||||
request_source: recorded
|
||||
schema_version: 1
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,66 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC"}}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: POST
|
||||
uri: http://parity-provider.invalid/v1/ocr
|
||||
response:
|
||||
body:
|
||||
string: '{"pages":[{"index":0,"markdown":"$$\\text{DOC } 123$$","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":80,"width":320},"confidence_scores":null,"blocks":[{"top_left_x":0,"top_left_y":0,"bottom_right_x":320,"bottom_right_y":80,"content":"$$\\text{DOC
|
||||
} 123$$","confidence_scores":null,"type":"equation"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":438}}'
|
||||
headers:
|
||||
CF-RAY:
|
||||
- a34ffd7c6b4a2083-SJC
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:35:51 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Strict-Transport-Security:
|
||||
- max-age=15552000; includeSubDomains; preload
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-allow-origin:
|
||||
- '*'
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
mistral-correlation-id:
|
||||
- 01a06443-aa1f-7bb1-8eaf-f992fe2ce362
|
||||
x-envoy-upstream-service-time:
|
||||
- '161'
|
||||
x-kong-proxy-latency:
|
||||
- '12'
|
||||
x-kong-request-id:
|
||||
- 01a06443-aa1f-7bb1-8eaf-f992fe2ce362
|
||||
x-kong-upstream-latency:
|
||||
- '162'
|
||||
x-ratelimit-limit-ocr-pages-minute:
|
||||
- '60'
|
||||
x-ratelimit-ocr-pages-query-cost:
|
||||
- '1'
|
||||
x-ratelimit-remaining-ocr-pages-minute:
|
||||
- '2'
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
recorded_at: '2026-09-02T22:35:51.552358+00:00'
|
||||
ttl_seconds: 0
|
||||
version: 1
|
||||
x-litellm:
|
||||
case:
|
||||
litellm_input:
|
||||
document:
|
||||
image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC
|
||||
type: image_url
|
||||
model: mistral/mistral-ocr-latest
|
||||
request_source: recorded
|
||||
schema_version: 1
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,66 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: '{"model":"mistral-ocr-latest","document":{"type":"document_url","document_url":"https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"}}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: POST
|
||||
uri: http://parity-provider.invalid/v1/ocr
|
||||
response:
|
||||
body:
|
||||
string: '{"pages":[{"index":0,"markdown":"Dummy PDF file","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":87,"height":1018,"width":719},"confidence_scores":null,"blocks":[{"top_left_x":65,"top_left_y":82,"bottom_right_x":222,"bottom_right_y":108,"content":"Dummy
|
||||
PDF file","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":13264}}'
|
||||
headers:
|
||||
CF-RAY:
|
||||
- a34ffd604a71938c-SJC
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:35:47 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Strict-Transport-Security:
|
||||
- max-age=15552000; includeSubDomains; preload
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-allow-origin:
|
||||
- '*'
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
mistral-correlation-id:
|
||||
- 01a06443-987c-7007-876d-bbd07c05013b
|
||||
x-envoy-upstream-service-time:
|
||||
- '233'
|
||||
x-kong-proxy-latency:
|
||||
- '13'
|
||||
x-kong-request-id:
|
||||
- 01a06443-987c-7007-876d-bbd07c05013b
|
||||
x-kong-upstream-latency:
|
||||
- '234'
|
||||
x-ratelimit-limit-ocr-pages-minute:
|
||||
- '60'
|
||||
x-ratelimit-ocr-pages-query-cost:
|
||||
- '1'
|
||||
x-ratelimit-remaining-ocr-pages-minute:
|
||||
- '58'
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
recorded_at: '2026-09-02T22:35:47.022798+00:00'
|
||||
ttl_seconds: 0
|
||||
version: 1
|
||||
x-litellm:
|
||||
case:
|
||||
litellm_input:
|
||||
document:
|
||||
document_url: https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf
|
||||
type: document_url
|
||||
model: mistral/mistral-ocr-latest
|
||||
request_source: recorded
|
||||
schema_version: 1
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,71 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: !!binary |
|
||||
LS1mOTY3MGM3NTExM2E4OTEzMTE4ODllNzk0NjcyMjgxNA0KQ29udGVudC1EaXNwb3NpdGlvbjog
|
||||
Zm9ybS1kYXRhOyBuYW1lPSJmaWxlIjsgZmlsZW5hbWU9ImRvY3VtZW50Ig0KQ29udGVudC1UeXBl
|
||||
OiBpbWFnZS9wbmcNCg0KiVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4
|
||||
nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMw
|
||||
hAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYw
|
||||
AUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAib
|
||||
Pgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxh
|
||||
AoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjC
|
||||
BAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9n
|
||||
B4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCCDQot
|
||||
LWY5NjcwYzc1MTEzYTg5MTMxMTg4OWU3OTQ2NzIyODE0LS0NCg==
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Content-Type:
|
||||
- multipart/form-data; boundary=f9670c75113a891311889e7946722814
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: POST
|
||||
uri: http://parity-provider.invalid/upload
|
||||
response:
|
||||
body:
|
||||
string: '{"file_id":"reducto://d5a7481f-fa60-434a-9627-1a87623bdbba.png","presigned_url":null}'
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:36:47 GMT
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
- request:
|
||||
body: '{"input":"reducto://d5a7481f-fa60-434a-9627-1a87623bdbba.png"}'
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- litellm/1.101.0
|
||||
method: POST
|
||||
uri: http://parity-provider.invalid/parse
|
||||
response:
|
||||
body:
|
||||
string: '{"response_type":"parse","job_id":"65a5b269-f352-4042-bb0c-5a6322a63e10","duration":0.7565219402313232,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/65a5b269-f352-4042-bb0c-5a6322a63e10/5cba5b4a-8303-4cdc-b27a-53492254656b.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260902%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260902T223648Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=81be3175477f2766ea40533a32642598f63e42775494bb6190cdf5a1ef8cc7a1","studio_link":"https://studio.reducto.ai/job/65a5b269-f352-4042-bb0c-5a6322a63e10","usage":{"num_pages":1,"credits":2.0,"credit_breakdown":{"page":1.0,"complex":1.0},"page_billing_breakdown":{"1":["complex","page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"Partial
|
||||
cropped text fragment reading \"00C 123\".","embed":"Partial cropped text
|
||||
fragment reading \"00C 123\".","enriched":null,"enrichment_success":false,"blocks":[{"type":"Figure","bbox":{"left":0.071875,"top":0.1375,"width":0.925,"height":0.725,"page":1,"original_page":1},"content":"Partial
|
||||
cropped text fragment reading \"00C 123\".","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7219684422016144},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}'
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Wed, 02 Sep 2026 22:36:49 GMT
|
||||
status:
|
||||
code: 200
|
||||
message: ''
|
||||
recorded_at: '2026-09-02T22:36:50.166175+00:00'
|
||||
ttl_seconds: 0
|
||||
version: 1
|
||||
x-litellm:
|
||||
case:
|
||||
litellm_input:
|
||||
document:
|
||||
image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC
|
||||
type: image_url
|
||||
model: reducto/parse-v3
|
||||
request_source: recorded
|
||||
schema_version: 1
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue