mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
99 lines
3.7 KiB
Python
99 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import jsonschema
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).parents[2]
|
|
GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py"
|
|
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
|
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
|
|
|
|
|
def build_validator(schema: dict) -> jsonschema.Draft202012Validator:
|
|
return jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
|
|
|
|
|
|
def load_generator():
|
|
spec = importlib.util.spec_from_file_location("generate_model_prices_schema", GENERATOR_PATH)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def committed_schema() -> dict:
|
|
return json.loads(SCHEMA_PATH.read_text())
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def prices() -> dict:
|
|
return json.loads(PRICES_PATH.read_text())
|
|
|
|
|
|
def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict):
|
|
generator = load_generator()
|
|
regenerated = json.loads(generator.render(generator.build_schema(prices)))
|
|
assert regenerated == committed_schema, (
|
|
"model_prices_and_context_window.schema.json is out of sync; "
|
|
"run `python ci_cd/generate_model_prices_schema.py` and commit the result"
|
|
)
|
|
|
|
|
|
def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict):
|
|
validator = build_validator(committed_schema)
|
|
errors = [
|
|
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
|
|
for error in validator.iter_errors(prices)
|
|
]
|
|
assert errors == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"entry",
|
|
[
|
|
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": "0.01"},
|
|
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": -1},
|
|
{"litellm_provider": "openai", "mode": "not_a_real_mode"},
|
|
{"mode": "chat"},
|
|
{"litellm_provider": "openai", "deprecation_date": "June 2026"},
|
|
{"litellm_provider": "openai", "deprecation_date": "2026-99-99"},
|
|
{"litellm_provider": "openai", "deprecation_date": "2026-13-01"},
|
|
{"litellm_provider": "openai", "deprecation_date": "2026-01-32"},
|
|
{"litellm_provider": "openai", "deprecation_date": "2026-01-00"},
|
|
{"litellm_provider": "openai", "deprecation_date": "2026-02-31"},
|
|
{"litellm_provider": "openai", "supported_modalities": ["smell"]},
|
|
{"litellm_provider": "openai", "supports_vision": "yes"},
|
|
{"litellm_provider": "openai", "max_tokens": 8191.5},
|
|
{"litellm_provider": "openai", "tiered_pricing": [{"unknown_tier_field": 1}]},
|
|
],
|
|
ids=[
|
|
"cost_as_string",
|
|
"negative_cost",
|
|
"unknown_mode",
|
|
"missing_provider",
|
|
"non_iso_deprecation_date",
|
|
"impossible_month_and_day",
|
|
"month_out_of_range",
|
|
"day_out_of_range",
|
|
"day_zero",
|
|
"calendar_impossible_day",
|
|
"unknown_modality",
|
|
"boolean_flag_as_string",
|
|
"fractional_max_tokens",
|
|
"unknown_tiered_pricing_field",
|
|
],
|
|
)
|
|
def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict):
|
|
validator = build_validator(committed_schema)
|
|
assert not validator.is_valid({"some-model": entry})
|
|
|
|
|
|
def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict):
|
|
validator = build_validator(committed_schema)
|
|
assert validator.is_valid({"some-model": {"litellm_provider": "openai"}})
|
|
assert validator.is_valid({"some-model": {"litellm_provider": "openai", "brand_new_field": {"nested": True}}})
|