diff --git a/.github/workflows/monitor_databricks_pricing.yml b/.github/workflows/monitor_databricks_pricing.yml new file mode 100644 index 00000000000..884d8169435 --- /dev/null +++ b/.github/workflows/monitor_databricks_pricing.yml @@ -0,0 +1,88 @@ +name: Monitor Databricks Pricing + +# Daily monitor of the Databricks Foundation Model Serving pricing pages. +# Before PR #38950 merges this workflow only lives on the leecoder/litellm +# fork (manual dispatch); after the merge it runs on BerriAI/litellm's cron. +# PRs are always created in the repository the workflow runs in, so the +# built-in GITHUB_TOKEN is sufficient - no PAT or secret required. +on: + schedule: + - cron: "0 2 * * *" # daily 02:00 UTC + workflow_dispatch: + # external trigger: POST /repos///dispatches {"event_type":"dbx-monitor"} + repository_dispatch: + types: [dbx-monitor] + +permissions: + contents: write + pull-requests: write + +env: + UPSTREAM_REPO: BerriAI/litellm + UPSTREAM_BASE: litellm_internal_staging + # branch that carries the monitor script until PR #38950 lands upstream + SCRIPT_REF: feat/dbx-pricing-monitor-clean + +jobs: + monitor-db-pricing: + # the merged cron fires on both repos; upstream owns the schedule so the + # two never race on the same base. manual triggers run on either repo. + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + fetch-depth: 0 + repository: ${{ env.UPSTREAM_REPO }} + ref: ${{ env.UPSTREAM_BASE }} + + - name: Bootstrap monitor script until #38950 lands upstream + run: | + if [ ! -f scripts/monitor_databricks_pricing.py ]; then + mkdir -p scripts + curl -fsSL "https://raw.githubusercontent.com/leecoder/litellm/${SCRIPT_REF}/scripts/monitor_databricks_pricing.py" \ + -o scripts/monitor_databricks_pricing.py + fi + + - name: Set up Python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: "3.12" + + - name: Run monitor + id: monitor + shell: bash + run: | + set -o pipefail + python scripts/monitor_databricks_pricing.py | tee /tmp/monitor_out.txt + if grep -q '^NO_CHANGE' /tmp/monitor_out.txt; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Open PR if changed + if: steps.monitor.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN_FOR_PUSH: ${{ github.token }} + run: | + BRANCH="monitor-dbx-pricing-$(date +'%Y-%m-%d')" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH" + git add model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + git commit -m "chore(model_prices): refresh Databricks Foundation Model Serving rates - automated monitor detected changed DBU rates on the Databricks pricing pages; refreshed the mapped databricks/* entries." + git push "https://x-access-token:${GITHUB_TOKEN_FOR_PUSH}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:refs/heads/$BRANCH" + { + cat /tmp/dbx_monitor_pr_body.md + echo + echo "---" + echo "Auto-generated by the Databricks pricing monitor. \`PROMO_SKIPPED\`/\`REVIEW\`/\`MISSING_FROM_PAGE\` lines need human attention; \`UPDATED\` lines were applied automatically." + } > /tmp/dbx_pr_body_final.md + gh pr create --repo "$GITHUB_REPOSITORY" \ + --base "$UPSTREAM_BASE" \ + --head "$BRANCH" \ + --title "chore(model_prices): refresh Databricks Foundation Model Serving rates" \ + --body-file /tmp/dbx_pr_body_final.md diff --git a/scripts/monitor_databricks_pricing.py b/scripts/monitor_databricks_pricing.py new file mode 100644 index 00000000000..1a06dcd618f --- /dev/null +++ b/scripts/monitor_databricks_pricing.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""Monitor Databricks Foundation Model Serving pricing and update LiteLLM's +model_prices_and_context_window.json + packaged backup when rates change. + +Triggered daily by .github/workflows/monitor_databricks_pricing.yml. + +Behavior (registry-wide, not a fixed model list): +- Fetches both official pricing pages (open FMS + proprietary FMS) and parses + every "Standard Pay Per Token" DBU table. Numeric columns are mapped by + header text (Input / Output / Cache read / Cache write), so column + reordering is safe. Priority/Batch/Provisioned tables are ignored. +- Compares page rates against every mapped registry entry and reports: + UPDATED - entry stores the published list rate and the page + value moved: rate fields refreshed in place. + PROMO_SKIPPED - entry stores the promotional rate (page list x 0.8); + left untouched, page value reported for a human. + PROMO_ON_PAGE - page displays a promotional price for an entry + storing the list rate; left untouched for a human. + REVIEW - stored rate matches neither pattern; manual check. + RATES_AVAILABLE - page publishes rates for an entry that has none. + NOT_IN_REGISTRY - page lists a mapped model the registry lacks. + UNMAPPED_PAGE_MODEL- page lists a model with no mapping (new model?). + MISSING_FROM_PAGE - entry priced from these pages is no longer listed. +- Long-context tier rows and per-modality sub-rows (image/audio tokens) are + skipped; only rate fields are touched, all other entry metadata is kept. +- If any entry was UPDATED, both JSON files are rewritten and a report is + written to /tmp/dbx_monitor_pr_body.md for the workflow's PR body. + Otherwise "NO_CHANGE" is printed so the workflow skips PR creation. + Exit code is always 0. +""" + +import json +import re +import sys +import urllib.request +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[1] +MAIN_MAP = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_MAP = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" +PR_BODY_PATH = Path("/tmp/dbx_monitor_pr_body.md") + +DBU_TO_USD = 0.07 +REL_TOL = 1.5e-3 # page rates carry 3-decimal rounding noise +PROMO_RATIO = 0.8 +PROMO_TOL = 0.02 + +FMS_PAGE = "https://www.databricks.com/product/pricing/foundation-model-serving" +PROPRIETARY_PAGE = ( + "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" +) +PAGES = (FMS_PAGE, PROPRIETARY_PAGE) + +# Header names (lowercased) of the numeric columns we consume. "cache write +# (1hr)" tiers are not modeled by LiteLLM and are dropped. +NUMERIC_HEADERS = ("input", "output", "cache read", "cache write") +DROPPED_HEADERS = ("cache write (1hr)",) + +# Rowspan continuation rows on the proprietary page carry tier/modality +# qualifiers in the first cell; they never start a new model row. +QUALIFIER_LABELS = ("long context", "image tokens", "audio tokens", "in-geo", "global") +USABLE_QUALIFIERS = ("", "short context", "text tokens") + +# Page label (markup stripped, lowercased) -> registry keys under +# "databricks/". Labels missing here are reported as UNMAPPED_PAGE_MODEL +# (new-model signal); keys missing from the registry are reported as +# NOT_IN_REGISTRY. +LABEL_TO_KEYS: Dict[str, List[str]] = { + # Open Foundation Model Serving page + "kimi k3": ["databricks-kimi-k3"], + "glm-5.2, 5.3": ["databricks-glm-5-2", "databricks-glm-5-3"], + "deepseek v4 pro": ["databricks-deepseek-v4-pro-0813"], + "inkling": ["databricks-inkling"], + "glm-5.3 flash": ["databricks-glm-5-3-flash"], + "deepseek v4 flash": ["databricks-deepseek-v4-flash-0731"], + "qwen 3.5 122b": ["databricks-qwen35-122b-a10b"], + "llama 4 maverick": ["databricks-llama-4-maverick"], + "llama 3.3 70b": ["databricks-meta-llama-3-3-70b-instruct"], + "qwen 3 80b instruct": ["databricks-qwen3-next-80b-a3b-instruct"], + "gpt-oss-120b": ["databricks-gpt-oss-120b"], + "gemma 3 12b": ["databricks-gemma-3-12b"], + "llama 3.1 8b": ["databricks-meta-llama-3-1-8b-instruct"], + "gpt-oss-20b": ["databricks-gpt-oss-20b"], + "gte": ["databricks-gte-large-en"], + "bge large": ["databricks-bge-large-en"], + "qwen 3 0.6b embedding": ["databricks-qwen3-embedding-0-6b"], + # Proprietary Foundation Model Serving page + "gpt-5.6 sol": ["databricks-gpt-5-6-sol"], + "gpt-5.6 terra": ["databricks-gpt-5-6-terra"], + "gpt-5.6 luna": ["databricks-gpt-5-6-luna"], + "gpt-5.5": ["databricks-gpt-5-5"], + "gpt-5.4 pro, 5.5 pro": ["databricks-gpt-5-5-pro"], + "gpt-5.4": ["databricks-gpt-5-4"], + "gpt-5.4 mini": ["databricks-gpt-5-4-mini"], + "gpt-5.4 nano": ["databricks-gpt-5-4-nano"], + "gpt-5.2 codex, 5.3 codex": ["databricks-gpt-5-2-codex", "databricks-gpt-5-3-codex"], + "gpt-5.2": ["databricks-gpt-5-2"], + "gpt-5, 5.1": ["databricks-gpt-5", "databricks-gpt-5-1"], + "gpt-5.1 codex max": ["databricks-gpt-5-1-codex-max"], + "gpt-5.1 codex mini": ["databricks-gpt-5-1-codex-mini"], + "gpt-5 mini": ["databricks-gpt-5-mini"], + "gpt-5 nano": ["databricks-gpt-5-nano"], + "claude fable 5.1": ["databricks-claude-fable-5-1"], + "claude fable 5": ["databricks-claude-fable-5"], + "claude opus 4.5, 4.6, 4.7, 4.8, 5": [ + "databricks-claude-opus-4-5", + "databricks-claude-opus-4-6", + "databricks-claude-opus-4-7", + "databricks-claude-opus-4-8", + "databricks-claude-opus-5", + ], + "claude opus 4, 4.1": ["databricks-claude-opus-4", "databricks-claude-opus-4-1"], + "claude sonnet 5": ["databricks-claude-sonnet-5"], + "claude sonnet 4.5, 4.6": [ + "databricks-claude-sonnet-4-5", + "databricks-claude-sonnet-4-6", + ], + "claude sonnet 4": ["databricks-claude-sonnet-4", "databricks-claude-sonnet-4-1"], + "claude haiku 4.5": ["databricks-claude-haiku-4-5"], + "gemini 3.0 pro, 3.1 pro": ["databricks-gemini-3-1-pro", "databricks-gemini-3-pro"], + "gemini 2.5 pro": ["databricks-gemini-2-5-pro"], + "gemini 3.7 flash, 3.8 flash": [ + "databricks-gemini-3-7-flash", + "databricks-gemini-3-8-flash", + ], + "gemini 3.6 flash": ["databricks-gemini-3-6-flash"], + "gemini 3.5 flash": ["databricks-gemini-3-5-flash"], + "gemini 3.0 flash": ["databricks-gemini-3-flash"], + "gemini 2.5 flash": ["databricks-gemini-2-5-flash"], + "gemini 3.5 flash lite": ["databricks-gemini-3-5-flash-lite"], + "gemini 3.1 flash lite": ["databricks-gemini-3-1-flash-lite"], + "gemini 3 pro image": ["databricks-gemini-3-pro-image"], + "gemini 3.1 flash image": ["databricks-gemini-3-1-flash-image"], + "grok 4.6": ["databricks-grok-4-6"], +} + +# Page labels with no text-token registry mapping today: image-generation +# models bill per-image/vendor pass-through, and Kimi K2.7 is not in the +# supported-models docs. Remove from here once entries exist. +IGNORED_LABELS = ( + "kimi k2.7", + "gpt image 1", + "gpt image 1 mini", + "gpt image 1.5", + "gpt image 2", + "gemini 3.1 flash lite image", +) + +# Rate fields an entry stores for a page-published column. +FIELD_BY_HEADER = { + "input": ("input_cost_per_token", "input_dbu_cost_per_token"), + "output": ("output_cost_per_token", "output_dbu_cost_per_token"), + "cache read": ("cache_read_input_token_cost", None), + "cache write": ("cache_creation_input_token_cost", None), +} + + +def fetch(url: str, max_bytes: int = 5_000_000) -> str: + """Fetch page HTML; returns text. Raises on non-200.""" + req = urllib.request.Request(url, headers={"User-Agent": "litellm-price-monitor/1.0"}) + with urllib.request.urlopen(req, timeout=60) as resp: + if resp.status != 200: + raise RuntimeError("HTTP {} fetching {}".format(resp.status, url)) + return resp.read(max_bytes + 1).decode("utf-8", errors="replace") + + +def _cell_text(html: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", html)).strip() + + +def _parse_number(cell: str) -> Optional[float]: + txt = cell.replace(",", "").strip() + if re.fullmatch(r"\d+(?:\.\d+)?", txt): + return float(txt) + return None + + +def _is_placeholder(cell: str) -> bool: + """True for dash/n/a cells meaning 'not published' - neither number nor label.""" + return cell.strip() in ("-", "—", "–", "n/a", "N/A", "na") + + +def parse_standard_pp_token_tables( + html: str, +) -> Dict[str, List[Tuple[str, Tuple[Optional[float], ...], Tuple[str, ...]]]]: + """Extract rows of every "Standard Pay Per Token" table. + + Returns label -> list of (qualifier, numbers, numeric_cols), where numbers + align with that table's numeric header order. + """ + parsed: Dict[str, List[Tuple[str, Tuple[Optional[float], ...], Tuple[str, ...]]]] = {} + for tm in re.finditer(r"]*>(.*?)", html, re.S): + table = tm.group(1) + head = re.search(r"(.*?)", table, re.S) + if head is None: + continue + header_texts = [ + _cell_text(th) for th in re.findall(r"]*>(.*?)", head.group(1), re.S) + ] + if not any("Standard Pay Per Token" in h for h in header_texts): + continue + numeric_cols = tuple( + h.lower() + for h in header_texts + if h.lower() in NUMERIC_HEADERS and h.lower() not in DROPPED_HEADERS + ) + for rm in re.finditer(r"(.*?)", table, re.S): + cells = [ + _cell_text(c) + for c in re.findall(r"]*>(.*?)", rm.group(1), re.S) + ] + if not cells: + continue + label = re.sub(r"[\*⌖]+", "", cells[0]).strip().lower() + if not label or label == "model" or label in header_texts: + continue + if label in QUALIFIER_LABELS: + continue # rowspan continuation row (tier / modality rate) + qualifier = "" + numbers: List[Optional[float]] = [] + for cell in cells[1:]: + if _is_placeholder(cell): + numbers.append(None) + continue + num = _parse_number(cell) + if num is not None: + numbers.append(num) + elif cell and not qualifier: + qualifier = cell.lower() + if len(numbers) < 1 or qualifier not in USABLE_QUALIFIERS: + continue # label-only rows / single-metric or modality rows + numbers.extend([None] * (len(numeric_cols) - len(numbers))) + parsed.setdefault(label, []).append( + (qualifier, tuple(numbers[: len(numeric_cols)]), numeric_cols) + ) + return parsed + + +def dbu_to_usd(dbu: Optional[float]) -> Optional[float]: + if dbu is None: + return None + return dbu / 1_000_000 * DBU_TO_USD + + +def _approx(a: Optional[float], b: Optional[float]) -> bool: + if a is None or b is None: + return a is None and b is None + return abs(a - b) <= max(abs(b) * REL_TOL, 1e-12) + + +def classify(entry: Dict, page_in: float, page_out: Optional[float]) -> str: + """How the entry's stored rates relate to the page row.""" + stored_in = entry.get("input_cost_per_token") + if stored_in is None: + return "unpriced" + usd_in = dbu_to_usd(page_in) + usd_out = dbu_to_usd(page_out) if page_out is not None else None + if usd_in is None: + return "no-input-column" + if _approx(stored_in, usd_in) and ( + usd_out is None or _approx(entry.get("output_cost_per_token"), usd_out) + ): + return "list" + if usd_in and abs(stored_in / usd_in - PROMO_RATIO) <= PROMO_TOL: + stored_out = entry.get("output_cost_per_token") + if ( + usd_out is None + or stored_out is None + or abs(stored_out / usd_out - PROMO_RATIO) <= PROMO_TOL + ): + return "promo" + if usd_in and abs(usd_in / stored_in - PROMO_RATIO) <= PROMO_TOL: + return "page-promo" + return "mismatch" + + +def _refresh_cache_fields( + entry: Dict, header_values: Dict[str, Optional[float]] +) -> Tuple[bool, List[str]]: + """Sync cache rates with the page; keep n/a conventions tracking input.""" + notes: List[str] = [] + changed = False + input_usd = entry.get("input_cost_per_token") + for header, field in ( + ("cache read", "cache_read_input_token_cost"), + ("cache write", "cache_creation_input_token_cost"), + ): + page_dbu = header_values.get(header) + if page_dbu is not None: + target = dbu_to_usd(page_dbu) + note = "{} DBU {}".format(field, page_dbu) + elif input_usd and ( + entry.get(field) is None or _approx(entry.get(field), input_usd) + ): + # not on the page: bill cache at input; custom conventions (gemini 0.1x) fall through + target = input_usd + note = "{}=input (not published)".format(field) + else: + continue + if target is not None and not _approx(entry.get(field), target): + notes.append("{}: {} -> {}".format(note, entry.get(field), target)) + entry[field] = target + changed = True + return changed, notes + + +def update_entry( + entry: Dict, + key: str, + numbers: Tuple[Optional[float], ...], + numeric_cols: Tuple[str, ...], + page_url: str, +) -> Tuple[bool, str]: + """Apply page rates to one registry entry. Returns (changed, report line).""" + header_values = dict(zip(numeric_cols, numbers)) + page_in = header_values.get("input") + page_out = header_values.get("output") + if page_in is None: + return False, "REVIEW {}: page row has no input column".format(key) + status = classify(entry, page_in, page_out) + if status == "promo": + return False, ( + "PROMO_SKIPPED {}: entry stores the promotional rate; page list input={} output={}".format( + key, page_in, page_out + ) + ) + if status == "page-promo": + return False, ( + "PROMO_ON_PAGE {}: page shows promotional pricing (input={}); " + "entry keeps list rate {}".format( + key, page_in, entry.get("input_cost_per_token") + ) + ) + if status == "mismatch": + stored = entry.get("input_cost_per_token") + ratio = stored / dbu_to_usd(page_in) if stored and page_in else 0 + return False, ( + "REVIEW {}: stored input={} vs page list input={} DBU (ratio {:.3f}) " + "- manual check".format(key, stored, page_in, ratio) + ) + changed = False + detail: List[str] = [] + for header in ("input", "output"): + page_dbu = header_values.get(header) + if page_dbu is None: + continue # e.g. embeddings: output not published - preserve stored + usd_field, dbu_field = FIELD_BY_HEADER[header] + target_usd = dbu_to_usd(page_dbu) + if not _approx(entry.get(usd_field), target_usd): + detail.append("{} {}->{}".format(header, entry.get(usd_field), target_usd)) + entry[usd_field] = target_usd + changed = True + if dbu_field is not None: + target_dbu = page_dbu / 1_000_000 + if not _approx(entry.get(dbu_field), target_dbu): + entry[dbu_field] = target_dbu + changed = True + cache_changed, cache_notes = _refresh_cache_fields(entry, header_values) + detail.extend(cache_notes) + changed = changed or cache_changed + if changed and entry.get("source") != page_url: + entry["source"] = page_url + if changed: + return True, "UPDATED {}: {}".format(key, "; ".join(detail)) + return False, "" + + +def main() -> int: + pages = [(url, fetch(url)) for url in PAGES] + + with MAIN_MAP.open() as f: + main_data = json.load(f) + with BACKUP_MAP.open() as f: + backup_data = json.load(f) + + tracked_keys: set = set() + changed = False + report_lines: List[str] = [] + + for page_url, html in pages: + tables = parse_standard_pp_token_tables(html) + for label, rows in sorted(tables.items()): + if label in IGNORED_LABELS: + continue + keys = LABEL_TO_KEYS.get(label) + qualifier, numbers, numeric_cols = rows[0] + if keys is None: + report_lines.append( + "UNMAPPED_PAGE_MODEL: '{}' on {} lists input={} output={} DBU/1M " + "- add a mapping or a registry entry".format( + label, Path(page_url).name, numbers[0], numbers[1] + ) + ) + continue + for key in keys: + full_key = "databricks/" + key + tracked_keys.add(full_key) + entry = main_data.get(full_key) + if entry is None: + report_lines.append( + "NOT_IN_REGISTRY {}: '{}' on {} lists rates ({} DBU in) " + "but the registry has no entry".format( + full_key, label, Path(page_url).name, numbers[0] + ) + ) + continue + if not entry.get("input_cost_per_token"): + report_lines.append( + "RATES_AVAILABLE {}: '{}' now publishes rates ({} DBU in / {} out) " + "- entry currently unpriced".format( + full_key, label, numbers[0], numbers[1] + ) + ) + continue + was_changed, line = update_entry( + entry, full_key, numbers, numeric_cols, page_url + ) + changed = changed or was_changed + if line: + report_lines.append(line) + + # Entries priced from these pages that vanished from them: retirement signal. + for full_key, entry in main_data.items(): + if not full_key.startswith("databricks/") or full_key in tracked_keys: + continue + if entry.get("input_cost_per_token") and entry.get("source") in PAGES: + report_lines.append( + "MISSING_FROM_PAGE {}: priced entry no longer on the pricing pages " + "- check for retirement".format(full_key) + ) + + if not changed: + sys.stdout.write("NO_CHANGE\n") + for line in report_lines: + sys.stdout.write(line + "\n") + return 0 + + for full_key, entry in main_data.items(): + if full_key.startswith("databricks/") and full_key in backup_data: + backup_data[full_key] = entry + + with MAIN_MAP.open("w") as f: + json.dump(main_data, f, indent=4) + f.write("\n") + with BACKUP_MAP.open("w") as f: + json.dump(backup_data, f, indent=4) + f.write("\n") + + body = [ + "Automated daily check of the Databricks Foundation Model Serving pricing pages", + "([open](https://www.databricks.com/product/pricing/foundation-model-serving),", + "[proprietary](https://www.databricks.com/product/pricing/proprietary-foundation-model-serving))", + "detected published-rate changes. Rate fields refreshed in place; metadata untouched.", + "", + "## Monitor report", + "", + "```", + ] + body.extend(report_lines) + body.append("```") + PR_BODY_PATH.write_text("\n".join(body) + "\n") + + sys.stdout.write("CHANGED\n") + for line in report_lines: + sys.stdout.write(line + "\n") + sys.stdout.write( + "WROTE updated model map, backup and PR body to {}\n".format(PR_BODY_PATH) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())