diff --git a/.github/workflows/ci-quality-gate.yml b/.github/workflows/ci-quality-gate.yml index ff8480bf..e250072f 100644 --- a/.github/workflows/ci-quality-gate.yml +++ b/.github/workflows/ci-quality-gate.yml @@ -114,10 +114,10 @@ jobs: python3 scripts/check_dual_publish.py # Proposed by audit/newgen-2026-06 and never built, which is why retired - # model IDs survived two later audits. Advisory while the existing - # references are cleaned up; flip to blocking once the tree is clean. - - name: Retired model identifier lint (gate G7 — advisory) - continue-on-error: true + # model IDs survived two later audits. The tree is now clean, so this is + # blocking. A deliberate reference needs an entry with a reason in + # scripts/check_model_freshness_allowlist.txt. + - name: Retired model identifier lint (gate G7 — blocking) run: | python3 scripts/check_model_freshness.py --all diff --git a/SKILL_PIPELINE.md b/SKILL_PIPELINE.md index c32e79e3..d50b2ac4 100644 --- a/SKILL_PIPELINE.md +++ b/SKILL_PIPELINE.md @@ -111,7 +111,7 @@ After skill is finalized: ```bash python -m scripts.run_loop \ --eval-set --skill-path \ - --model anthropic/claude-opus-4-6 --max-iterations 5 --verbose + --model anthropic/claude-opus-5 --max-iterations 5 --verbose ``` 4. Apply `best_description` to SKILL.md frontmatter diff --git a/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md b/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md index 17ff8463..8c723d80 100644 --- a/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md +++ b/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md @@ -18,8 +18,8 @@ Per million tokens, USD: | Tier | Example models | Input | Output | |---|---|---|---| -| Frontier-premium | Claude Sonnet 4.6, GPT-4o-tier | $3.00 | $15.00 | -| Frontier-economy | Gemini 2.5 Flash, Claude Haiku 4.5-tier | $1.25 | $5.00 | +| Frontier-premium | Claude Sonnet 5, frontier-mid tier | $3.00 | $15.00 | +| Frontier-economy | Claude Haiku 4.5, small-model tier | $1.25 | $5.00 | | Open-hosted | Llama 3.1 70B / Qwen 2.5 72B via Together, Fireworks, OpenRouter | $0.50 | $1.50 | | Open-economy | 8B-13B-class hosted | $0.10 | $0.30 | @@ -126,7 +126,7 @@ If your utilization is 30% instead of 70%, your effective cost per token roughly ### 2. Capability Drift - Provider updates models silently or with brief notice - Your prompts may produce different outputs after upgrade -- Mitigation: pin model IDs (e.g., `claude-sonnet-4-6` vs `claude-sonnet-latest`) +- Mitigation: pin model IDs (e.g., `claude-sonnet-5` vs `claude-sonnet-latest`) - Cost: regression eval runs on every model swap ### 3. Rate Limits diff --git a/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py b/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py index 04141cef..cb18922f 100644 --- a/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py +++ b/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py @@ -48,8 +48,8 @@ SAMPLE: Dict[str, Any] = { # 2026 API pricing per million tokens, $USD (input / output) API_PRICING = { - "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"}, - "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"}, + "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"}, + "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"}, "open-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"}, } diff --git a/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py b/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py index da0d537f..2b1f4c03 100644 --- a/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py +++ b/c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py @@ -54,8 +54,8 @@ SAMPLE: Dict[str, Any] = { # 2026 API pricing per million tokens, $USD (input / output). These are illustrative; # real pricing changes; rerun this calculator quarterly. API_PRICING = { - "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"}, - "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"}, + "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"}, + "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"}, "open-router-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"}, } diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md index 17ff8463..8c723d80 100644 --- a/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md +++ b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md @@ -18,8 +18,8 @@ Per million tokens, USD: | Tier | Example models | Input | Output | |---|---|---|---| -| Frontier-premium | Claude Sonnet 4.6, GPT-4o-tier | $3.00 | $15.00 | -| Frontier-economy | Gemini 2.5 Flash, Claude Haiku 4.5-tier | $1.25 | $5.00 | +| Frontier-premium | Claude Sonnet 5, frontier-mid tier | $3.00 | $15.00 | +| Frontier-economy | Claude Haiku 4.5, small-model tier | $1.25 | $5.00 | | Open-hosted | Llama 3.1 70B / Qwen 2.5 72B via Together, Fireworks, OpenRouter | $0.50 | $1.50 | | Open-economy | 8B-13B-class hosted | $0.10 | $0.30 | @@ -126,7 +126,7 @@ If your utilization is 30% instead of 70%, your effective cost per token roughly ### 2. Capability Drift - Provider updates models silently or with brief notice - Your prompts may produce different outputs after upgrade -- Mitigation: pin model IDs (e.g., `claude-sonnet-4-6` vs `claude-sonnet-latest`) +- Mitigation: pin model IDs (e.g., `claude-sonnet-5` vs `claude-sonnet-latest`) - Cost: regression eval runs on every model swap ### 3. Rate Limits diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py index 04141cef..cb18922f 100644 --- a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py +++ b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py @@ -48,8 +48,8 @@ SAMPLE: Dict[str, Any] = { # 2026 API pricing per million tokens, $USD (input / output) API_PRICING = { - "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"}, - "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"}, + "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"}, + "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"}, "open-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"}, } diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py index da0d537f..2b1f4c03 100644 --- a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py +++ b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py @@ -54,8 +54,8 @@ SAMPLE: Dict[str, Any] = { # 2026 API pricing per million tokens, $USD (input / output). These are illustrative; # real pricing changes; rerun this calculator quarterly. API_PRICING = { - "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"}, - "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"}, + "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"}, + "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"}, "open-router-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"}, } diff --git a/engineering-team/TEAM_STRUCTURE_GUIDE.md b/engineering-team/TEAM_STRUCTURE_GUIDE.md index e22343fc..136f11e6 100644 --- a/engineering-team/TEAM_STRUCTURE_GUIDE.md +++ b/engineering-team/TEAM_STRUCTURE_GUIDE.md @@ -341,7 +341,7 @@ python scripts/model_deployment_pipeline.py --train --config model_config.yaml # 4. Optimize prompts cd ../senior-prompt-engineer -python scripts/prompt_optimizer.py --model gpt-4 --task classification +python scripts/prompt_optimizer.py classification_prompt.txt --analyze --model claude # 5. Deploy with DevOps cd ../senior-devops diff --git a/engineering-team/skills/senior-ml-engineer/SKILL.md b/engineering-team/skills/senior-ml-engineer/SKILL.md index 59943690..6205ef1f 100644 --- a/engineering-team/skills/senior-ml-engineer/SKILL.md +++ b/engineering-team/skills/senior-ml-engineer/SKILL.md @@ -149,12 +149,23 @@ def call_llm_with_retry(provider: LLMProvider, prompt: str) -> str: ### Cost Management -| Provider | Input Cost | Output Cost | -|----------|------------|-------------| -| GPT-4 | $0.03/1K | $0.06/1K | -| GPT-3.5 | $0.0005/1K | $0.0015/1K | -| Claude 3 Opus | $0.015/1K | $0.075/1K | -| Claude 3 Haiku | $0.00025/1K | $0.00125/1K | +Do not hardcode prices, and do not trust a price table you find in a document +(including this one). Providers reprice several times a year, and a stale +figure produces a confidently wrong business case. + +Work in tiers and look the current numbers up at request time: + +| Tier | Typical use | Relative cost | +|------|-------------|---------------| +| Small | Classification, extraction, routing, short output | 1x baseline | +| Mid | Summarisation, structured output, moderate reasoning | ~10-25x small | +| Large | Multi-step reasoning, code generation, long context | ~50-100x small | + +Read the live rate from your provider's pricing page and pass it in, the way +`engineering-team/skills/senior-prompt-engineer/scripts/prompt_optimizer.py` +takes `--price-per-mtok`. +The ratios between tiers are far more stable than the absolute prices, so +build the model-routing decision on the ratio. --- diff --git a/engineering-team/skills/senior-ml-engineer/references/llm_integration_guide.md b/engineering-team/skills/senior-ml-engineer/references/llm_integration_guide.md index 6e715ab1..0cd90c0e 100644 --- a/engineering-team/skills/senior-ml-engineer/references/llm_integration_guide.md +++ b/engineering-team/skills/senior-ml-engineer/references/llm_integration_guide.md @@ -34,7 +34,9 @@ class LLMProvider(ABC): pass class OpenAIProvider(LLMProvider): - def __init__(self, api_key: str, model: str = "gpt-4"): + # No default model: a hardcoded default is the thing that goes stale. + # Pass the model your deployment is pinned to, from config. + def __init__(self, api_key: str, model: str): self.client = OpenAI(api_key=api_key) self.model = model @@ -47,7 +49,7 @@ class OpenAIProvider(LLMProvider): return response.choices[0].text class AnthropicProvider(LLMProvider): - def __init__(self, api_key: str, model: str = "claude-3-opus"): + def __init__(self, api_key: str, model: str = "claude-opus-5"): self.client = Anthropic(api_key=api_key) self.model = model @@ -158,14 +160,19 @@ def create_chat_messages(user_query: str, context: str) -> List[Dict]: ```python import tiktoken -def count_tokens(text: str, model: str = "gpt-4") -> int: - """Count tokens for a given text and model.""" - encoding = tiktoken.encoding_for_model(model) +def count_tokens(text: str, encoding_name: str = "cl100k_base") -> int: + """Count tokens for a given text. + + Takes an encoding name rather than a model name: encodings change far more + slowly than model IDs, and tiktoken.encoding_for_model() raises KeyError on + any model it has not shipped a mapping for yet. + """ + encoding = tiktoken.get_encoding(encoding_name) return len(encoding.encode(text)) -def truncate_to_token_limit(text: str, max_tokens: int, model: str = "gpt-4") -> str: +def truncate_to_token_limit(text: str, max_tokens: int, encoding_name: str = "cl100k_base") -> str: """Truncate text to fit within token limit.""" - encoding = tiktoken.encoding_for_model(model) + encoding = tiktoken.get_encoding(encoding_name) tokens = encoding.encode(text) if len(tokens) <= max_tokens: @@ -176,12 +183,13 @@ def truncate_to_token_limit(text: str, max_tokens: int, model: str = "gpt-4") -> ### Context Window Management -| Model | Context Window | Effective Limit | -|-------|----------------|-----------------| -| GPT-4 | 8,192 | ~6,000 (leave room for response) | -| GPT-4-32k | 32,768 | ~28,000 | -| Claude 3 | 200,000 | ~180,000 | -| Llama 3 | 8,192 | ~6,000 | +Query the model's advertised context window at runtime rather than table it +here; every fixed number in this position has gone stale within a year. + +The rule that does not change: reserve headroom for the response. Budget +roughly 75-90% of the window for input and leave the remainder for output, +then cap output explicitly with `max_tokens` so a runaway generation cannot +overflow the window or the bill. ### Chunking Strategy @@ -206,12 +214,9 @@ def chunk_text(text: str, chunk_size: int = 1000, overlap: int = 100) -> List[st ### Cost Calculation -| Provider | Input Cost | Output Cost | Example (1K tokens) | -|----------|------------|-------------|---------------------| -| GPT-4 | $0.03/1K | $0.06/1K | $0.09 | -| GPT-3.5 | $0.0005/1K | $0.0015/1K | $0.002 | -| Claude 3 Opus | $0.015/1K | $0.075/1K | $0.09 | -| Claude 3 Haiku | $0.00025/1K | $0.00125/1K | $0.0015 | +Prices move several times a year, so this guide does not carry a rate table. +Pull the current per-million-token input and output prices from your +provider's pricing page and inject them as configuration. ### Cost Tracking @@ -229,26 +234,24 @@ class LLMUsage: def calculate_cost( input_tokens: int, output_tokens: int, - model: str + input_price_per_mtok: float, + output_price_per_mtok: float, ) -> float: - """Calculate cost based on token usage.""" - PRICING = { - "gpt-4": {"input": 0.03, "output": 0.06}, - "gpt-3.5-turbo": {"input": 0.0005, "output": 0.0015}, - "claude-3-opus": {"input": 0.015, "output": 0.075}, - } + """Calculate cost from caller-supplied rates (USD per million tokens). - prices = PRICING.get(model, {"input": 0.01, "output": 0.03}) - - input_cost = (input_tokens / 1000) * prices["input"] - output_cost = (output_tokens / 1000) * prices["output"] + Rates are parameters, not constants. A hardcoded price table is wrong + within a year and silently produces a confidently incorrect business case. + Load these from config so they can be updated without a code change. + """ + input_cost = (input_tokens / 1_000_000) * input_price_per_mtok + output_cost = (output_tokens / 1_000_000) * output_price_per_mtok return input_cost + output_cost ``` ### Cost Optimization Strategies -1. **Use smaller models for simple tasks** - GPT-3.5 for classification, GPT-4 for reasoning +1. **Use smaller models for simple tasks** - small tier for classification, large tier for reasoning 2. **Cache common responses** - Store results for repeated queries 3. **Batch requests** - Combine multiple items in single prompt 4. **Truncate context** - Only include relevant information diff --git a/engineering/llm-cost-optimizer/skills/llm-cost-optimizer/SKILL.md b/engineering/llm-cost-optimizer/skills/llm-cost-optimizer/SKILL.md index 3dd22507..86f049d9 100644 --- a/engineering/llm-cost-optimizer/skills/llm-cost-optimizer/SKILL.md +++ b/engineering/llm-cost-optimizer/skills/llm-cost-optimizer/SKILL.md @@ -60,9 +60,13 @@ Sort by: feature × model × token count. Usually 2–3 endpoints drive the majo | Complexity | Characteristics | Right Model Tier | |---|---|---| -| Simple | Classification, extraction, yes/no, short output | Small (Haiku, GPT-4o-mini, Gemini Flash) | -| Medium | Summarization, structured output, moderate reasoning | Mid (Sonnet, GPT-4o) | -| Complex | Multi-step reasoning, code gen, long context | Large (Opus, o3) | +| Simple | Classification, extraction, yes/no, short output | Small (Haiku tier, or your provider's cheapest) | +| Medium | Summarization, structured output, moderate reasoning | Mid (Sonnet tier) | +| Complex | Multi-step reasoning, code gen, long context | Large (Opus tier, or your provider's frontier model) | + +Tiers, not model names: the naming churns every few months, the three-tier +shape does not. Check your provider's current lineup and price list when you +apply this. **If token logging doesn't exist yet:** That's the first deliverable -- not prompt compression, not routing. You cannot optimize what you cannot see. Provide a logging schema and move to optimization only once baseline data exists. diff --git a/engineering/prompt-governance/skills/prompt-governance/SKILL.md b/engineering/prompt-governance/skills/prompt-governance/SKILL.md index c517fe62..00584138 100644 --- a/engineering/prompt-governance/skills/prompt-governance/SKILL.md +++ b/engineering/prompt-governance/skills/prompt-governance/SKILL.md @@ -77,7 +77,7 @@ prompts: - id: summarizer description: "Summarize support tickets for agent triage" owner: platform-team - model: claude-sonnet-4-5 + model: claude-sonnet-5 versions: - version: 1.1.0 file: summarizer/v1.1.0.md diff --git a/engineering/skills/agent-designer/README.md b/engineering/skills/agent-designer/README.md index 5a023e71..47a4e902 100644 --- a/engineering/skills/agent-designer/README.md +++ b/engineering/skills/agent-designer/README.md @@ -358,7 +358,7 @@ with open('my_tools_openai.json') as f: # Use with OpenAI function calling response = openai.ChatCompletion.create( - model="gpt-4", + model=OPENAI_MODEL, # from config; don't hardcode a model ID messages=[{"role": "user", "content": "Search for AI news"}], functions=schemas['functions'] ) @@ -377,7 +377,7 @@ with open('my_tools_anthropic.json') as f: # Use with Anthropic tool use client = anthropic.Anthropic() response = client.messages.create( - model="claude-3-opus-20240229", + model="claude-opus-5", messages=[{"role": "user", "content": "Search for AI news"}], tools=schemas['tools'] ) diff --git a/engineering/skills/agent-designer/agent_evaluator.py b/engineering/skills/agent-designer/agent_evaluator.py index 8d86c56a..f76eca5e 100644 --- a/engineering/skills/agent-designer/agent_evaluator.py +++ b/engineering/skills/agent-designer/agent_evaluator.py @@ -133,7 +133,6 @@ class AgentEvaluator: def __init__(self): self.error_patterns = self._define_error_patterns() self.performance_thresholds = self._define_performance_thresholds() - self.cost_benchmarks = self._define_cost_benchmarks() def _define_error_patterns(self) -> Dict[str, Dict[str, Any]]: """Define common error patterns and their classifications""" @@ -217,23 +216,15 @@ class AgentEvaluator: "throughput": {"excellent": 100, "good": 50, "acceptable": 20, "poor": 5} # tasks per hour } - def _define_cost_benchmarks(self) -> Dict[str, Any]: - """Define cost benchmarks for different operations""" - return { - "token_costs": { - "gpt-4": {"input": 0.00003, "output": 0.00006}, - "gpt-3.5-turbo": {"input": 0.000002, "output": 0.000002}, - "claude-3": {"input": 0.000015, "output": 0.000075} - }, - "operation_costs": { - "simple_task": 0.005, - "complex_task": 0.050, - "research_task": 0.020, - "analysis_task": 0.030, - "generation_task": 0.015 - } - } - + # _define_cost_benchmarks() was removed. It hardcoded per-token prices for + # gpt-4, gpt-3.5-turbo and claude-3 — all retired, all priced at 2024 + # rates — and the result was assigned to self.cost_benchmarks and never + # read by anything. Cost analysis in this tool uses the cost_usd field the + # caller supplies in each execution log, which is the only figure that can + # be accurate. See engineering-team/skills/senior-prompt-engineer/scripts/ + # prompt_optimizer.py for the --price-per-mtok pattern if a price is + # genuinely needed. + def parse_execution_logs(self, logs_data: List[Dict[str, Any]]) -> List[ExecutionLog]: """Parse raw execution logs into structured format""" logs = [] diff --git a/engineering/skills/agent-designer/assets/sample_execution_logs.json b/engineering/skills/agent-designer/assets/sample_execution_logs.json index 13ec29bd..0f7b45bb 100644 --- a/engineering/skills/agent-designer/assets/sample_execution_logs.json +++ b/engineering/skills/agent-designer/assets/sample_execution_logs.json @@ -38,7 +38,7 @@ } ], "results": { - "summary": "Found 15 relevant sources covering recent AI developments including GPT-4 improvements, autonomous vehicle progress, and medical AI applications.", + "summary": "Found 15 relevant sources covering recent AI developments including frontier model improvements, autonomous vehicle progress, and medical AI applications.", "sources_found": 15, "quality_score": 0.92 }, diff --git a/engineering/skills/agent-designer/references/evaluation_methodology.md b/engineering/skills/agent-designer/references/evaluation_methodology.md index 3b430f5b..24920fe9 100644 --- a/engineering/skills/agent-designer/references/evaluation_methodology.md +++ b/engineering/skills/agent-designer/references/evaluation_methodology.md @@ -653,8 +653,8 @@ trend_analysis: ### API Usage - **Token Consumption**: 2.4M tokens/day - **Cost Breakdown**: - - GPT-4: 68% of token costs - - GPT-3.5: 28% of token costs + - Large-tier model: 68% of token costs + - Small-tier model: 28% of token costs - Other models: 4% of token costs ``` diff --git a/scripts/check_model_freshness.py b/scripts/check_model_freshness.py index 6a79db80..5d32c309 100644 --- a/scripts/check_model_freshness.py +++ b/scripts/check_model_freshness.py @@ -44,6 +44,10 @@ EXCLUDED_DIRS = { SCAN_EXTENSIONS = (".md", ".py", ".json", ".yaml", ".yml", ".sh", ".txt") +# This linter and its allowlist necessarily name retired models; scanning them +# would just require self-referential allowlist entries. +SELF_FILES = {"check_model_freshness.py", "check_model_freshness_allowlist.txt"} + # Retired or superseded identifiers, as (regex, label) pairs. Word-ish bounded # so `claude-3` does not match `claude-3x-something` unintentionally. RETIRED_PATTERNS = [ @@ -64,7 +68,9 @@ RETIRED_PATTERNS = [ "Claude 4 family (superseded by Claude 5)"), (r"\banthropic/claude-[a-z]+-4[.-]\d+\b", "Claude 4 family (superseded by Claude 5)"), (r"\bgpt-3\.5(?:-turbo)?\b", "gpt-3.5 (retired)"), - (r"\bgpt-4(?:-32k|o|o-mini)?\b(?!\S)", "gpt-4 family (superseded)"), + # Longer variants first: alternation is ordered, so `o-mini` must precede `o` + # or `gpt-4o-mini` reports as `gpt-4o`. + (r"\bgpt-4(?:-32k|o-mini|o)?\b", "gpt-4 family (superseded)"), (r"\bGPT-4(?:o|-32k)?\b", "gpt-4 family (superseded)"), (r"\btext-embedding-ada-002\b", "ada-002 embeddings (superseded)"), (r"\bgemini-1\.5(?:-[a-z]+)?\b", "Gemini 1.5 (superseded)"), @@ -153,7 +159,7 @@ def collect(repo_root): for dirpath, dirnames, filenames in os.walk(repo_root): dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIRS] for fn in filenames: - if fn in EXCLUDED_DIRS: + if fn in EXCLUDED_DIRS or fn in SELF_FILES: continue if fn.endswith(SCAN_EXTENSIONS): targets.append(os.path.join(dirpath, fn)) diff --git a/scripts/check_model_freshness_allowlist.txt b/scripts/check_model_freshness_allowlist.txt index 797ea520..e54bda91 100644 --- a/scripts/check_model_freshness_allowlist.txt +++ b/scripts/check_model_freshness_allowlist.txt @@ -8,14 +8,6 @@ # # Every entry needs a one-line reason above it. -# The linter's own deny-list and this allowlist's examples. -scripts/check_model_freshness.py :: claude -scripts/check_model_freshness.py :: gpt- -scripts/check_model_freshness.py :: gemini- -scripts/check_model_freshness.py :: GPT-4 -scripts/check_model_freshness_allowlist.txt :: gpt- -scripts/check_model_freshness_allowlist.txt :: GPT-4 - # The EU AI Act's own systemic-risk examples are frozen in the cited text; # renaming them would misquote the regulation. */references/ai_risk_governance.md :: systemic risk @@ -24,3 +16,13 @@ scripts/check_model_freshness_allowlist.txt :: GPT-4 # warning already at the top of the file. */references/embedding_model_benchmark.md :: ada-002 */references/embedding_model_benchmark.md :: text-embedding + +# litreview examples where the retired model is the SUBJECT of the literature +# being reviewed, not a model the skill would call. Renaming these would +# misdescribe the worked example. +*/litreview/references/framework_selection.md :: Med-PaLM +*/litreview/scripts/cross_search_aggregator.py :: medical Q&A + +# Dated citation: Computer Use shipped on Claude 3.5 Sonnet, and the "+ later" +# is the point of the sentence. +*/notebooklm/references/browser_automation_canon.md :: Computer Use documentation