scriptoriumadmin/claude-skills
1
0
Fork 0
mirror of https://github.com/alirezarezvani/claude-skills.git synced 2026-08-28 04:24:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge pull request #938 from benrfairless/fix/stale-model-references

Browse source 
fix(models): remove retired model IDs and stale pricing, flip G7 blocking
This commit is contained in:
Alireza Rezvani 2026-08-21 10:43:47 +02:00 • committed by GitHub
commit 1f501dd6f3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 111 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/ci-quality-gate.yml vendored
View file

@ -114,10 +114,10 @@ jobs:
python3 scripts/check_dual_publish.py
# Proposed by audit/newgen-2026-06 and never built, which is why retired
# model IDs survived two later audits. Advisory while the existing
# references are cleaned up; flip to blocking once the tree is clean.
- name: Retired model identifier lint (gate G7 — advisory)
continue-on-error: true
# model IDs survived two later audits. The tree is now clean, so this is
# blocking. A deliberate reference needs an entry with a reason in
# scripts/check_model_freshness_allowlist.txt.
- name: Retired model identifier lint (gate G7 — blocking)
run: |
python3 scripts/check_model_freshness.py --all

2
SKILL_PIPELINE.md
View file

@ -111,7 +111,7 @@ After skill is finalized:
```bash
python -m scripts.run_loop \
--eval-set <trigger-eval.json> --skill-path <path> \
--model anthropic/claude-opus-4-6 --max-iterations 5 --verbose
--model anthropic/claude-opus-5 --max-iterations 5 --verbose
```
4. Apply `best_description` to SKILL.md frontmatter

6
c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md
View file

@ -18,8 +18,8 @@ Per million tokens, USD:
| Tier | Example models | Input | Output |
|---|---|---|---|
| Frontier-premium | Claude Sonnet 4.6, GPT-4o-tier | $3.00 | $15.00 |
| Frontier-economy | Gemini 2.5 Flash, Claude Haiku 4.5-tier | $1.25 | $5.00 |
| Frontier-premium | Claude Sonnet 5, frontier-mid tier | $3.00 | $15.00 |
| Frontier-economy | Claude Haiku 4.5, small-model tier | $1.25 | $5.00 |
| Open-hosted | Llama 3.1 70B / Qwen 2.5 72B via Together, Fireworks, OpenRouter | $0.50 | $1.50 |
| Open-economy | 8B-13B-class hosted | $0.10 | $0.30 |
@ -126,7 +126,7 @@ If your utilization is 30% instead of 70%, your effective cost per token roughly
### 2. Capability Drift
- Provider updates models silently or with brief notice
- Your prompts may produce different outputs after upgrade
- Mitigation: pin model IDs (e.g., `claude-sonnet-4-6` vs `claude-sonnet-latest`)
- Mitigation: pin model IDs (e.g., `claude-sonnet-5` vs `claude-sonnet-latest`)
- Cost: regression eval runs on every model swap
### 3. Rate Limits

4
c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py
View file

@ -48,8 +48,8 @@ SAMPLE: Dict[str, Any] = {
# 2026 API pricing per million tokens, $USD (input / output)
API_PRICING = {
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"},
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"},
"open-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"},
}

4
c-level-advisor/chief-ai-officer-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py
View file

@ -54,8 +54,8 @@ SAMPLE: Dict[str, Any] = {
# 2026 API pricing per million tokens, $USD (input / output). These are illustrative;
# real pricing changes; rerun this calculator quarterly.
API_PRICING = {
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"},
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"},
"open-router-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"},
}

6
c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md
View file

@ -18,8 +18,8 @@ Per million tokens, USD:
| Tier | Example models | Input | Output |
|---|---|---|---|
| Frontier-premium | Claude Sonnet 4.6, GPT-4o-tier | $3.00 | $15.00 |
| Frontier-economy | Gemini 2.5 Flash, Claude Haiku 4.5-tier | $1.25 | $5.00 |
| Frontier-premium | Claude Sonnet 5, frontier-mid tier | $3.00 | $15.00 |
| Frontier-economy | Claude Haiku 4.5, small-model tier | $1.25 | $5.00 |
| Open-hosted | Llama 3.1 70B / Qwen 2.5 72B via Together, Fireworks, OpenRouter | $0.50 | $1.50 |
| Open-economy | 8B-13B-class hosted | $0.10 | $0.30 |
@ -126,7 +126,7 @@ If your utilization is 30% instead of 70%, your effective cost per token roughly
### 2. Capability Drift
- Provider updates models silently or with brief notice
- Your prompts may produce different outputs after upgrade
- Mitigation: pin model IDs (e.g., `claude-sonnet-4-6` vs `claude-sonnet-latest`)
- Mitigation: pin model IDs (e.g., `claude-sonnet-5` vs `claude-sonnet-latest`)
- Cost: regression eval runs on every model swap
### 3. Rate Limits

4
c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py
View file

@ -48,8 +48,8 @@ SAMPLE: Dict[str, Any] = {
# 2026 API pricing per million tokens, $USD (input / output)
API_PRICING = {
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"},
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"},
"open-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"},
}

4
c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py
View file

@ -54,8 +54,8 @@ SAMPLE: Dict[str, Any] = {
# 2026 API pricing per million tokens, $USD (input / output). These are illustrative;
# real pricing changes; rerun this calculator quarterly.
API_PRICING = {
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"},
"frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 5 / frontier-mid tier"},
"frontier-economy": {"input": 1.25, "output": 5.00, "label": "Claude Haiku 4.5 / small-model tier"},
"open-router-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"},
}

2
engineering-team/TEAM_STRUCTURE_GUIDE.md
View file

@ -341,7 +341,7 @@ python scripts/model_deployment_pipeline.py --train --config model_config.yaml
# 4. Optimize prompts
cd ../senior-prompt-engineer
python scripts/prompt_optimizer.py --model gpt-4 --task classification
python scripts/prompt_optimizer.py classification_prompt.txt --analyze --model claude
# 5. Deploy with DevOps
cd ../senior-devops

23
engineering-team/skills/senior-ml-engineer/SKILL.md
View file

@ -149,12 +149,23 @@ def call_llm_with_retry(provider: LLMProvider, prompt: str) -> str:
### Cost Management
| Provider | Input Cost | Output Cost |
|----------|------------|-------------|
| GPT-4 | $0.03/1K | $0.06/1K |
| GPT-3.5 | $0.0005/1K | $0.0015/1K |
| Claude 3 Opus | $0.015/1K | $0.075/1K |
| Claude 3 Haiku | $0.00025/1K | $0.00125/1K |
Do not hardcode prices, and do not trust a price table you find in a document
(including this one). Providers reprice several times a year, and a stale
figure produces a confidently wrong business case.
Work in tiers and look the current numbers up at request time:
| Tier | Typical use | Relative cost |
|------|-------------|---------------|
| Small | Classification, extraction, routing, short output | 1x baseline |
| Mid | Summarisation, structured output, moderate reasoning | ~10-25x small |
| Large | Multi-step reasoning, code generation, long context | ~50-100x small |
Read the live rate from your provider's pricing page and pass it in, the way
`engineering-team/skills/senior-prompt-engineer/scripts/prompt_optimizer.py`
takes `--price-per-mtok`.
The ratios between tiers are far more stable than the absolute prices, so
build the model-routing decision on the ratio.
---

65
engineering-team/skills/senior-ml-engineer/references/llm_integration_guide.md
View file

@ -34,7 +34,9 @@ class LLMProvider(ABC):
pass
class OpenAIProvider(LLMProvider):
def __init__(self, api_key: str, model: str = "gpt-4"):
# No default model: a hardcoded default is the thing that goes stale.
# Pass the model your deployment is pinned to, from config.
def __init__(self, api_key: str, model: str):
self.client = OpenAI(api_key=api_key)
self.model = model
@ -47,7 +49,7 @@ class OpenAIProvider(LLMProvider):
return response.choices[0].text
class AnthropicProvider(LLMProvider):
def __init__(self, api_key: str, model: str = "claude-3-opus"):
def __init__(self, api_key: str, model: str = "claude-opus-5"):
self.client = Anthropic(api_key=api_key)
self.model = model
@ -158,14 +160,19 @@ def create_chat_messages(user_query: str, context: str) -> List[Dict]:
```python
import tiktoken
def count_tokens(text: str, model: str = "gpt-4") -> int:
"""Count tokens for a given text and model."""
encoding = tiktoken.encoding_for_model(model)
def count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
"""Count tokens for a given text.
Takes an encoding name rather than a model name: encodings change far more
slowly than model IDs, and tiktoken.encoding_for_model() raises KeyError on
any model it has not shipped a mapping for yet.
"""
encoding = tiktoken.get_encoding(encoding_name)
return len(encoding.encode(text))
def truncate_to_token_limit(text: str, max_tokens: int, model: str = "gpt-4") -> str:
def truncate_to_token_limit(text: str, max_tokens: int, encoding_name: str = "cl100k_base") -> str:
"""Truncate text to fit within token limit."""
encoding = tiktoken.encoding_for_model(model)
encoding = tiktoken.get_encoding(encoding_name)
tokens = encoding.encode(text)
if len(tokens) <= max_tokens:
@ -176,12 +183,13 @@ def truncate_to_token_limit(text: str, max_tokens: int, model: str = "gpt-4") ->
### Context Window Management
| Model | Context Window | Effective Limit |
|-------|----------------|-----------------|
| GPT-4 | 8,192 | ~6,000 (leave room for response) |
| GPT-4-32k | 32,768 | ~28,000 |
| Claude 3 | 200,000 | ~180,000 |
| Llama 3 | 8,192 | ~6,000 |
Query the model's advertised context window at runtime rather than table it
here; every fixed number in this position has gone stale within a year.
The rule that does not change: reserve headroom for the response. Budget
roughly 75-90% of the window for input and leave the remainder for output,
then cap output explicitly with `max_tokens` so a runaway generation cannot
overflow the window or the bill.
### Chunking Strategy
@ -206,12 +214,9 @@ def chunk_text(text: str, chunk_size: int = 1000, overlap: int = 100) -> List[st
### Cost Calculation
| Provider | Input Cost | Output Cost | Example (1K tokens) |
|----------|------------|-------------|---------------------|
| GPT-4 | $0.03/1K | $0.06/1K | $0.09 |
| GPT-3.5 | $0.0005/1K | $0.0015/1K | $0.002 |
| Claude 3 Opus | $0.015/1K | $0.075/1K | $0.09 |
| Claude 3 Haiku | $0.00025/1K | $0.00125/1K | $0.0015 |
Prices move several times a year, so this guide does not carry a rate table.
Pull the current per-million-token input and output prices from your
provider's pricing page and inject them as configuration.
### Cost Tracking
@ -229,26 +234,24 @@ class LLMUsage:
def calculate_cost(
input_tokens: int,
output_tokens: int,
model: str
input_price_per_mtok: float,
output_price_per_mtok: float,
) -> float:
"""Calculate cost based on token usage."""
PRICING = {
"gpt-4": {"input": 0.03, "output": 0.06},
"gpt-3.5-turbo": {"input": 0.0005, "output": 0.0015},
"claude-3-opus": {"input": 0.015, "output": 0.075},
}
"""Calculate cost from caller-supplied rates (USD per million tokens).
prices = PRICING.get(model, {"input": 0.01, "output": 0.03})
input_cost = (input_tokens / 1000) * prices["input"]
output_cost = (output_tokens / 1000) * prices["output"]
Rates are parameters, not constants. A hardcoded price table is wrong
within a year and silently produces a confidently incorrect business case.
Load these from config so they can be updated without a code change.
"""
input_cost = (input_tokens / 1_000_000) * input_price_per_mtok
output_cost = (output_tokens / 1_000_000) * output_price_per_mtok
return input_cost + output_cost
```
### Cost Optimization Strategies
1. **Use smaller models for simple tasks** - GPT-3.5 for classification, GPT-4 for reasoning
1. **Use smaller models for simple tasks** - small tier for classification, large tier for reasoning
2. **Cache common responses** - Store results for repeated queries
3. **Batch requests** - Combine multiple items in single prompt
4. **Truncate context** - Only include relevant information

10
engineering/llm-cost-optimizer/skills/llm-cost-optimizer/SKILL.md
View file

@ -60,9 +60,13 @@ Sort by: feature × model × token count. Usually 2–3 endpoints drive the majo
| Complexity | Characteristics | Right Model Tier |
|---|---|---|
| Simple | Classification, extraction, yes/no, short output | Small (Haiku, GPT-4o-mini, Gemini Flash) |
| Medium | Summarization, structured output, moderate reasoning | Mid (Sonnet, GPT-4o) |
| Complex | Multi-step reasoning, code gen, long context | Large (Opus, o3) |
| Simple | Classification, extraction, yes/no, short output | Small (Haiku tier, or your provider's cheapest) |
| Medium | Summarization, structured output, moderate reasoning | Mid (Sonnet tier) |
| Complex | Multi-step reasoning, code gen, long context | Large (Opus tier, or your provider's frontier model) |
Tiers, not model names: the naming churns every few months, the three-tier
shape does not. Check your provider's current lineup and price list when you
apply this.
**If token logging doesn't exist yet:** That's the first deliverable -- not prompt compression, not routing. You cannot optimize what you cannot see. Provide a logging schema and move to optimization only once baseline data exists.

2
engineering/prompt-governance/skills/prompt-governance/SKILL.md
View file

@ -77,7 +77,7 @@ prompts:
- id: summarizer
description: "Summarize support tickets for agent triage"
owner: platform-team
model: claude-sonnet-4-5
model: claude-sonnet-5
versions:
- version: 1.1.0
file: summarizer/v1.1.0.md

4
engineering/skills/agent-designer/README.md
View file

@ -358,7 +358,7 @@ with open('my_tools_openai.json') as f:
# Use with OpenAI function calling
response = openai.ChatCompletion.create(
model="gpt-4",
model=OPENAI_MODEL, # from config; don't hardcode a model ID
messages=[{"role": "user", "content": "Search for AI news"}],
functions=schemas['functions']
)
@ -377,7 +377,7 @@ with open('my_tools_anthropic.json') as f:
# Use with Anthropic tool use
client = anthropic.Anthropic()
response = client.messages.create(
model="claude-3-opus-20240229",
model="claude-opus-5",
messages=[{"role": "user", "content": "Search for AI news"}],
tools=schemas['tools']
)

27
engineering/skills/agent-designer/agent_evaluator.py
View file

@ -133,7 +133,6 @@ class AgentEvaluator:
def __init__(self):
self.error_patterns = self._define_error_patterns()
self.performance_thresholds = self._define_performance_thresholds()
self.cost_benchmarks = self._define_cost_benchmarks()
def _define_error_patterns(self) -> Dict[str, Dict[str, Any]]:
"""Define common error patterns and their classifications"""
@ -217,23 +216,15 @@ class AgentEvaluator:
"throughput": {"excellent": 100, "good": 50, "acceptable": 20, "poor": 5} # tasks per hour
}
def _define_cost_benchmarks(self) -> Dict[str, Any]:
"""Define cost benchmarks for different operations"""
return {
"token_costs": {
"gpt-4": {"input": 0.00003, "output": 0.00006},
"gpt-3.5-turbo": {"input": 0.000002, "output": 0.000002},
"claude-3": {"input": 0.000015, "output": 0.000075}
},
"operation_costs": {
"simple_task": 0.005,
"complex_task": 0.050,
"research_task": 0.020,
"analysis_task": 0.030,
"generation_task": 0.015
}
}
# _define_cost_benchmarks() was removed. It hardcoded per-token prices for
# gpt-4, gpt-3.5-turbo and claude-3 — all retired, all priced at 2024
# rates — and the result was assigned to self.cost_benchmarks and never
# read by anything. Cost analysis in this tool uses the cost_usd field the
# caller supplies in each execution log, which is the only figure that can
# be accurate. See engineering-team/skills/senior-prompt-engineer/scripts/
# prompt_optimizer.py for the --price-per-mtok pattern if a price is
# genuinely needed.
def parse_execution_logs(self, logs_data: List[Dict[str, Any]]) -> List[ExecutionLog]:
"""Parse raw execution logs into structured format"""
logs = []

2
engineering/skills/agent-designer/assets/sample_execution_logs.json
View file

@ -38,7 +38,7 @@
}
],
"results": {
"summary": "Found 15 relevant sources covering recent AI developments including GPT-4 improvements, autonomous vehicle progress, and medical AI applications.",
"summary": "Found 15 relevant sources covering recent AI developments including frontier model improvements, autonomous vehicle progress, and medical AI applications.",
"sources_found": 15,
"quality_score": 0.92
},

4
engineering/skills/agent-designer/references/evaluation_methodology.md
View file

@ -653,8 +653,8 @@ trend_analysis:
### API Usage
- **Token Consumption**: 2.4M tokens/day
- **Cost Breakdown**:
- GPT-4: 68% of token costs
- GPT-3.5: 28% of token costs
- Large-tier model: 68% of token costs
- Small-tier model: 28% of token costs
- Other models: 4% of token costs
```

10
scripts/check_model_freshness.py
View file

@ -44,6 +44,10 @@ EXCLUDED_DIRS = {
SCAN_EXTENSIONS = (".md", ".py", ".json", ".yaml", ".yml", ".sh", ".txt")
# This linter and its allowlist necessarily name retired models; scanning them
# would just require self-referential allowlist entries.
SELF_FILES = {"check_model_freshness.py", "check_model_freshness_allowlist.txt"}
# Retired or superseded identifiers, as (regex, label) pairs. Word-ish bounded
# so `claude-3` does not match `claude-3x-something` unintentionally.
RETIRED_PATTERNS = [
@ -64,7 +68,9 @@ RETIRED_PATTERNS = [
"Claude 4 family (superseded by Claude 5)"),
(r"\banthropic/claude-[a-z]+-4[.-]\d+\b", "Claude 4 family (superseded by Claude 5)"),
(r"\bgpt-3\.5(?:-turbo)?\b", "gpt-3.5 (retired)"),
(r"\bgpt-4(?:-32k|o|o-mini)?\b(?!\S)", "gpt-4 family (superseded)"),
# Longer variants first: alternation is ordered, so `o-mini` must precede `o`
# or `gpt-4o-mini` reports as `gpt-4o`.
(r"\bgpt-4(?:-32k|o-mini|o)?\b", "gpt-4 family (superseded)"),
(r"\bGPT-4(?:o|-32k)?\b", "gpt-4 family (superseded)"),
(r"\btext-embedding-ada-002\b", "ada-002 embeddings (superseded)"),
(r"\bgemini-1\.5(?:-[a-z]+)?\b", "Gemini 1.5 (superseded)"),
@ -153,7 +159,7 @@ def collect(repo_root):
for dirpath, dirnames, filenames in os.walk(repo_root):
dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIRS]
for fn in filenames:
if fn in EXCLUDED_DIRS:
if fn in EXCLUDED_DIRS or fn in SELF_FILES:
continue
if fn.endswith(SCAN_EXTENSIONS):
targets.append(os.path.join(dirpath, fn))

18
scripts/check_model_freshness_allowlist.txt
View file

@ -8,14 +8,6 @@
#
# Every entry needs a one-line reason above it.
# The linter's own deny-list and this allowlist's examples.
scripts/check_model_freshness.py :: claude
scripts/check_model_freshness.py :: gpt-
scripts/check_model_freshness.py :: gemini-
scripts/check_model_freshness.py :: GPT-4
scripts/check_model_freshness_allowlist.txt :: gpt-
scripts/check_model_freshness_allowlist.txt :: GPT-4
# The EU AI Act's own systemic-risk examples are frozen in the cited text;
# renaming them would misquote the regulation.
*/references/ai_risk_governance.md :: systemic risk
@ -24,3 +16,13 @@ scripts/check_model_freshness_allowlist.txt :: GPT-4
# warning already at the top of the file.
*/references/embedding_model_benchmark.md :: ada-002
*/references/embedding_model_benchmark.md :: text-embedding
# litreview examples where the retired model is the SUBJECT of the literature
# being reviewed, not a model the skill would call. Renaming these would
# misdescribe the worked example.
*/litreview/references/framework_selection.md :: Med-PaLM
*/litreview/scripts/cross_search_aggregator.py :: medical Q&A
# Dated citation: Computer Use shipped on Claude 3.5 Sonnet, and the "+ later"
# is the point of the sentence.
*/notebooklm/references/browser_automation_canon.md :: Computer Use documentation