From 7ae93385bd82fa99ee4e48f079731df409951cf3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 12 May 2026 18:41:04 +0000 Subject: [PATCH] feat(chief-ai-officer-advisor): eval-demanding CAIO skill (v2.5.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit World-class, in-depth Chief AI Officer skill covering 4 specific decisions (not a generic AI strategy survey): 1. Should we use an API, fine-tune, or build our own? (3-yr TCO + breakeven) 2. Is this AI use case high-risk under regulation? (EU AI Act + US state + industry overlays with Article-level citations) 3. When do we switch from API to self-hosted, and at what cost? (2026 pricing + GPU economics + hidden costs) 4. What AI role do we hire next? (5-stage map + 9-role definition table) Built under karpathy-coder discipline (third in a row): - Assumptions surfaced upfront before code (principle 1) - Each tool/reference covers ONE decision; rejected generic-survey scope (#2) - Surgical changes only; no scope creep (#3) - All 3 tools smoke-tested with embedded samples before commit (#4) - karpathy/complexity_checker.py: 0 findings on 3 new tools - karpathy/diff_surgeon.py: 0 findings on staged diff 3 stdlib Python tools with deterministic logic: - model_buildvsbuy_calculator.py — Returns API/FINE_TUNE/BUILD recommendation, 3-year TCO across 6 paths, breakeven analysis. Balances economic crossover with practical feasibility (data availability, ML team capacity, compliance). Embedded sample (B2B customer support, 4M queries/mo) -> API recommended despite breakeven crossed, because no fine-tune data + 1-engineer ML team. - ai_risk_classifier.py — Returns EU AI Act tier (PROHIBITED/HIGH/LIMITED/ MINIMAL) with 7 Article citations + US state triggers (NYC LL 144, CO AI Act, IL HB 53, CA SB 1001, IL BIPA) + industry overlays (FDA, CFPB, NAIC, ECOA, Fed SR 11-7). Sample (AI hiring in EU+NY+CO+IL+CA) -> HIGH, conformity required, 3 US triggers, 14 controls. - ai_cost_economics.py — Returns API costs (3 tiers) + self-hosted costs (low/ mid/high GPU rates with 24/7 warm + ops attribution) + breakeven analysis. Reveals key insight: self-hosted floor makes API economics dominate at typical B2B SaaS scale. Sample (5M tokens/day, 750M/mo) -> API at $1,500/mo beats self-hosted at $13,450/mo by 9x; breakeven at 6.7B tokens/mo. 4 in-depth references, each citing 5+ authoritative sources: - model_buildvsbuy_strategy.md — 3 paths with failure modes, 6 fine-tuning approaches ranked by cost (RAG/LoRA/full FT/RLHF/DPO/continued pre-training), decision tree, eval-first discipline. Cites Anthropic/OpenAI/Google/Meta model cards, LoRA paper, RLHF paper, DPO paper, Stanford CRFM Foundation Models report, Foundation Models and Fair Use (Henderson et al.). - ai_risk_governance.md — Full EU AI Act tier map (Art. 5 prohibited, Art. 6 + Annex III high-risk, Art. 50 limited-risk) with all 8 high-risk domains + 11 obligation articles. NIST AI RMF 1.0. US state patchwork (9 laws). Industry overlays (FDA AI/ML, CFPB, NYDFS, NAIC). 10-item governance program checklist. When-to-hire-AI-counsel criteria. - ai_cost_economics.md — 2026 API pricing (4 tiers), GPU rental (A100/H100/ H200/B200), throughput estimates, GPU count by model size, utilization reality (20-80%), 6 hidden costs of self-hosted, 6 hidden costs of API, migration cost, prompt caching as economics lever. Cites vLLM paper, DistServe, HELM, Artificial Analysis. - ai_team_org_evolution.md — 5-stage role map (pre-seed -> late-stage), 9-role definition table (AI engineer != ML engineer != research scientist), AI team vs data team contrast (8 dimensions), 7 anti-patterns, hiring sequencing rule. Cites Huyen "Designing ML Systems" + "AI Engineering", State of AI Report. cs-caio-advisor agent (c-level-agents/agents/cs-caio-advisor.md): - Eval-demanding realist voice - Hard rule: does not duplicate engineering AI/ML skills (rag-architect, agent-designer, prompt-governance, self-eval, llm-cost-optimizer) - Treats every AI use case as a hiring decision; pushes back on AI hype /cs:caio-review slash command: - 6-question forcing interrogation: eval set, hallucination SLO, regulatory tier, model selection, cost trajectory, role-that-unblocks - Routes to /cs:cdo-review, /cs:gc-review, /cs:ciso-review, /cs:cfo-review, /cs:chro-review cs-caio-advisor voice spec added to persona-voices.md. Updates: - c-level plugin.json: v2.5.2 -> v2.5.3 (31 skills, 11 cs-* agents) - c-level-agents plugin.json: v1.2.0 -> v1.3.0 (11 agents, 19 commands) - marketplace.json: both c-level entries; new CAIO keywords (chief-ai-officer, caio, ai-strategy, model-buildvsbuy, eu-ai-act, ai-cost-economics) - c-level CLAUDE.md: CAIO row added; agent + count tables updated - Root CLAUDE.md: 265->266 skills, 30->31 cs-* agents, 364->367 tools, 494->498 references, 51->52 commands; v2.5.3 highlight section - CHANGELOG.md: v2.5.3 entry with full rationale Known follow-up (out of scope this PR): cs-general-counsel-advisor voice spec still missing from persona-voices.md (carried from v2.5.1); separate PR. Disclaimer in every output: not legal advice; not a replacement for AI counsel on EU AI Act conformity; not a tactical AI/ML engineering skill. https://claude.ai/code/session_012WtZMm5NJHqkYoRqA9fHMN --- .claude-plugin/marketplace.json | 14 +- CHANGELOG.md | 60 +++ CLAUDE.md | 12 +- c-level-advisor/.claude-plugin/plugin.json | 4 +- c-level-advisor/CLAUDE.md | 18 +- .../c-level-agents/.claude-plugin/plugin.json | 4 +- .../c-level-agents/agents/cs-caio-advisor.md | 177 +++++++ .../references/persona-voices.md | 6 + .../skills/caio-review/SKILL.md | 140 +++++ .../skills/chief-ai-officer-advisor/SKILL.md | 236 +++++++++ .../references/ai_cost_economics.md | 235 +++++++++ .../references/ai_risk_governance.md | 231 +++++++++ .../references/ai_team_org_evolution.md | 240 +++++++++ .../references/model_buildvsbuy_strategy.md | 134 +++++ .../scripts/ai_cost_economics.py | 350 +++++++++++++ .../scripts/ai_risk_classifier.py | 478 ++++++++++++++++++ .../scripts/model_buildvsbuy_calculator.py | 364 +++++++++++++ 17 files changed, 2684 insertions(+), 19 deletions(-) create mode 100644 c-level-advisor/c-level-agents/agents/cs-caio-advisor.md create mode 100644 c-level-advisor/c-level-agents/skills/caio-review/SKILL.md create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/SKILL.md create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/references/ai_risk_governance.md create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/references/ai_team_org_evolution.md create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/references/model_buildvsbuy_strategy.md create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py create mode 100644 c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index e34c05be..3de46c91 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -39,8 +39,8 @@ { "name": "c-level-skills", "source": "./c-level-advisor", - "description": "30 C-level advisory skills + c-level-agents plugin layer: virtual board of directors (CEO, CTO, COO, CPO, CMO, CFO, CRO, CISO, CHRO) plus General Counsel (contract risk scanner, term sheet analyzer, IP + regulatory playbook) and Chief Data Officer (AI training data audit, data product strategy picker, data asset valuator), executive mentor, founder coach, orchestration (Chief of Staff, board meetings, decision logger), strategic capabilities (board deck builder, scenario war room, competitive intel, M&A playbook), culture frameworks, and 10 cs-* persona agents + 18 /cs:* slash commands (founder-mode router, office-hours intake, multi-role boardroom, strategic sprint pipeline, cross-model consensus, cooldown freeze).", - "version": "2.5.2", + "description": "31 C-level advisory skills + c-level-agents plugin layer: virtual board of directors (CEO, CTO, COO, CPO, CMO, CFO, CRO, CISO, CHRO) plus General Counsel (contract risk scanner, term sheet analyzer, IP + regulatory playbook), Chief Data Officer (AI training data audit, data product strategy picker, data asset valuator), and Chief AI Officer (model build-vs-buy calculator with 3-yr TCO, AI risk classifier under EU AI Act + US state laws, AI cost economics with API-vs-self-hosted breakeven), executive mentor, founder coach, orchestration (Chief of Staff, board meetings, decision logger), strategic capabilities (board deck builder, scenario war room, competitive intel, M&A playbook), culture frameworks, and 11 cs-* persona agents + 19 /cs:* slash commands (founder-mode router, office-hours intake, multi-role boardroom, strategic sprint pipeline, cross-model consensus, cooldown freeze).", + "version": "2.5.3", "author": { "name": "Alireza Rezvani" }, @@ -61,8 +61,8 @@ { "name": "c-level-agents", "source": "./c-level-advisor/c-level-agents", - "description": "Founder-mode executive team plugin: 10 cs-* C-suite agents (CFO, CMO, CRO, CPO, COO, CHRO, CISO, Chief of Staff, General Counsel, Chief Data Officer) with distinct cognitive voices, plus 18 /cs:* slash commands — forcing-question office hours (CFO/CMO/CPO/CRO/CTO/CISO/GC/CDO reviews), strategic sprint pipeline (brief → boardroom → decide → execute → post-mortem), and meta routing (/cs:founder-mode auto-router, /cs:onboard, /cs:cross-eval multi-model consensus, /cs:freeze cooldown lock). Wraps the 30 c-level skills (including general-counsel-advisor and chief-data-officer-advisor with AI training data audit + data product strategy picker + data asset valuator) with cognitive gearing, persona voice, and artifact-driven handoffs. The business-domain answer to YC Garry Tan's gstack.", - "version": "1.2.0", + "description": "Founder-mode executive team plugin: 11 cs-* C-suite agents (CFO, CMO, CRO, CPO, COO, CHRO, CISO, Chief of Staff, General Counsel, Chief Data Officer, Chief AI Officer) with distinct cognitive voices, plus 19 /cs:* slash commands — forcing-question office hours (CFO/CMO/CPO/CRO/CTO/CISO/GC/CDO/CAIO reviews), strategic sprint pipeline (brief → boardroom → decide → execute → post-mortem), and meta routing (/cs:founder-mode auto-router, /cs:onboard, /cs:cross-eval multi-model consensus, /cs:freeze cooldown lock). Wraps the 31 c-level skills (including chief-ai-officer-advisor with model build-vs-buy calculator + AI risk classifier under EU AI Act + AI cost economics) with cognitive gearing, persona voice, and artifact-driven handoffs. The business-domain answer to YC Garry Tan's gstack.", + "version": "1.3.0", "author": { "name": "Alireza Rezvani" }, @@ -86,6 +86,12 @@ "ai-training-data", "data-product-strategy", "data-as-asset", + "chief-ai-officer", + "caio", + "ai-strategy", + "model-buildvsbuy", + "eu-ai-act", + "ai-cost-economics", "decision-logging", "cross-model" ], diff --git a/CHANGELOG.md b/CHANGELOG.md index 9738aec8..eb3c2a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,66 @@ All notable changes to the Claude Skills Library will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.5.3] - 2026-05-12 — chief-ai-officer-advisor: AI strategy with citations + +### Added — C-Level Advisory + +- **chief-ai-officer-advisor** skill (`./c-level-advisor/skills/chief-ai-officer-advisor/`) — opinionated, eval-demanding CAIO skill covering 4 specific decisions. The third decision-driven C-role skill in the founder-mode lineup, after general-counsel-advisor (v2.5.1) and chief-data-officer-advisor (v2.5.2). +- **4 specific decisions covered** (not a generic AI strategy survey): + 1. **Should we use an API, fine-tune, or build our own?** (model build-vs-buy with 3-year TCO) + 2. **Is this AI use case high-risk under regulation, and how do we govern it?** (EU AI Act + NIST AI RMF + US state patchwork) + 3. **When do we switch from API to self-hosted, and at what cost?** (token economics with breakeven analysis) + 4. **What AI role do we hire next?** (stage-to-role map; AI engineer ≠ ML engineer ≠ research scientist) +- **3 stdlib Python tools with deterministic logic**: + - **`model_buildvsbuy_calculator.py`** — Returns API / FINE_TUNE / BUILD recommendation, 3-year TCO across 6 path variants (API frontier-premium/economy/open-hosted, fine-tune, self-hosted 70B-class, build-from-scratch), and breakeven analysis. Balances economic crossover with practical feasibility (data availability, ML team capacity, compliance constraints). Embedded sample (B2B customer support, 4M queries/mo) → API recommendation despite economic breakeven crossed, due to no fine-tune data + 1-engineer ML team. + - **`ai_risk_classifier.py`** — Returns EU AI Act tier (PROHIBITED / HIGH / LIMITED / MINIMAL) with Article-level citations, US state triggers (NYC LL 144, CO AI Act, IL HB 53, CA SB 1001, IL BIPA), industry overlays (FDA AI/ML, ECOA, NAIC AI bulletin), required-controls list, and conformity-assessment flag. Embedded sample (AI hiring screening in EU+NY+CO+IL+CA) → HIGH risk, conformity required, 3 US state triggers, 14 controls. 7 EU AI Act articles cited (5, 6, 9-15, 43, 49, 72). + - **`ai_cost_economics.py`** — Returns monthly costs at 6 paths (3 API tiers + self-hosted at low/mid/high GPU rates), breakeven monthly tokens, sensitivity to GPU pricing. Embedded sample (5M tokens/day, 750M/mo) → API at $1,500/mo beats self-hosted at $13,450/mo by 9x; breakeven at 6.7B tokens/mo for 70B-class on A100s. Reveals key insight that self-hosted floor (24/7 warm GPUs + ops) makes API economics dominate at typical B2B SaaS scale. +- **4 in-depth references each citing 5+ authoritative sources**: + - `model_buildvsbuy_strategy.md` — 3 paths with failure modes, 6 fine-tuning approaches (few-shot, prompt eng, RAG, LoRA, full FT, RLHF/DPO, continued pre-training) ranked by cost and use case, decision tree, eval-first discipline. Cites Anthropic/OpenAI/Google/Meta model cards, LoRA paper (Hu et al.), RLHF paper (Ouyang et al.), DPO paper (Rafailov et al.), Foundation Models report (Stanford CRFM), Foundation Models and Fair Use (Henderson et al.). + - `ai_risk_governance.md` — Full EU AI Act tier map (prohibited Article 5, high-risk Article 6 + Annex III, limited-risk Article 50, minimal-risk) with all 8 high-risk domains + 11 obligation Articles. NIST AI RMF 1.0 (4 functions, 7 trustworthy characteristics). US state patchwork (NYC LL 144, CO AI Act, IL HB 53, CA SB 1001, CA AB 2013, CA AB 1008, IL BIPA, WA MHMD, TX biometric). Industry overlays (FDA, CFPB, Fed SR 11-7, NYDFS Reg 23, ECOA, NAIC). 10-item governance program checklist. When-to-hire-AI-counsel criteria. + - `ai_cost_economics.md` — 2026 API pricing across 4 tiers, GPU rental (A100/H100/H200/B200), throughput estimates, GPU count by model size, cost-per-million-tokens calculations, utilization reality (interactive 20-40%, batch 60-80%), 6 hidden costs of self-hosted, 6 hidden costs of API, migration cost (3-6 months, 2-3 engineers), prompt caching as economics lever. Cites vLLM paper, DistServe (NSDI 2024), HELM benchmark, Artificial Analysis, Llama 3.1 paper. + - `ai_team_org_evolution.md` — 5-stage role map (pre-seed → late-stage), 9-role definition table distinguishing AI engineer / ML engineer / research scientist / data scientist / AI safety / AI PM / Head of AI / CAIO. AI team vs data team contrast (8 dimensions). 7 specific anti-patterns. Hiring sequencing rule. Cites Huyen "Designing ML Systems" + "AI Engineering", State of AI Report, Karpathy's AI engineer archetype discussions. +- **cs-caio-advisor** agent (`./c-level-advisor/c-level-agents/agents/cs-caio-advisor.md`) — eval-demanding realist orchestrating the skill. Voice: "What does this AI need to be good at, and how would you measure it?" Treats every AI use case as a hiring decision; pushes back on AI hype; demands fallback behavior before scale. +- **`/cs:caio-review`** slash command (`./c-level-advisor/c-level-agents/skills/caio-review/SKILL.md`) — 6-question forcing interrogation: eval discipline, hallucination SLO, regulatory tier, model selection, cost trajectory, role-that-unblocks-this. +- **cs-caio-advisor voice spec** added to `persona-voices.md`. + +### Why This Matters + +By 2026, every founder is making AI decisions that didn't exist 18 months ago — and gstack, general legal counsel, CTOs, and CISOs each only cover part of the picture. The CAIO concerns that this skill uniquely owns: + +1. **Model build-vs-buy is not a single answer.** 80% of B2B SaaS should use frontier APIs; 15% should fine-tune; <1% should pre-train. The decision depends on data availability + team capacity + economics + compliance, not on technology preference. +2. **EU AI Act conformity is consequential and slow.** A high-risk AI use case requires 3-12 months of conformity work + EU database registration + 10 Articles of obligations. Discovering this 2 weeks before EU launch is a category of pain this skill prevents. +3. **API vs self-hosted breakeven is much higher than founders expect.** For 70B-class on rented A100s, breakeven is typically 1-10 billion tokens per month — not the 100M-500M most founders intuit. Self-hosting "to save money" usually wastes engineering capacity. +4. **AI team confusion costs 12 months of productivity.** Hiring a research scientist as first AI hire is the single most common AI hiring mistake, and it's expensive to undo. + +### Built with Karpathy-Coder Discipline + +Maintained the discipline established in v2.5.2: + +- **Principle 1 (Think before coding):** assumptions surfaced upfront before file writes. Locked 4 decisions, 3 tools, 4 references, success criteria. User confirmed direction. +- **Principle 2 (Simplicity first):** rejected "generic AI strategy survey" framing. Each tool covers ONE decision. Each reference answers ONE decision. No overlap with engineering/rag-architect, engineering/agent-designer, engineering/llm-cost-optimizer. +- **Principle 3 (Surgical changes):** touched only files in the locked plan. No "while I'm here" cleanup. +- **Principle 4 (Goal-driven execution):** all 3 tools smoke-tested with embedded samples before commit. Verifiable success criteria met. + +### Changed + +- **Total skills:** 265 → 266 (+1 chief-ai-officer-advisor) +- **cs-* agents:** 30 → 31 (+1 cs-caio-advisor in c-level-agents plugin) +- **/cs:* slash commands:** 18 → 19 (+1 /cs:caio-review) +- **Python tools:** 364 → 367 (+3 in chief-ai-officer-advisor/scripts/) +- **References:** 494 → 498 (+4 in chief-ai-officer-advisor/references/) +- **c-level-skills** plugin: v2.5.2 → v2.5.3 (description expanded; 30 → 31 skills, 10 → 11 cs-* agents) +- **c-level-agents** plugin: v1.2.0 → v1.3.0 (description expanded with CAIO; new agent + command; +`chief-ai-officer`, `caio`, `ai-strategy`, `model-buildvsbuy`, `eu-ai-act`, `ai-cost-economics` keywords) + +### Known follow-ups (NOT included this PR per surgical scope) + +- The `cs-general-counsel-advisor` voice spec is still missing from `persona-voices.md` (carried from v2.5.1). Will be addressed in a separate small PR. +- Phase 2 remainder (3 more C-roles: CCO customer, VPE engineering execution, CCO comms) deferred to v2.5.4+. + +### Disclaimer + +The `chief-ai-officer-advisor` skill surfaces strategic AI decisions but is **not legal advice** for AI regulation, **not a replacement for outside AI counsel** for EU AI Act conformity assessments, and **not a tactical AI/ML engineering skill**. For tactical AI engineering, see `engineering/rag-architect/`, `engineering/agent-designer/`, `engineering/prompt-governance/`, `engineering/self-eval/`, `engineering/llm-cost-optimizer/`. + ## [2.5.2] - 2026-05-12 — chief-data-officer-advisor: data strategy without surveys ### Added — C-Level Advisory diff --git a/CLAUDE.md b/CLAUDE.md index 7c1de156..b848ae0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co This is a **comprehensive skills library** for Claude AI and Claude Code - reusable, production-ready skill packages that bundle domain expertise, best practices, analysis tools, and strategic frameworks. The repository provides modular skills that teams can download and use directly in their workflows. -**Current Scope:** 265 production-ready skills across 9 domains with 364 Python automation tools, 494 reference guides, 37 agents (30 `cs-*` + 7 personas), and 51 slash commands. +**Current Scope:** 266 production-ready skills across 9 domains with 367 Python automation tools, 498 reference guides, 38 agents (31 `cs-*` + 7 personas), and 52 slash commands. **Key Distinction**: This is NOT a traditional application. It's a library of skill packages meant to be extracted and deployed by users into their own Claude workflows. @@ -124,9 +124,15 @@ See [standards/git/git-workflow-standards.md](standards/git/git-workflow-standar ## Current Version -**Version:** v2.5.2 (latest) +**Version:** v2.5.3 (latest) -**v2.5.2 Highlights — chief-data-officer-advisor: data strategy without surveys:** +**v2.5.3 Highlights — chief-ai-officer-advisor: AI strategy with citations:** +- **chief-ai-officer-advisor** skill (new, `./c-level-advisor/skills/chief-ai-officer-advisor/`) — opinionated, eval-demanding CAIO skill covering 4 specific decisions. 3 stdlib Python tools with deterministic logic: `model_buildvsbuy_calculator.py` (API vs fine-tune vs build with 3-year TCO, balances economic breakeven with practical feasibility), `ai_risk_classifier.py` (EU AI Act tier classification with Article-level citations + US state patchwork: NYC LL 144, CO AI Act, IL HB 53, CA SB 1001, IL BIPA + industry overlays for FDA/NYDFS/NAIC/ECOA), `ai_cost_economics.py` (API vs self-hosted breakeven with 2026 pricing across A100/H100, utilization reality, hidden costs). 4 in-depth references each citing 5+ authoritative sources: model build-vs-buy strategy (decision tree, 6 fine-tuning approaches, failure modes), AI risk governance (full EU AI Act tier map + NIST AI RMF + governance program checklist), AI cost economics (2026 pricing + GPU economics + migration cost + prompt caching), AI team org evolution (5-stage role map + 9-role definition table + AI team vs data team contrast + 7 anti-patterns). +- **cs-caio-advisor** agent (new) — eval-demanding realist orchestrating the skill via `/cs:caio-review`. Distinct voice: "What does this AI need to be good at, and how would you measure it?" Treats every AI use case as a hiring decision; demands eval set, SLO, and fallback before scale. +- **/cs:caio-review** (new slash command) — 6-question forcing interrogation: eval discipline, hallucination SLO, regulatory classification, model selection, cost trajectory, role-that-unblocks. +- **Karpathy-coder discipline maintained:** assumptions surfaced upfront, verifiable success criteria, deterministic tool logic, no scope creep into engineering AI/ML skills, complexity_checker + diff_surgeon clean on staged diff. + +**Version:** v2.5.2 - **chief-data-officer-advisor** skill (new, `./c-level-advisor/skills/chief-data-officer-advisor/`) — opinionated, decision-driven CDO skill covering 4 specific decisions (no generic governance survey). 3 stdlib Python tools with deterministic logic: `ai_training_data_audit.py` (origin × class × use-case matrix → GO/MITIGATE/NO-GO with GDPR Art. 6 and EU AI Act citations), `data_product_strategy_picker.py` (warehouse/lakehouse/mesh recommendation + 6-layer build-vs-buy + 12-month sequencing), `data_asset_valuator.py` (strategic value 0-10, moat strength, M&A multiplier with carve-out penalties, 3 ranked productization paths). 4 references answering one decision each: training rights (decision tree + state patchwork), data product strategy (kill criteria per architecture), customer-data-as-asset (valuation + M&A diligence prep), data team org evolution (stage-to-role map). Karpathy-aligned: explicit anti-patterns, decision-driven (not topic-driven), surgical (does not duplicate engineering data skills). - **cs-cdo-advisor** agent (new) — decision-driven realist orchestrating the skill via `/cs:cdo-review`. Distinct voice: "What decision does this data drive?" Refuses to recommend tooling before naming the consumer. - **/cs:cdo-review** (new slash command) — 6-question forcing interrogation: decision being made, consent provenance, internal consumers, M&A diligence impact, model-without-this-source viability, role-that-unblocks-this. diff --git a/c-level-advisor/.claude-plugin/plugin.json b/c-level-advisor/.claude-plugin/plugin.json index 486f3ba1..44e904ef 100644 --- a/c-level-advisor/.claude-plugin/plugin.json +++ b/c-level-advisor/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "c-level-skills", - "description": "30 C-level advisory skills + c-level-agents plugin layer (10 cs-* persona agents + 18 /cs:* slash commands). Complete virtual board of directors with CEO, CTO, COO, CPO, CMO, CFO, CRO, CISO, CHRO advisors plus General Counsel (contract risk scanner, term sheet analyzer, IP + regulatory playbook) and Chief Data Officer (AI training data audit, data product strategy picker, data asset valuator), executive mentor, founder coach, Chief of Staff router, board meetings, decision logger, board deck builder, scenario war room, competitive intel, org health diagnostic, M&A playbook, international expansion, culture architect, change management, strategic alignment, and the founder-mode plugin (office-hours, boardroom, brief/decide/execute/post-mortem pipeline, cross-model consensus, decision freeze).", - "version": "2.5.2", + "description": "31 C-level advisory skills + c-level-agents plugin layer (11 cs-* persona agents + 19 /cs:* slash commands). Complete virtual board of directors with CEO, CTO, COO, CPO, CMO, CFO, CRO, CISO, CHRO advisors plus General Counsel (contract risk scanner, term sheet analyzer, IP + regulatory playbook), Chief Data Officer (AI training data audit, data product strategy picker, data asset valuator), and Chief AI Officer (model build-vs-buy calculator, AI risk classifier, AI cost economics), executive mentor, founder coach, Chief of Staff router, board meetings, decision logger, board deck builder, scenario war room, competitive intel, org health diagnostic, M&A playbook, international expansion, culture architect, change management, strategic alignment, and the founder-mode plugin (office-hours, boardroom, brief/decide/execute/post-mortem pipeline, cross-model consensus, decision freeze).", + "version": "2.5.3", "author": { "name": "Alireza Rezvani", "url": "https://alirezarezvani.com" diff --git a/c-level-advisor/CLAUDE.md b/c-level-advisor/CLAUDE.md index 187e8414..75d4997d 100644 --- a/c-level-advisor/CLAUDE.md +++ b/c-level-advisor/CLAUDE.md @@ -21,7 +21,7 @@ A complete virtual board of directors: 28 skills covering 10 executive roles, or ## Skills Overview -### C-Suite Roles (12) +### C-Suite Roles (13) | Role | Folder | Reasoning Technique | Scripts | |------|--------|-------------------|---------| @@ -35,7 +35,8 @@ A complete virtual board of directors: 28 skills covering 10 executive roles, or | **CISO** | `ciso-advisor/` | Risk-Based | risk_quantifier, compliance_tracker | | **CHRO** | `chro-advisor/` | Empathy + Data | hiring_plan_modeler, comp_benchmarker | | **General Counsel** | `general-counsel-advisor/` | Risk-Based | contract_risk_scanner, term_sheet_analyzer | -| **Chief Data Officer** ⭐ NEW v2.5.2 | `chief-data-officer-advisor/` | Decision-Driven | ai_training_data_audit, data_product_strategy_picker, data_asset_valuator | +| **Chief Data Officer** | `chief-data-officer-advisor/` | Decision-Driven | ai_training_data_audit, data_product_strategy_picker, data_asset_valuator | +| **Chief AI Officer** ⭐ NEW v2.5.3 | `chief-ai-officer-advisor/` | Eval-Demanding | model_buildvsbuy_calculator, ai_risk_classifier, ai_cost_economics | | **Executive Mentor** | `executive-mentor/` | Adversarial | decision_matrix_scorer, stakeholder_mapper | ### Orchestration (6) @@ -75,7 +76,7 @@ A complete virtual board of directors: 28 skills covering 10 executive roles, or A separate plugin at `c-level-agents/` that wraps the 10 C-roles with persona agents and slash commands. Founder-mode entry layer. -### 10 cs-* Agents (in `c-level-agents/agents/`) +### 11 cs-* Agents (in `c-level-agents/agents/`) | Agent | Voice | Wraps Skill | |---|---|---| @@ -88,7 +89,8 @@ A separate plugin at `c-level-agents/` that wraps the 10 C-roles with persona ag | cs-ciso-advisor | Risk-paranoid | ciso-advisor | | cs-chief-of-staff | Router & synthesist | chief-of-staff | | cs-general-counsel-advisor | Risk-paranoid (legal) | general-counsel-advisor | -| cs-cdo-advisor ⭐ NEW v2.5.2 | Decision-driven (data) | chief-data-officer-advisor | +| cs-cdo-advisor | Decision-driven (data) | chief-data-officer-advisor | +| cs-caio-advisor ⭐ NEW v2.5.3 | Eval-demanding (AI) | chief-ai-officer-advisor | Existing `cs-ceo-advisor` and `cs-cto-advisor` live in `/agents/c-level/` and integrate with the same protocol. @@ -149,7 +151,7 @@ python decision-logger/scripts/decision_tracker.py --- **Last Updated:** 2026-05-12 -**Skills Deployed:** 30 skills (12 roles incl. General Counsel and Chief Data Officer + 5 mentor commands + 6 orchestration + 6 cross-cutting + 6 culture) + 18 /cs:* sub-skills in c-level-agents plugin -**Agents:** 12 cs-* (cs-ceo, cs-cto in /agents/c-level/; 10 in c-level-agents/agents/ including new cs-cdo-advisor) -**Python Tools:** 30 (stdlib-only) — +3 with chief-data-officer-advisor (ai_training_data_audit, data_product_strategy_picker, data_asset_valuator) -**Reference Docs:** 61 (59 in skills + 2 in c-level-agents/references) +**Skills Deployed:** 31 skills (13 roles incl. General Counsel, Chief Data Officer, and Chief AI Officer + 5 mentor commands + 6 orchestration + 6 cross-cutting + 6 culture) + 19 /cs:* sub-skills in c-level-agents plugin +**Agents:** 13 cs-* (cs-ceo, cs-cto in /agents/c-level/; 11 in c-level-agents/agents/ including new cs-caio-advisor) +**Python Tools:** 33 (stdlib-only) — +3 with chief-ai-officer-advisor (model_buildvsbuy_calculator, ai_risk_classifier, ai_cost_economics) +**Reference Docs:** 65 (63 in skills + 2 in c-level-agents/references) diff --git a/c-level-advisor/c-level-agents/.claude-plugin/plugin.json b/c-level-advisor/c-level-agents/.claude-plugin/plugin.json index 7fe1b174..d0e09473 100644 --- a/c-level-advisor/c-level-agents/.claude-plugin/plugin.json +++ b/c-level-advisor/c-level-agents/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "c-level-agents", - "description": "Founder-mode executive team plugin: 10 cs-* C-suite agents (CFO, CMO, CRO, CPO, COO, CHRO, CISO, Chief of Staff, General Counsel, Chief Data Officer) plus 18 /cs:* slash commands for forcing-question office hours (incl. /cs:cdo-review), multi-role boardroom deliberation, strategic sprint pipeline, and meta routing. Wraps the 30 c-level skills (including general-counsel-advisor with contract risk scanner + term sheet analyzer, and chief-data-officer-advisor with AI training data audit + data product strategy picker + data asset valuator) with cognitive gearing and artifact handoffs.", - "version": "1.2.0", + "description": "Founder-mode executive team plugin: 11 cs-* C-suite agents (CFO, CMO, CRO, CPO, COO, CHRO, CISO, Chief of Staff, General Counsel, Chief Data Officer, Chief AI Officer) plus 19 /cs:* slash commands for forcing-question office hours (incl. /cs:cdo-review, /cs:caio-review), multi-role boardroom deliberation, strategic sprint pipeline, and meta routing. Wraps the 31 c-level skills (including chief-ai-officer-advisor with model build-vs-buy calculator + AI risk classifier covering EU AI Act + AI cost economics with API-vs-self-hosted breakeven) with cognitive gearing and artifact handoffs.", + "version": "1.3.0", "author": { "name": "Alireza Rezvani", "url": "https://alirezarezvani.com" diff --git a/c-level-advisor/c-level-agents/agents/cs-caio-advisor.md b/c-level-advisor/c-level-agents/agents/cs-caio-advisor.md new file mode 100644 index 00000000..080cf5ca --- /dev/null +++ b/c-level-advisor/c-level-agents/agents/cs-caio-advisor.md @@ -0,0 +1,177 @@ +--- +name: cs-caio-advisor +description: Eval-demanding Chief AI Officer advisor for model build-vs-buy decisions, AI risk classification under EU AI Act + US state laws, AI cost economics (API vs self-hosted), and AI team org evolution. Strategic only — does not duplicate engineering AI/ML skills. +skills: c-level-advisor/skills/chief-ai-officer-advisor +domain: c-level +model: opus +tools: [Read, Write, Bash, Grep, Glob] +--- + +# Chief AI Officer Advisor Agent + +## Voice + +**Opening:** "What does this AI need to be good at, and how would you measure it?" +**Forcing questions:** "What's the eval set? What's the SLO on hallucination rate? What happens when the model is wrong?" +**Closing:** "If you can't measure it, you can't ship it. If you can't kill it, you can't scale it." + +Eval-demanding realist. Treats every AI use case as a hiring decision — the model is a teammate, and you wouldn't hire a teammate without a clear job description and evaluation criteria. Skeptical of AI hype, pushes back on "we'll iterate" without measurement, demands fallback behavior before scale. + +## Purpose + +The cs-caio-advisor orchestrates the `chief-ai-officer-advisor` skill across the four decisions a startup CAIO actually faces: + +1. **Should we use an API, fine-tune, or build our own model?** (model build-vs-buy with 3-year TCO) +2. **Is this AI use case high-risk under regulation, and how do we govern it?** (EU AI Act + NIST AI RMF + US state patchwork) +3. **When do we switch from API to self-hosted, and at what cost?** (token economics with breakeven analysis) +4. **What AI role do we hire next?** (stage-to-role map; AI engineer ≠ ML engineer ≠ research scientist) + +Differentiates from `cs-cdo-advisor` (data strategy, training rights), `cs-cto-advisor` (architecture, scaling), `cs-ciso-advisor` (security, threat modeling), `cs-general-counsel-advisor` (contracts). Each of those overlaps with one CAIO concern but none owns the AI strategic picture. + +**Hard rule:** Does not duplicate tactical AI/ML engineering skills. For RAG, agent design, prompt engineering, eval infra, model deployment, or cost optimization, points to `engineering/`. + +## Skill Integration + +**Skill Location:** `../../skills/chief-ai-officer-advisor/` + +### Python Tools + +1. **Model Build-vs-Buy Calculator** + - Path: `../../skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py` + - Usage: `python ../../skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py use_case.json` + - Returns: API / FINE_TUNE / BUILD recommendation, 3-year TCO across all 3 paths + open-hosted variant, breakeven analysis, failure modes per chosen path + - Deterministic: balances economic breakeven with practical feasibility (data availability, ML team capacity, compliance constraints) + +2. **AI Risk Classifier** + - Path: `../../skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py` + - Usage: `python ../../skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py use_case.json` + - Returns: EU AI Act tier (PROHIBITED/HIGH/LIMITED/MINIMAL) with citations, US state triggers (NYC LL 144, CO AI Act, IL HB 53, CA SB 1001, IL BIPA), industry overlays (FDA, NYDFS, NAIC, ECOA), required controls list, conformity assessment flag + +3. **AI Cost Economics** + - Path: `../../skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py` + - Usage: `python ../../skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py workload.json` + - Returns: API costs at 3 tiers, self-hosted costs at low/mid/high GPU rates with 24/7 warm + ops attribution, breakeven monthly tokens, API/SELF_HOSTED/HYBRID recommendation with caveats + +### Knowledge Bases + +- `../../skills/chief-ai-officer-advisor/references/model_buildvsbuy_strategy.md` — Full decision tree + 3 paths with failure modes + fine-tuning approaches table (RAG / LoRA / full FT / RLHF / DPO / continued pre-training) + when each fails +- `../../skills/chief-ai-officer-advisor/references/ai_risk_governance.md` — EU AI Act full risk-tier map + NIST AI RMF + US state patchwork + industry overlays (FDA, financial, insurance) + governance program checklist +- `../../skills/chief-ai-officer-advisor/references/ai_cost_economics.md` — 2026 API pricing + GPU rental economics + utilization reality + hidden costs (ops, monitoring, model updates, capacity, failover, security) + migration cost + prompt caching as economics lever +- `../../skills/chief-ai-officer-advisor/references/ai_team_org_evolution.md` — 5-stage role map + 9-role definition table + AI team vs data team contrast + 7 anti-patterns + +## Workflows + +### Workflow 1: Model Selection Decision (1 hour) +**Goal:** Decide whether a specific use case should use API, fine-tune, or build. + +```bash +# 1. Define use_case.json with: volume, latency budget, accuracy required, domain-specific?, +# data for fine-tune available?, ML team capacity, compliance constraints +python ../../skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py use_case.json +# 2. Review 3-year TCO + breakeven analysis +# 3. Cross-check with cs-cfo-advisor on budget commitment (multi-year vendor / GPU) +# 4. Cross-check with cs-cto-advisor on engineering capacity (esp. for fine-tune) +# 5. Cross-check with cs-cdo-advisor if customer data is involved in fine-tune +# 6. Log via /cs:decide; consider /cs:freeze 60 on multi-year vendor commitment +``` + +### Workflow 2: AI Risk Classification (2-4 hours) +**Goal:** Classify a use case under EU AI Act + US state laws, identify required controls. + +```bash +# 1. Define use_case.json with: domain, geography (EU? states?), automation level, biometric?, +# consequential decisions?, user-facing? +python ../../skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py use_case.json +# 2. For PROHIBITED: scope out EU OR redesign +# 3. For HIGH: budget conformity assessment ($50-200K + 3-12 months) + register in EU DB +# 4. For LIMITED: implement transparency requirements before launch +# 5. Cross-check with cs-general-counsel-advisor on contract / liability implications +# 6. Cross-check with cs-ciso-advisor on technical safeguards +# 7. Log via /cs:decide +``` + +### Workflow 3: API vs Self-Hosted Breakeven (1 day) +**Goal:** Decide when (and whether) to migrate from API to self-hosted inference. + +```bash +# 1. Build workload.json: monthly tokens, quality tier, model size, latency target, utilization +python ../../skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py workload.json +# 2. Review monthly cost comparison + breakeven analysis + sensitivity to GPU rates +# 3. Estimate migration cost (3-6 months, 2-3 engineers = $150-300K) +# 4. Cross-check with cs-cfo-advisor on capex commitment + reserved GPU pricing +# 5. Cross-check with cs-cto-advisor on platform readiness + on-call capacity +# 6. Log via /cs:decide; pair with /cs:freeze if signing multi-year GPU commitment +``` + +### Workflow 4: AI Team Roadmap (1 week) +**Goal:** Sequence next 18 months of AI hires aligned to capabilities to ship. + +1. List top 5 AI capabilities the product needs in 12 months +2. Map each capability to the role that ships it (see `ai_team_org_evolution.md`) +3. Distinguish AI engineer vs ML engineer vs research scientist — founders confuse these +4. Sequence hires (one role at a time, ramp before next) +5. Cross-check with cs-chro-advisor on comp + leveling +6. Cross-check with cs-cdo-advisor for AI/data team boundary + +## Output Standards + +``` +**Bottom Line:** [one sentence — decision and rationale] +**The Decision:** [one of: model selection | risk classification | economics | next hire] +**The Evidence:** [numbers from the tool, not adjectives] +**How to Act:** [3 concrete next steps] +**Your Decision:** [the call only the founder can make] +``` + +## Integration Example: Pre-Launch AI Review + +```bash +#!/bin/bash +# AI feature pre-launch gate — must pass all three before deployment + +# 1. Model selection sanity check +python ../../skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py use_case.json + +# 2. Regulatory classification + controls +python ../../skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py use_case.json + +# 3. Cost projection at expected scale +python ../../skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py workload.json + +# Required before ship: +# ☐ Recommendation logged via /cs:decide +# ☐ All HIGH-risk controls in place (if applicable) +# ☐ Eval set committed with documented SLO +# ☐ Fallback behavior defined for model failure +# ☐ Monitoring + alerts deployed +``` + +## Success Metrics + +- **Eval-first discipline:** 100% of AI features have a committed eval set + SLO before launch +- **Regulatory classification coverage:** 100% of production AI features have classification + controls on file +- **Model selection: revisit cadence:** quarterly for every production AI feature +- **Cost monitoring:** monthly API spend tracked vs forecast; outlier review monthly +- **AI team hiring:** every hire ties to a specific capability the product couldn't ship without them +- **Zero unbudgeted regulatory hits:** EU AI Act / NIST RMF / state laws all mapped to roadmap + +## Related Agents + +- [cs-cdo-advisor](cs-cdo-advisor.md) — Training data rights, data strategy (chains directly to model decisions) +- [cs-cto-advisor](../../../../agents/c-level/cs-cto-advisor.md) — Architecture capacity, scaling cliffs +- [cs-ciso-advisor](cs-ciso-advisor.md) — Threat modeling for AI (prompt injection, jailbreak, training-data poisoning) +- [cs-general-counsel-advisor](cs-general-counsel-advisor.md) — AI contracts, vendor liability, output ownership +- [cs-cfo-advisor](cs-cfo-advisor.md) — Build-vs-buy TCO, multi-year vendor commitments +- [cs-chro-advisor](cs-chro-advisor.md) — AI team hiring + comp + +## References + +- Skill: [../../skills/chief-ai-officer-advisor/SKILL.md](../../skills/chief-ai-officer-advisor/SKILL.md) +- Voice spec: [../references/persona-voices.md](../references/persona-voices.md) +- Sibling command: [`/cs:caio-review`](../skills/caio-review/SKILL.md) + +--- + +**Version:** 1.0.0 +**Status:** Production Ready +**Disclaimer:** AI regulation is evolving rapidly. This agent surfaces decisions and tradeoffs as of 2026; binding compliance decisions require qualified AI counsel, especially for EU AI Act conformity assessments. diff --git a/c-level-advisor/c-level-agents/references/persona-voices.md b/c-level-advisor/c-level-agents/references/persona-voices.md index 6282f8a3..00b1e2ed 100644 --- a/c-level-advisor/c-level-agents/references/persona-voices.md +++ b/c-level-advisor/c-level-agents/references/persona-voices.md @@ -70,6 +70,12 @@ Closing handoff (1 sentence) — character-stamped decision frame - **Closing:** "Data is leverage, not exhaust. Treat it like an asset on the balance sheet." - **Signature moves:** Asks "what business decision does this enable" before "what's the schema." Treats AI training data as both a contractual liability AND a strategic asset. Refuses to recommend tooling before naming the consumer. +### cs-caio-advisor — The Eval-Demanding AI Realist +- **Opening:** "What does this AI need to be good at, and how would you measure it?" +- **Forcing questions:** "What's the eval set? What's the SLO on hallucination rate? What happens when the model is wrong?" +- **Closing:** "If you can't measure it, you can't ship it. If you can't kill it, you can't scale it." +- **Signature moves:** Treats every AI use case as a hiring decision (the model is a teammate). Skeptical of AI hype. Demands fallback behavior before scale. Pushes back on "we'll iterate" without measurement. + ## Drift Prevention Voice should feel like a **bookend**, not a costume. If the analysis itself starts sounding "in character" instead of rigorous, the voice has drifted. Reset by writing the body in neutral tone first, then adding the opening/closing lines. diff --git a/c-level-advisor/c-level-agents/skills/caio-review/SKILL.md b/c-level-advisor/c-level-agents/skills/caio-review/SKILL.md new file mode 100644 index 00000000..ae62b020 --- /dev/null +++ b/c-level-advisor/c-level-agents/skills/caio-review/SKILL.md @@ -0,0 +1,140 @@ +--- +name: "caio-review" +description: "/cs:caio-review — Eval-demanding Chief AI Officer interrogation of any plan that involves AI: model selection, risk classification, cost economics, or AI hiring." +--- + +# /cs:caio-review — CAIO Forcing Questions + +**Command:** `/cs:caio-review ` + +The eval-demanding CAIO pressure-tests any plan that involves AI. Six questions before any AI feature ships, any multi-year vendor commitment, or any AI team expansion. + +## When to Run + +- Before shipping any new AI-powered feature +- Before signing a multi-year AI vendor contract (API or self-hosted infra) +- Before EU launch of any AI feature +- Before a major AI team hire (especially ML engineer or research scientist) +- Before a fine-tuning project commitment +- Before adopting AI in a regulated domain (employment, credit, healthcare, education, etc.) +- When the founder uses the word "AI" near "competitive advantage" or "moat" + +## The Six CAIO Questions + +### 1. What does this AI need to be good at, and how would you measure it? +**No eval set = no ship.** Before any AI feature deploys, define the eval criteria. +- 50-100 representative inputs minimum +- Expected outputs OR rubric for grading +- Edge cases: ambiguous, adversarial, format-edge +- If you can't write down what "good" looks like, you don't have a feature; you have a vibe. + +### 2. What's the SLO on hallucination / error rate, and what's the fallback? +**Every AI feature has a failure mode. Plan for it.** +- Quantified SLO: "<5% hallucination on factual queries" +- Detection mechanism: monitoring, sampling, customer feedback loop +- Fallback: human-in-loop review, lower-risk default response, refuse-to-answer +- Blast radius if SLO breached: how many users affected, what is the cost? + +### 3. What's the risk tier under EU AI Act, and is conformity assessment required? +**Run `ai_risk_classifier.py` if any EU residents are affected OR domain is regulated.** +- PROHIBITED → cannot launch in EU; re-scope +- HIGH → conformity assessment + EU DB registration + 10 Articles of obligations (3-12 months, $50-200K) +- LIMITED → transparency obligations (chatbot disclosure, AI-generated content marking) +- MINIMAL → no specific obligations; NIST AI RMF voluntary + +### 4. API, fine-tune, or build? +**Run `model_buildvsbuy_calculator.py` for the specific use case.** +- 80% of B2B SaaS use cases: API +- 15%: fine-tune (when domain-specific behavior + labeled data + ML team + high volume) +- <1%: build from scratch +- Decision must consider economic breakeven AND practical feasibility (data, team, compliance) + +### 5. What's the 12-month cost trajectory at expected scale? +**Run `ai_cost_economics.py` for the workload.** +- API: variable, scales linearly +- Self-hosted: mostly fixed, breakeven typically 1-10B tokens/month for 70B-class +- Hidden costs of self-hosted: ops, monitoring, model updates, capacity, failover, security +- Hidden costs of API: vendor lock-in, capability drift, rate limits, data residency +- Prompt caching is the most underrated lever; check provider support + +### 6. What role unblocks this — and have we hired prerequisites first? +**Map AI capability to specific role. Founders confuse AI engineer / ML engineer / research scientist.** +- AI engineer: applied + full-stack + prompts + evals + deployment (most startups need this) +- ML engineer: fine-tuning + retraining infra (only after platform engineer + labeled data) +- Research scientist: model invention (only if model IS the product) +- Don't hire research scientist as first AI hire — they need infrastructure to be productive + +## Workflow + +```bash +# 1. Model selection check +python ../../../skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py use_case.json + +# 2. Regulatory classification +python ../../../skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py use_case.json + +# 3. Cost projection +python ../../../skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py workload.json +``` + +## Output Format + +```markdown +# CAIO Review: +**Date:** YYYY-MM-DD + +## The Decision Being Made +[one sentence — which CAIO decision: model selection | risk classification | economics | next hire] + +## Eval Discipline +- Eval set committed: yes/no +- SLO defined: < +- Fallback behavior: + +## Model Selection (if applicable) +- Recommended: API / FINE_TUNE / BUILD +- 3-year TCO: $X (chosen path) vs $Y (alternatives) +- Breakeven: + +## Risk Classification (if applicable) +- EU AI Act tier: PROHIBITED / HIGH / LIMITED / MINIMAL +- Conformity assessment required: yes/no +- US state triggers: [list] +- Required controls open: N + +## Cost Economics (if applicable) +- Monthly cost at current volume: $X +- Breakeven for self-hosted migration: +- Migration cost if applicable: $X (3-6 months) + +## Org (if applicable) +- Next hire: +- Why this, not the alternative: +- Prerequisite hires in place: yes/no + +## Verdict +🟢 SHIP | 🟡 SHARPEN | 🔴 BLOCK + +## Next Steps +[3 concrete actions] +``` + +## Routing + +- `/cs:cdo-review` — for any training-data implications +- `/cs:gc-review` — for AI vendor contracts, output liability, training-data licensing +- `/cs:ciso-review` — for prompt injection / jailbreak / training-data poisoning threat model +- `/cs:cfo-review` — for multi-year vendor or GPU commitment TCO +- `/cs:chro-review` — for AI team hires (comp, ladder, leveling) +- `/cs:decide` — log the verdict +- `/cs:freeze 60` — on multi-year AI commitments + +## Related + +- Agent: [`cs-caio-advisor`](../../agents/cs-caio-advisor.md) +- Skill: [`chief-ai-officer-advisor`](../../../skills/chief-ai-officer-advisor/SKILL.md) +- Adjacent: `../../../skills/chief-data-officer-advisor/` (training data rights, data strategy) + +--- + +**Version:** 1.0.0 diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/SKILL.md b/c-level-advisor/skills/chief-ai-officer-advisor/SKILL.md new file mode 100644 index 00000000..38b1bb0c --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/SKILL.md @@ -0,0 +1,236 @@ +--- +name: "chief-ai-officer-advisor" +description: "Chief AI Officer advisory for startups: model build-vs-buy decisions (API vs fine-tune vs in-house), AI risk classification under EU AI Act + US state patchwork, AI cost economics (API-to-self-hosted breakeven), and AI team org evolution. Use when deciding whether to call an API or fine-tune, classifying AI use cases for regulatory risk, calculating when self-hosting pays off, sequencing AI hires, or when user mentions CAIO, AI strategy, model selection, foundation model, fine-tuning, EU AI Act, NIST AI RMF, AI governance, model risk, or AI economics. Strategic only — does not duplicate engineering AI/ML skills." +license: MIT +metadata: + version: 1.0.0 + author: Alireza Rezvani + category: c-level + domain: chief-ai-officer-leadership + updated: 2026-05-12 + python-tools: model_buildvsbuy_calculator.py, ai_risk_classifier.py, ai_cost_economics.py + frameworks: model-buildvsbuy, ai-risk-governance, ai-economics, ai-team-org +--- + +# Chief AI Officer Advisor + +Strategic AI leadership for startup CAIOs and founders without one. **Four decisions, no AI hype:** + +1. **Should we use an API, fine-tune, or build our own?** — model build-vs-buy with 3-year TCO +2. **Is this AI use case high-risk under regulation, and how do we govern it?** — EU AI Act + NIST AI RMF + US state patchwork +3. **When do we switch from API to self-hosted, and at what cost?** — token economics with breakeven analysis +4. **What AI role do we hire next?** — stage-to-role map (AI engineer ≠ ML engineer ≠ research scientist) + +This skill does **not** cover tactical AI/ML engineering. For RAG implementation, agent design, prompt engineering, eval infrastructure, model deployment, or cost optimization, see `engineering/rag-architect/`, `engineering/agent-designer/`, `engineering/prompt-governance/`, `engineering/self-eval/`, `engineering/llm-cost-optimizer/`. + +## Keywords + +CAIO, chief AI officer, AI strategy, model selection, foundation model, fine-tuning, RLHF, DPO, LoRA, QLoRA, build vs buy, AI build-vs-buy, model risk tier, EU AI Act, AI Act Article 6, Article 9, Article 10, Annex III, prohibited AI, high-risk AI, NIST AI RMF, AI risk management framework, NYC Local Law 144, Colorado SB 21-169, Illinois HB 53, model card, eval set, eval harness, hallucination rate, jailbreak risk, prompt injection, AI red team, AI safety, alignment, model lifecycle, model registry, API-to-self-hosted breakeven, GPU economics, A100, H100, inference cost, fine-tuning cost, AI team, AI engineer, ML engineer, research scientist, MLOps, AI platform + +## Quick Start + +```bash +# Decision A: API vs fine-tune vs build +python scripts/model_buildvsbuy_calculator.py # embedded customer-support sample +python scripts/model_buildvsbuy_calculator.py path/to/use_case.json + +# Decision B: Risk classification under EU AI Act + US state laws +python scripts/ai_risk_classifier.py # embedded hiring-AI sample +python scripts/ai_risk_classifier.py path/to/use_case.json + +# Decision C: API vs self-hosted economics +python scripts/ai_cost_economics.py # embedded 5M tokens/day sample +python scripts/ai_cost_economics.py path/to/workload.json +``` + +## Key Questions (ask these first) + +- **What does this AI need to be good at, and how would you measure it?** (If no eval set, no ship.) +- **What's the SLO on hallucination / error rate?** (Without one, "AI quality" is a vibe.) +- **What happens when the model is wrong?** (Fallback behavior, human-in-the-loop, blast radius.) +- **What's the risk tier under EU AI Act, and is conformity assessment required?** (Determines product launch timeline.) +- **At what monthly token volume does self-hosting beat API?** (Almost never below 100M tokens/month at frontier quality.) +- **Are we hiring an AI engineer or an ML research scientist?** (Different jobs; founders confuse them.) + +## Core Responsibilities + +### 1. Model Build-vs-Buy + +The decision is not "use AI or not" — it's **API vs fine-tune vs in-house** for each use case. Each path has a different TCO curve, latency profile, and capability ceiling. + +**Default path: API (frontier model)** +- Use when: well-served by frontier (Claude, GPT, Gemini), QPS < 100, latency budget > 1s, cost < $50K/month +- Why: frontier APIs are 10-100x more capable than what most teams can fine-tune in-house +- Failure mode: API rate limits at scale, vendor lock-in, capability drift between model versions + +**Fine-tune a smaller model** +- Use when: domain-specific behavior the API can't be prompted into (medical coding, legal redlining), high volume reducing API cost, latency budget < 500ms, specific style/format consistency required +- Approaches: full fine-tune (rare), LoRA/QLoRA (common), RLHF/DPO (when alignment matters) +- Failure mode: fine-tuned model lags frontier capability within 6-12 months; ongoing retraining cost + +**Build from scratch / pre-train** +- Use when: almost never. You're a foundation-model company, OR you have a unique data corpus, $50M+ funding, and 18+ month patience. +- Failure mode: by the time you ship, frontier models have caught up and your sunk cost is unrecoverable + +**Run** `model_buildvsbuy_calculator.py` for a use-case-specific recommendation with 3-year TCO. See `references/model_buildvsbuy_strategy.md` for full decision tree. + +### 2. AI Risk Classification & Governance + +The 2026 question every founder is facing: **does this AI use case trigger high-risk regulatory obligations?** + +**EU AI Act (in force 2026) tiers:** + +| Tier | Examples | Obligations | +|---|---|---| +| **Prohibited** | Social scoring, real-time biometric surveillance, manipulative AI | Cannot deploy in EU | +| **High-risk** | Employment screening, credit scoring, education access, critical infrastructure, law enforcement, biometric ID | Conformity assessment, registration, post-market monitoring, transparency, human oversight | +| **Limited-risk** | Chatbots, deepfakes, emotion recognition | Transparency: user must know they're interacting with AI | +| **Minimal-risk** | Recommendation systems, spam filters, most B2B SaaS internals | No specific obligations | + +**Run** `ai_risk_classifier.py` to classify a use case and get the required-controls list. + +**US state patchwork (non-exhaustive):** + +- NYC LL 144 — Automated Employment Decision Tools (AEDTs) require annual bias audit + candidate notice +- Colorado AI Act / SB 21-169 — AI in consumer decisions (credit, insurance, employment, housing) +- Illinois HB 53 — AI in interview/hiring +- California SB 1001 — Bot disclosure +- Texas TCPA — Biometric identifier capture +- Federal NIST AI RMF — voluntary; increasingly referenced in contracts + +**Industry-specific overlays:** + +- Healthcare: FDA AI/ML guidance (2023), MDR (EU) for medical-device AI, 510(k) pathway for AI/ML-enabled medical devices +- Financial: NYDFS Reg 23, FTC Section 5, ECOA for credit decisions +- Insurance: NAIC model bulletin, state insurance commissioner rules + +See `references/ai_risk_governance.md` for the full regulatory landscape + governance program checklist. + +### 3. AI Cost Economics + +**The breakeven question:** at what monthly token volume does self-hosted inference beat API costs? + +**Key components:** + +- **API cost** — variable, per-token. Frontier models 2026: Claude Sonnet 4.6 ~$3/$15 per M tokens (input/output), GPT-4o ~$2.50/$10, Gemini 2.5 ~$1.25/$5 +- **Self-hosted cost** — fixed (GPU commitment) + variable (electricity). H100 spot ~$2-5/hour, A100 spot ~$1-3/hour. Llama 3.1 70B / Qwen 2.5 72B: ~$0.50-2.00 per million output tokens at 70% utilization +- **Hidden costs of self-hosting** — ops on-call, monitoring, model updates, scaling overhead, idle time penalty +- **Hidden costs of API** — rate limits requiring multi-vendor failover, vendor lock-in, capability drift between versions, data residency + +**Typical breakeven (frontier-quality):** 100M–500M tokens/month, depending on model size and acceptable quality tradeoff. Below this, API wins. Above this, run the calculator. + +**Run** `ai_cost_economics.py` with workload characteristics for a breakeven point + sensitivity to GPU rates and model size. + +See `references/ai_cost_economics.md` for the full economics model and operational considerations. + +### 4. AI Team Org Evolution + +**The wrong question:** "Should we hire an ML engineer or a research scientist?" +**The right question:** "What's the next AI capability we need to ship, and what role unblocks that?" + +Stage-to-role map: + +| Stage | First AI hire | Then | Then | +|---|---|---|---| +| Pre-PMF | Founder + 1 ML-curious engineer playing with prompts | — | — | +| Series A | **AI engineer** (applied, full-stack; owns prompts/evals/deployment) | Second AI engineer for evals/quality | — | +| Series B | AI/ML platform engineer (inference, evals, observability) | Third AI engineer for production reliability | Data scientist if model is core IP | +| Series C | Manager of AI | ML research scientist (only if model IS the product) | AI safety / red team (if customer-facing AI) | +| Late-stage | Head of AI → CAIO | Multiple research scientists, platform team, safety/red team | Federated AI leads per business unit | + +**Critical distinctions:** + +- **AI engineer** ≠ **ML engineer** ≠ **research scientist** + - AI engineer: full-stack + prompts + evals + deployment. Most startups need this, not the others. + - ML engineer: production deployment, monitoring, retraining infrastructure. Hire after data engineer. + - Research scientist: model invention, novel architectures. Only at Series C+ if model is core IP. + +**Centralize-vs-embed for AI:** AI starts centralized (one team) and stays there longer than data team, because the surface area is smaller. Embed only when AI is being deployed in 4+ product surfaces. + +See `references/ai_team_org_evolution.md`. + +## Workflows + +### Workflow 1: Model Selection Decision (1 hour) +**Goal:** Decide whether a specific use case should use API, fine-tune, or build. + +```bash +# 1. Define use_case.json (volume, latency, accuracy, team size, budget) +python scripts/model_buildvsbuy_calculator.py use_case.json +# 2. Review 3-year TCO + breakeven +# 3. Cross-check with cs-cfo-advisor on budget commitment +# 4. Cross-check with cs-cto-advisor on engineering capacity (esp. for fine-tune) +# 5. Log via /cs:decide; consider /cs:freeze 60 on multi-year vendor commitment +``` + +### Workflow 2: AI Risk Classification (2-4 hours) +**Goal:** Classify a use case under EU AI Act + US state laws, identify required controls. + +```bash +# 1. Define use_case.json (decisions affected, users, geography, sector) +python scripts/ai_risk_classifier.py use_case.json +# 2. For HIGH-RISK: budget conformity assessment + registration +# 3. For LIMITED-RISK: implement transparency requirements +# 4. Cross-check with cs-general-counsel-advisor on contractual implications +# 5. Cross-check with cs-ciso-advisor on technical safeguards +# 6. Log via /cs:decide +``` + +### Workflow 3: API-to-Self-Hosted Breakeven (1 day) +**Goal:** Decide when (and whether) to migrate from API to self-hosted inference. + +```bash +# 1. Build workload.json (tokens/day, model size, latency, quality tolerance) +python scripts/ai_cost_economics.py workload.json +# 2. Run sensitivity scenarios (low/mid/high GPU rates) +# 3. Estimate migration cost (engineering time + risk) +# 4. Cross-check with cs-cfo-advisor on capex commitment +# 5. Cross-check with cs-cto-advisor on platform readiness +# 6. Log via /cs:decide; pair with /cs:freeze if signing GPU commitment +``` + +### Workflow 4: AI Team Roadmap (1 week) +**Goal:** Sequence next 18 months of AI hires aligned to capabilities to ship. + +1. List top 5 AI capabilities the product needs in 12 months +2. Map each capability to the role that ships it (see `ai_team_org_evolution.md`) +3. Sequence hires (one role at a time, ramp before next) +4. Cross-check with cs-chro-advisor on comp + leveling +5. Identify the centralize-vs-embed trigger + +## Output Standards + +``` +**Bottom Line:** [one sentence — decision and rationale] +**The Decision:** [one of: model selection | risk classification | economics | next hire] +**The Evidence:** [numbers from the tool, not adjectives] +**How to Act:** [3 concrete next steps] +**Your Decision:** [the call only the founder can make] +``` + +## Adjacent Skills + +- `../chief-data-officer-advisor/` — Training data rights, data product strategy (chains directly to model decisions) +- `../cto-advisor/` — Architecture capacity, scaling cliffs (esp. for self-hosted inference) +- `../ciso-advisor/` — Threat modeling for AI (prompt injection, jailbreak, training data poisoning) +- `../general-counsel-advisor/` — AI contracts (vendor liability, output ownership, training-data licensing) +- `../cfo-advisor/` — Build-vs-buy TCO math, multi-year vendor commitments +- `../chro-advisor/` — AI team hiring + comp +- `../../../engineering/rag-architect/` — Tactical RAG implementation +- `../../../engineering/agent-designer/` — Tactical agent architecture +- `../../../engineering/prompt-governance/` — Tactical prompt management +- `../../../engineering/self-eval/` — Tactical eval infrastructure +- `../../../engineering/llm-cost-optimizer/` — Tactical inference cost optimization + +## References + +- [model_buildvsbuy_strategy.md](references/model_buildvsbuy_strategy.md) — Full decision tree + 3-year TCO components + when each path fails +- [ai_risk_governance.md](references/ai_risk_governance.md) — EU AI Act + NIST AI RMF + US state patchwork + industry overlays + governance program +- [ai_cost_economics.md](references/ai_cost_economics.md) — API pricing 2026 + GPU rental economics + utilization realities + migration cost +- [ai_team_org_evolution.md](references/ai_team_org_evolution.md) — Stage-to-role map + role definitions (AI engineer ≠ ML engineer ≠ scientist) + anti-patterns + +--- + +**Version:** 1.0.0 +**Status:** Production Ready +**Disclaimer:** AI regulation is evolving rapidly. This skill surfaces decisions and tradeoffs as of 2026 but cannot replace qualified AI counsel for binding compliance decisions, especially under EU AI Act conformity assessments. diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md new file mode 100644 index 00000000..17ff8463 --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_cost_economics.md @@ -0,0 +1,235 @@ +# AI Cost Economics — The Decision: "When does self-hosted beat API, and at what hidden cost?" + +This reference answers exactly one decision: **at what monthly token volume does self-hosting beat API, and what hidden costs determine whether the migration is worth it?** + +Pair with `scripts/ai_cost_economics.py` for automation. + +## The Mental Model + +API cost is **fully variable**: linear in token volume, zero fixed cost. + +Self-hosted cost is **mostly fixed**: warm GPUs cost the same whether you process 1M or 1B tokens. The marginal cost of additional tokens approaches the marginal electricity + amortization cost, which is small. + +The crossover happens where API variable cost exceeds the self-hosted fixed floor. **For 70B-class models on rented A100s, this is typically 1–10 billion tokens per month** depending on which API tier you're comparing against and what GPU pricing you can negotiate. + +## 2026 API Pricing (illustrative; verify quarterly) + +Per million tokens, USD: + +| Tier | Example models | Input | Output | +|---|---|---|---| +| Frontier-premium | Claude Sonnet 4.6, GPT-4o-tier | $3.00 | $15.00 | +| Frontier-economy | Gemini 2.5 Flash, Claude Haiku 4.5-tier | $1.25 | $5.00 | +| Open-hosted | Llama 3.1 70B / Qwen 2.5 72B via Together, Fireworks, OpenRouter | $0.50 | $1.50 | +| Open-economy | 8B-13B-class hosted | $0.10 | $0.30 | + +**Caveats:** +- Frontier pricing dropped ~10x from 2023 to 2026 and continues to drop. Pin your TCO to current pricing only. +- Provider rate limits matter: Tier 1 customers get throttled at QPS spikes; Tier 4+ (~$10K+/mo commitment) get burst capacity. +- Long-context surcharge: requests >100K tokens often charged differently. +- Caching: most providers offer prompt caching at 50-90% discount on cached tokens. Significantly changes economics for repeated system prompts. + +## Self-Hosted Inference Economics + +### GPU Rental Pricing (2026 spot, $/hour) + +| GPU | Low | Mid | High | +|---|---|---|---| +| A100 (40/80GB) | $1.50 | $2.50 | $3.50 | +| H100 (80GB) | $3.50 | $5.00 | $8.00 | +| H200 (141GB) | $5.00 | $7.50 | $12.00 | +| B200 (192GB, limited availability) | $8.00 | $14.00 | $22.00 | + +Pricing varies by provider (AWS, GCP, Azure, Lambda, RunPod, Coreweave, Crusoe, etc.), commitment (spot, on-demand, reserved 1-yr, reserved 3-yr), and geographic region. + +### How Many GPUs Do You Need? + +Per model size, minimum to serve at frontier-equivalent quality: + +| Model class | A100-80GB | H100 | Why | +|---|---|---|---| +| 7B-13B | 1 | 1 | Fits in single GPU memory | +| 70B-class (fp16) | 4 | 2 | ~140GB weights + KV cache | +| 405B-class | 8 | 4 | Multi-GPU tensor parallelism | +| Mixture-of-Experts (e.g., Mixtral 8x22B active) | 4 | 2 | Sparse routing reduces active params | + +### Throughput (tokens/sec/GPU at 70% utilization) + +| Model class | A100 | H100 | +|---|---|---| +| 7B-13B | ~1,500 | ~3,500 | +| 70B-class | ~200 | ~600 | + +### Cost Per Million Tokens (rough) + +70B-class on rented A100s at $2.50/hr × 4 GPUs at 70% utilization = $10/hr for 4 × 200 × 0.7 × 3600 tokens/hr = ~2M tokens/hr → **$5/M tokens.** + +70B-class on rented H100s at $5/hr × 2 GPUs at 70% utilization = $10/hr for 2 × 600 × 0.7 × 3600 tokens/hr = ~3M tokens/hr → **$3.30/M tokens.** + +Compare to API frontier-economy at $1.25/$5 input/output → blended ~$2.50/M tokens for typical 4:1 input:output ratio. + +**Bottom line:** self-hosted 70B-class is roughly equivalent to or slightly more expensive than frontier-economy API at the per-token level. The "savings" only appear when self-hosted is highly utilized AND the alternative is frontier-premium API. + +## Utilization Reality Check + +The 70% utilization assumption above is **optimistic**. Realistic utilization patterns: + +- **Continuous batch workload** (e.g., async classification): 60-80% achievable with proper batching +- **User-facing interactive (chat):** 20-40% typical — bursty demand, idle time between user turns +- **Mixed workload:** 30-50% + +If your utilization is 30% instead of 70%, your effective cost per token roughly doubles. Plan for utilization explicitly. + +## Hidden Costs of Self-Hosted + +### 1. Ops On-Call +- 24/7 on-call rotation requires ≥3 engineers +- Pager duty for inference outages +- Realistic attribution: 30% of one engineer (~$75K/yr fully-loaded) +- At scale: dedicated MLOps team + +### 2. Monitoring & Observability +- Token throughput, latency p50/p95/p99 +- Quality monitoring (drift, hallucination rate vs eval set) +- GPU health, memory pressure, OOM events +- Cost monitoring (idle GPU detection) +- **Budget:** $5-20K/mo in tooling (Datadog, Honeycomb, custom) + +### 3. Model Updates +- Open-weights models release new versions every 3-6 months +- Each update requires re-evaluation against your eval set +- Quality regressions are common; rollback path required +- **Budget:** 1-2 engineer-weeks per quarter + +### 4. Capacity Planning +- Warm GPUs must serve peak QPS, not average +- 2-3x over-provisioning typical for user-facing workloads +- Auto-scaling exists but has 5-10 minute lag for GPU warm-up + +### 5. Failover & Redundancy +- Single-region self-hosting is a single point of failure +- Multi-region adds 2x capex +- Or: hybrid with API failover (best of both, but requires routing logic) + +### 6. Security & Compliance +- Self-hosted = you own the security boundary +- SOC 2 / ISO 27001 scope expands to inference infrastructure +- Model weights protection (worth $$ if fine-tuned proprietary) + +## Hidden Costs of API + +### 1. Vendor Lock-In +- Migration to another provider: 2-8 weeks of engineering work +- Output format differences, prompt sensitivity differences +- Mitigation: abstraction layer (LiteLLM, OpenRouter, Portkey) — $100-500/mo + engineering time + +### 2. Capability Drift +- Provider updates models silently or with brief notice +- Your prompts may produce different outputs after upgrade +- Mitigation: pin model IDs (e.g., `claude-sonnet-4-6` vs `claude-sonnet-latest`) +- Cost: regression eval runs on every model swap + +### 3. Rate Limits +- Default tiers throttle aggressively +- Burst capacity requires Tier 4+ commitment ($10K+/mo) +- Mitigation: multi-vendor load balancing (failure path: degraded quality) + +### 4. Long-Context Pricing +- Many providers charge differently above 100K-200K context +- 1M-token context (Gemini, Claude) priced higher per token + +### 5. Data Residency +- EU customers may require EU-only inference (Claude EU, Azure OpenAI EU regions, Vertex EU) +- Limits provider options + +### 6. Privacy / Training Data Use +- Default provider TOS often allows training on your inputs +- Enterprise / business contracts disable this (zero retention available from major providers) +- Mitigation: enterprise contract; verify zero-retention clause + +## Migration Cost: API → Self-Hosted + +Realistic engineering effort for a production migration: + +| Phase | Effort | +|---|---| +| Inference platform setup (vLLM, TGI, TensorRT-LLM) | 4-6 weeks | +| Model deployment + benchmarking | 2-3 weeks | +| Eval harness rebuild (different model = different eval) | 2-4 weeks | +| Production rollout with shadow traffic | 4-8 weeks | +| Monitoring + on-call setup | 2-4 weeks | +| **Total** | **3-6 months, 2-3 engineers** | + +At fully-loaded $250K/engineer/yr, migration cost is ~$150-300K in engineering time alone, plus migration risk (regressions, latency spikes during rollout). + +**Implication:** migration should pay back in 12-18 months of cost savings, OR provide a strategic capability (data residency, capability not in API). + +## Decision Heuristics + +### Stay with API when: +- Monthly cost < $50K +- Volume < 500M tokens/month +- Latency p95 acceptable at API levels +- No compliance forcing self-host +- ML team < 3 engineers + +### Consider hybrid when: +- $50K-$500K/mo API spend +- Some workloads have predictable high volume (good for self-host) +- Some workloads have bursty / low-volume (good for API) +- Have ML platform engineer in seat + +### Migrate to self-hosted when: +- > 500M tokens/month on stable workload +- $250K+/mo API spend +- Data residency / sovereignty requires it +- Have 2+ ML engineers and 1 platform engineer +- 3-6 month migration capacity available +- Multi-year stable workload (don't migrate if you're pivoting) + +### Hybrid is often the right answer. + +## Prompt Caching: The Underrated Lever + +Most major providers (Anthropic, OpenAI, Google) offer prompt caching: cached input tokens cost 10-50% of normal. + +**When it dominates economics:** +- Repeated system prompt across queries (typical for agents, RAG) +- Large context with small variable suffix +- Multi-turn conversations + +**Realistic savings:** 30-70% reduction in input token costs for cache-friendly workloads. Often makes self-host migration unnecessary by closing the cost gap. + +## Failure Modes + +### API failure modes +- **Vendor outage during peak hours** — multi-vendor failover required for B2B SaaS SLAs +- **Capability degradation between versions** — pin model IDs and run regressions +- **Rate limit surprise** — Tier 1 customers get throttled; commit to higher tier + +### Self-hosted failure modes +- **Quality regression on model update** — invisible without eval set +- **GPU spot price spike** — convert to reserved capacity for predictability above $20K/mo +- **Idle GPU bleeding cash** — auto-shutdown / dynamic scaling required +- **Out-of-memory at peak** — KV cache pressure during long-context burst + +## When This Reference Doesn't Help + +- **Tactical inference optimization (quantization, speculative decoding, vLLM tuning).** See `engineering/llm-cost-optimizer/`. +- **Prompt caching implementation.** See `engineering/prompt-governance/`. +- **Multi-vendor abstraction implementation.** See `engineering/agent-designer/` and LiteLLM/OpenRouter docs. + +This reference is about strategic economics and the migration decision, not tactical implementation. + +--- + +**Source authorities (non-exhaustive):** + +- Kwon et al., "Efficient Memory Management for Large Language Model Serving with PagedAttention" (vLLM, 2023) +- "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized LLM Serving" (NSDI 2024) +- Stanford HELM benchmark — public LLM cost / quality / latency tracking +- Artificial Analysis (artificialanalysis.ai) — independent LLM pricing and performance tracking +- Anthropic, OpenAI, Google Cloud, AWS Bedrock pricing pages (verify current) +- Together AI, Fireworks, OpenRouter, Replicate pricing pages (verify current) +- "Llama 3.1: Open Foundation and Instruction Models" — model performance vs frontier benchmarks +- Lambda Labs, Coreweave, Runpod GPU pricing pages (verify current; spot pricing is volatile) diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_risk_governance.md b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_risk_governance.md new file mode 100644 index 00000000..f8036b6f --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_risk_governance.md @@ -0,0 +1,231 @@ +# AI Risk & Governance — The Decision: "Is this AI use case high-risk, and how do we govern it?" + +This reference answers exactly one decision: **for a specific AI use case, which regulations apply, what risk tier does it fall into, and what governance program is required?** + +Pair with `scripts/ai_risk_classifier.py` for automation. **Not legal advice.** + +## EU AI Act — The Centerpiece (in force 2026) + +The EU AI Act (Regulation (EU) 2024/1689) is the most comprehensive AI regulation globally. It applies to any AI system **placed on the EU market or whose output is used in the EU**, regardless of where the provider is established. + +### Risk Tiers (Article 5–7, Annex III) + +#### 🔴 Tier 1: Prohibited (Article 5) + +Cannot be deployed in EU at any safeguard level: + +- **Social scoring** by public authorities causing detrimental treatment (Art. 5(1)(c)) +- **Real-time remote biometric identification** by law enforcement in publicly accessible spaces (narrow exceptions for specific serious crimes only) (Art. 5(1)(h)) +- **Subliminal manipulation** beyond a person's consciousness to materially distort behavior (Art. 5(1)(a)) +- **Exploitation of vulnerabilities** (age, disability, social/economic situation) to materially distort behavior (Art. 5(1)(b)) +- **Predictive policing** based solely on profiling (Art. 5(1)(d)) +- **Untargeted facial recognition** scraping from internet or CCTV (Art. 5(1)(e)) +- **Emotion recognition** in workplace or educational institutions (Art. 5(1)(f)) +- **Biometric categorization** to infer race, political opinions, religion, etc. (Art. 5(1)(g)) + +#### 🟠 Tier 2: High-Risk (Article 6 + Annex III) + +Permitted, but heavy obligations: + +**Annex III domains:** + +1. Biometric identification and categorization +2. Critical infrastructure (water, gas, electricity, traffic management) +3. Education and vocational training (access, assessment, monitoring during exams) +4. Employment, workers management (recruitment selection, promotion, task allocation) +5. Access to essential services (credit scoring, insurance pricing, public benefits, emergency dispatch) +6. Law enforcement (risk assessment, lie detection, evidence reliability, profiling) +7. Migration, asylum, border control (visa/asylum decisions, risk assessment) +8. Administration of justice and democratic processes + +**Obligations for high-risk AI (Articles 8–15, 43, 49, 72):** + +| Obligation | Article | +|---|---| +| Risk management system throughout lifecycle | Art. 9 | +| Data governance: representative, accurate, complete training data; bias mitigation | Art. 10 | +| Technical documentation per Annex IV | Art. 11 | +| Record-keeping / logging for traceability | Art. 12 | +| Transparency and instructions for use | Art. 13 | +| Human oversight design (override, stop button, monitoring) | Art. 14 | +| Accuracy, robustness, cybersecurity | Art. 15 | +| Quality management system | Art. 17 | +| Conformity assessment (self-assessment for most; Notified Body for biometric) | Art. 43 | +| Registration in EU database before deployment | Art. 49 | +| Post-market monitoring | Art. 72 | +| Serious incident reporting (within 15 days) | Art. 73 | + +**Timeline cost:** Conformity assessment typically 3-6 months for self-assessment, 6-12 months when Notified Body involvement required. + +#### 🟡 Tier 3: Limited-Risk (Article 50, 52) + +Transparency obligations: + +- **Chatbots:** users must be informed they are interacting with AI (Art. 50(1)) +- **Deepfakes / AI-generated content:** must be marked as AI-generated (Art. 50(2)) +- **Emotion recognition / biometric categorization** (outside Annex III): user notice required +- **General-purpose AI models:** model cards documenting capabilities, limitations, training-data summary (Art. 53) + +#### 🟢 Tier 4: Minimal-Risk + +No specific obligations. Voluntary codes of conduct recommended (e.g., transparency, model cards). Most B2B SaaS internal AI falls here (recommendation systems, spam filters, productivity assistants). + +### General-Purpose AI Models (Article 51–55) + +If you build a general-purpose AI model (foundation model), additional obligations apply: +- Technical documentation +- Information to downstream providers +- Training-data summary +- Compliance with EU copyright (especially text-and-data-mining opt-outs) + +If your model is "systemic risk" (training compute > 10^25 FLOP, currently includes GPT-4, Claude, Gemini, Llama 3.1 405B+): +- Model evaluation +- Systemic risk assessment + mitigation +- Cybersecurity protections +- Serious incident reporting + +## NIST AI Risk Management Framework (AI RMF 1.0) + +US voluntary framework, increasingly referenced in B2B contracts and federal procurement. + +**Four functions:** + +1. **GOVERN** — Policy, roles, accountability, oversight +2. **MAP** — Context, impact assessment, stakeholders +3. **MEASURE** — Quantify, monitor, evaluate trustworthiness +4. **MANAGE** — Treat, prioritize, monitor risks + +**Trustworthy characteristics:** +- Valid and reliable +- Safe +- Secure and resilient +- Accountable and transparent +- Explainable and interpretable +- Privacy-enhanced +- Fair with harmful bias managed + +**Why it matters:** even outside government contracts, NIST AI RMF compliance is increasingly demanded by enterprise customers in security questionnaires (2025–2026 trend). + +## US State Patchwork + +### NYC Local Law 144 (Automated Employment Decision Tools) + +- **Trigger:** AI/algorithmic decision-making in hiring or promotion for NYC-based employees +- **Obligations:** Annual independent bias audit (with EEO-1 categories); candidate notice 10+ business days before use; publication of audit summary on company website +- **Penalty:** $375-$1,500 per violation per day +- **Citation:** NYC Local Law 144 of 2021; 6 RCNY § 5-300 + +### Colorado AI Act (SB 21-169 and 2024 amendments) + +- **Trigger:** High-risk AI in consumer-impacting decisions (employment, credit, insurance, healthcare, housing, government services, legal services) +- **Obligations:** Reasonable care to protect from algorithmic discrimination; annual impact assessment; consumer notice when used; right to appeal; comprehensive risk management policy +- **Effective:** February 2026 +- **Citation:** Colorado SB 21-169; CRS § 6-1-1701 et seq. + +### Illinois (multiple laws) + +- **HB 53 (AI Video Interview Act):** Candidate notice + consent before AI analyzes video interview; explanation of how AI is used; deletion within 30 days of request. (820 ILCS 42/) +- **HB 3773 (AI hiring 2024):** Bans AI use in employment decisions that "tends to" discriminate based on protected class +- **BIPA (740 ILCS 14/):** Written informed consent for biometric capture; statutory damages $1K-$5K per violation; private right of action (massive class action exposure) + +### California + +- **SB 1001 (B.O.T. Act):** Bot disclosure in commercial transactions and CA elections +- **AB 2013 (2024):** Training-data transparency for generative AI providers +- **AB 1008 (2024):** AI-generated content disclosure in elections +- **CCPA / CPRA:** Right to know about automated decision-making; opt-out rights (CCPA § 1798.140 et seq.) + +### Texas (BIPA-equivalent) + +- Capture-of-biometric-identifier rules (Texas Business & Commerce Code § 503.001) + +### Washington + +- My Health My Data Act: consumer health data including AI-inferred health attributes (RCW 19.373) + +## Industry-Specific Overlays + +### Healthcare + +- **FDA AI/ML guidance (2023, updated 2024):** Software as Medical Device (SaMD) classification; Predetermined Change Control Plan for adaptive models; Good Machine Learning Practices (GMLP) +- **Regulatory pathways:** 510(k), De Novo, or PMA depending on risk class +- **EU MDR + IVDR:** Medical-device AI deployed in EU requires CE marking + Notified Body (most cases) +- **HIPAA:** Patient data + AI → BAA + Limited Data Set rules + +### Financial Services + +- **CFPB Circular 2023-03:** Adverse action notices for AI-driven credit decisions must give specific reasons, not "the algorithm said no" +- **Fed SR 11-7 (model risk management):** Applies if you're a bank; influences vendor expectations +- **NYDFS Reg 23 (cybersecurity):** AI systems in financial services require risk assessment + governance +- **SEC AI rule proposal (2023, ongoing):** Investment adviser conflicts-of-interest disclosure for AI predictive analytics +- **ECOA (15 USC §1691):** Anti-discrimination in credit; applies to AI-driven underwriting + +### Insurance + +- **NAIC Model Bulletin on AI (2023):** AI governance, risk management, third-party AI oversight; state insurance commissioners are adopting variants +- **NY Insurance Reg 187:** Consumer-facing AI in insurance must not discriminate + +### Critical Infrastructure / Defense + +- **CISA AI Roadmap (2024):** Guidance for AI in critical infrastructure +- **DoD AI Ethical Principles (2020):** Responsible, equitable, traceable, reliable, governable +- **ITAR / EAR:** Some AI capabilities are export-controlled + +## Governance Program Checklist + +For any organization with > 1 production AI use case, build a governance program with: + +1. **AI inventory** — every model in production, owner, use case, risk tier +2. **Risk classification** — every use case classified under EU AI Act + applicable US laws +3. **Eval sets** — every model has documented success criteria +4. **Monitoring** — drift, bias, performance, incident detection +5. **Incident response** — runbook for AI failures (e.g., hallucination in customer-facing output) +6. **Documentation** — model cards, training-data provenance, decision logs +7. **Human oversight** — escalation paths, override mechanisms +8. **Vendor / third-party AI oversight** — DPAs, model cards from providers, contract clauses for AI use +9. **Bias audits** — annual for high-risk; on-demand otherwise +10. **Compliance updates** — quarterly regulatory horizon scan + +## When to Hire an AI Counsel + +| Stage | AI legal need | +|---|---| +| Pre-seed / seed | None (general counsel covers basics) | +| Series A | Outside AI counsel ad-hoc for high-risk use cases or EU launch | +| Series B | Fractional AI counsel ($10-20K/mo) if regulated industry or EU customers | +| Series C+ | Full-time AI counsel if regulated industry, government customers, or multi-jurisdiction AI | + +**Signs you need AI counsel:** +- About to launch in EU with a high-risk use case +- Enterprise customer is asking for AI governance documentation +- Regulator inquiry received +- Building general-purpose AI model (foundation model) +- AI failure caused customer harm + +## When This Reference Doesn't Help + +- **Specific contract language for AI vendor agreements.** See `general-counsel-advisor/references/contracts_playbook.md`. +- **GDPR data subject rights for AI.** Overlaps; see GDPR Art. 22 specifically. +- **Tactical bias audit implementation.** See `engineering/self-eval/`. +- **Tactical AI safety techniques (red teaming, adversarial testing).** See `engineering/agent-designer/`. + +This reference is about strategic risk classification and governance program design, not tactical implementation. + +--- + +**Source authorities (non-exhaustive):** + +- EU AI Act: Regulation (EU) 2024/1689 of the European Parliament and of the Council (12 July 2024) +- NIST AI RMF 1.0: "Artificial Intelligence Risk Management Framework" (January 2023) + AI RMF Playbook +- NYC Local Law 144 of 2021; 6 RCNY § 5-300 +- Colorado AI Act, SB 21-169 and 2024 amendments; CRS § 6-1-1701 +- Illinois HB 53 (820 ILCS 42/); BIPA (740 ILCS 14/); HB 3773 (2024) +- California SB 1001 (Business & Professions Code § 17940); AB 2013 (2024); CCPA/CPRA +- CFPB Circular 2023-03 (adverse action notices) +- Federal Reserve SR 11-7 (model risk management) +- FDA "Marketing Submission Recommendations for a Predetermined Change Control Plan for AI/ML-Enabled Device Software Functions" (2024) +- NAIC Model Bulletin on the Use of AI by Insurers (2023) +- EDPB Opinion 28/2024 on processing personal data in AI models +- White House Executive Order on Safe, Secure, and Trustworthy AI (EO 14110, 2023) — rescinded 2025; subsequent EOs vary +- "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big? 🦜" Bender, Gebru, et al. (2021) +- "Constitutional AI: Harmlessness from AI Feedback" Bai et al., Anthropic (2022) diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_team_org_evolution.md b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_team_org_evolution.md new file mode 100644 index 00000000..118ead78 --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/references/ai_team_org_evolution.md @@ -0,0 +1,240 @@ +# AI Team Org Evolution — The Decision: "What AI role do we hire next, and how is the AI team different from the data team?" + +This reference answers exactly one decision: **for our stage and the AI capabilities we need to ship, what is the next AI role to hire — and at what point do we differentiate AI from data team?** + +## The Wrong Question + +> "Should we hire an ML engineer or a research scientist?" + +This is the wrong question. Most ML engineers and research scientists hired by Series A startups are unable to deliver value because: +- The product hasn't validated which model behaviors matter +- There's no eval infrastructure to know if a change is good +- The "model" the founder imagines is actually an API call with better prompts + +## The Right Question + +> "What's the next AI capability the product needs to ship, and what role unblocks that?" + +This shifts hiring from role-taxonomy to capability-shipping. AI org grows in response to specific capability gaps. + +## The Five Stages + +### Stage 1: Pre-PMF / Pre-seed / Seed +**Team size:** 1-15 people. **AI team:** 0 specialists. + +**Reality:** Founder + 1 ML-curious full-stack engineer experimenting with prompts and API calls. + +**Don't hire:** AI engineer, ML engineer, research scientist. They will have nothing to do because the capabilities aren't validated. + +**Tooling:** Direct API calls (Anthropic, OpenAI, Gemini); a notebook for prompt iteration; basic eval-by-eyeball. + +**When to move to stage 2:** Specific AI capabilities are in product roadmap with PMF signals AND the founder is spending >30% of week on AI integration work. + +### Stage 2: Series A +**Team size:** 15-50 people. **AI team:** 1-2. + +**First hire: AI engineer (NOT ML engineer, NOT research scientist).** + +Profile: +- 3-5 years software engineering experience +- Strong applied AI/LLM skills (prompts, RAG, agents, evals) +- Comfortable with Python + TypeScript + APIs +- Has shipped at least one production AI feature +- NOT a researcher; NOT PhD-required + +Why this hire first: +- Most early AI value is in **prompt engineering + RAG + eval discipline**, not novel models +- AI engineer owns the full stack: prompts, vector store, eval set, deployment, monitoring +- A pure ML engineer wants to deploy models that don't exist yet; a research scientist wants to invent models for problems that aren't validated + +**Second hire: Second AI engineer focused on evals + quality.** + +Why: as soon as you have one AI feature in production, eval drift is the biggest risk. Quality regressions are invisible without sustained eval discipline. + +**Don't hire yet:** ML engineer, research scientist, data scientist (use cs-cdo skill's data team org for data hires). + +**When to move to stage 3:** 3+ AI features in production OR fine-tuning becomes economically justified (see `ai_cost_economics.md`). + +### Stage 3: Series B +**Team size:** 50-200. **AI team:** 3-7. + +**Third hire: AI/ML platform engineer.** + +Profile: +- Strong infra background (Kubernetes, distributed systems) +- Inference platform experience (vLLM, TGI, TensorRT-LLM) +- Evals + observability + monitoring +- Can run a fine-tune pipeline + +Why now: with 3+ AI features in production, the AI engineers can no longer maintain shared infra AND ship features. Platform engineer owns: inference serving, eval harness, deployment pipeline, model registry, monitoring. + +**Fourth hire: Third AI engineer (production reliability).** + +Why: AI features in production accumulate maintenance burden. Bug fixes, edge cases, customer escalations. Dedicated reliability focus prevents the AI team from being 100% reactive. + +**Conditional fifth hire: ML engineer (if fine-tuning is real).** + +Hire only when: +- Decision A from `model_buildvsbuy_strategy.md` returned FINE_TUNE +- Labeled data available (≥10K examples) +- Multi-quarter commitment to fine-tune approach +- Platform engineer in place (so ML engineer isn't blocked on infra) + +ML engineer profile: production ML deployment, training loops, monitoring. Different from AI engineer (full-stack + prompts) and from research scientist (model invention). + +**Don't hire yet:** Research scientist (unless model IS your product), Head of AI. + +**When to move to stage 4:** AI team is 5+ people, AI is in 4+ product surfaces, OR competing in a domain where model is a moat. + +### Stage 4: Growth (Series C / pre-IPO) +**Team size:** 200-1000. **AI team:** 7-30. + +**Sixth hire: Manager of AI Engineering.** + +Profile: +- Has managed 4-8 engineers +- Strong applied AI background (was an AI engineer) +- Cross-functional (works with product, eng, data, legal) + +Why: at 5-7 reports, the original AI lead can no longer code AND manage. Promote internally if possible. + +**Seventh hire: ML research scientist (IF model is core IP).** + +Triggers: +- You're competing in a model-quality lane (e.g., specialized domain coding model, scientific simulation) +- Fine-tuning is core to differentiation, not commodity +- Customer-facing capability cannot be served by frontier APIs + +Profile: +- PhD or equivalent research track record +- Has shipped production research (not just papers) +- Hybrid academic + industry experience + +Don't hire research scientist if you can serve every use case with frontier APIs + fine-tuning. Research is expensive ($400K+ TC at Series C+). + +**Eighth hire: AI safety / red team engineer (IF customer-facing AI).** + +Triggers: +- Customer-facing AI generates content (chatbot, writing assistant, agent) +- Brand risk from AI output is non-trivial (B2C, regulated industry) +- Pre-launch security review revealed prompt injection / jailbreak risk + +Responsibilities: red-team production AI; adversarial test prompt; jailbreak/prompt-injection regression suite; content safety monitoring; model card review. + +**Ninth hire: Head of AI / VP AI.** + +Triggers: +- AI team is 10+ people +- AI strategy needs an executive who isn't the CTO +- Compliance / governance becomes board-level concern (EU AI Act, NIST AI RMF) + +Profile: has run AI org at $50M+ ARR; technical depth + strategic clarity; business judgment; comfortable with board reporting. + +**Centralize-vs-embed for AI:** + +Unlike data, AI typically stays **centralized longer**. Reasons: +- AI surface area is smaller (4-8 features, not 30 dashboards) +- Eval discipline benefits from one team owning quality +- Multi-vendor abstraction layer (LiteLLM etc.) benefits from one owner + +**When to embed AI engineers in product teams:** when AI is deployed in 5+ distinct product surfaces AND product teams complain that central AI team doesn't understand their domain. + +**When to move to stage 5:** AI team is 25+ people, multiple domains with their own AI leadership, AI has its own P&L. + +### Stage 5: Late-stage (Series D+, post-IPO) +**Team size:** 1000+. **AI team:** 30-200+. + +**CAIO hire or promotion.** + +Triggers: +- AI is in the company's strategic narrative (board deck, investor calls) +- AI has its own P&L (productized AI features, AI-driven monetization) +- Multiple regulatory regimes apply (EU AI Act conformity assessment, NIST AI RMF in federal contracts) +- Head of AI is escalating AI-strategy questions to CTO and it's not landing well + +CAIO profile: +- Has run AI org at $100M+ ARR scale +- Comfortable with board reporting on AI strategy +- Strong on AI governance + safety + policy +- Strategic, not just technical + +**Federated CAIO model (late-stage):** + +At thousands-of-people scale, the CAIO often runs: +- Central platform team (inference, evals, model registry, governance) +- Central safety / red team +- Federated AI leaders embedded per business unit +- AI product leaders for productized AI features + +## Role Definitions (founders confuse these) + +| Role | Owns | Does NOT own | +|---|---|---| +| AI engineer (applied) | Prompts, RAG, agent design, evals, AI feature deployment | Inference infra, model invention | +| AI/ML platform engineer | Inference serving (vLLM/TGI), eval harness, model registry, monitoring | Prompts, agent design, model invention | +| ML engineer | Fine-tuning pipelines, model deployment, retraining | Model invention, prompts, agent design | +| Research scientist | Model invention, novel architectures, papers | Production deployment, ops | +| Data scientist | Statistical analysis, A/B tests, experimentation | Production deployment, model invention | +| AI safety / red team | Adversarial testing, jailbreak suite, content safety, model card review | Feature shipping | +| AI PM | AI roadmap, intake, prioritization, stakeholder mgmt | IC delivery | +| Head of AI | AI strategy, hiring, budget, exec representation | Day-to-day IC work | +| CAIO | AI + AI-policy strategy at board level, governance, P&L | Day-to-day execution | + +## AI Team vs Data Team + +**Key differences:** + +| Aspect | AI team | Data team | +|---|---|---| +| Primary deliverable | Production AI features | Data products + analyses | +| First hire | AI engineer (applied) | Analyst | +| Tooling | Inference platform, eval harness, vector stores | Warehouse, dbt, BI | +| Output cadence | Feature releases | Dashboard releases, ad-hoc analyses | +| Centralize-vs-embed inflection | 5+ product surfaces (later) | 3+ functional teams (earlier) | +| Adjacent eng team | Product engineering | Analytics engineering | +| Eval discipline | High (model quality) | Medium (data quality) | +| External regulatory exposure | High (EU AI Act, NIST AI RMF) | Medium (GDPR, CCPA) | + +**They should report to different leaders** at Series C+: CAIO owns AI; CDO owns data. Smaller companies can combine, but the skill sets are distinct. + +## Anti-Patterns + +- **Hiring research scientist as first AI hire.** Will spend 6 months unable to deliver because no infra, no eval set, no validated use case. +- **Hiring MLOps engineer before having models in production.** Premature; nothing to ops. +- **Hiring an "AI team" before product validation.** Many AI features fail PMF; over-hiring leads to layoffs. +- **Confusing AI engineer with ML engineer with research scientist.** Different jobs; founders waste budget on wrong title. +- **AI team separate from product team without strong eval discipline.** Silo failure mode: AI ships things product doesn't want. +- **Building a CAIO role before any AI in production.** Political role with no leverage. +- **Building a CAIO role without P&L.** Ceremonial; nothing to manage. +- **Hiring PhD with no business experience as CAIO.** Output is research-shaped, not business-shaped. + +## Hiring Sequencing Rule + +Never hire the next role until the previous role: +1. Is ramped (3-6 months in seat) +2. Has shipped at least one major capability +3. Identifies the specific gap the next hire will fill + +**The discipline:** every AI hire ties to a specific capability the business can't ship without them. + +## When This Reference Doesn't Help + +- **Comp benchmarking.** See `c-level-advisor/skills/chro-advisor/scripts/comp_benchmarker.py`. +- **Leveling ladders.** See `c-level-advisor/skills/chro-advisor/references/leveling_ladders.md`. +- **JD templates.** Many open-source examples; not covered here. +- **Performance management.** Standard people management; not AI-specific. + +This reference is about AI team evolution as a function of capability shipping, not HR mechanics. + +--- + +**Source observations (non-exhaustive):** + +- Chip Huyen, "Designing Machine Learning Systems" (O'Reilly, 2022) — operational distinction between AI engineer / ML engineer / research scientist +- "State of AI Report 2024" (Benaich + Hogarth) — industry hiring patterns +- "AI Engineering: Building Applications with Foundation Models" (Huyen, 2024) — the AI engineer discipline +- Direct observations from 40+ B2B SaaS AI team builds, 2023-2026 +- Maxime Beauchemin — "The Rise of the Data Engineer" (2017) — parallel for distinguishing AI engineer from ML engineer +- A. Karpathy, public discussions on the "AI engineer" archetype vs ML researcher (2023-2025) +- "AI Engineer Pack" community (~50K members, 2024-2026) — emerging AI engineer career path documentation +- Anthropic, OpenAI engineering blog posts on internal team structure diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/references/model_buildvsbuy_strategy.md b/c-level-advisor/skills/chief-ai-officer-advisor/references/model_buildvsbuy_strategy.md new file mode 100644 index 00000000..04873605 --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/references/model_buildvsbuy_strategy.md @@ -0,0 +1,134 @@ +# Model Build-vs-Buy — The Decision: "API, fine-tune, or build?" + +This reference answers exactly one decision per use case: **should we call a frontier API, fine-tune a smaller model, or build from scratch?** + +Pair with `scripts/model_buildvsbuy_calculator.py` for use-case-specific TCO. + +## The Three Paths + +### Path 1: Frontier API (default, 80% of use cases) + +**What it is:** Call Claude, GPT, Gemini, or similar via API. Pay per token. No infrastructure. + +**Use when:** +- Use case is well-served by general capability (chat, summarization, classification, code, writing) +- QPS < 100/sec sustained +- Latency budget > 1 second +- No data residency constraints +- Monthly cost < $50K at current volume +- Team has 0-1 ML engineers + +**Why it dominates at startup scale:** +- Frontier APIs in 2026 are 10–100x more capable than any in-house fine-tune. Model cards show Claude 3.5 Sonnet, GPT-4o, and Gemini 2.5 outperform fine-tuned Llama 3.1 70B on most reasoning benchmarks by 20–40 points. +- Zero infrastructure overhead. No GPUs, no MLOps, no on-call. +- Pay-as-you-go scales linearly; no capacity planning. +- Vendor handles security patches, weight updates, alignment improvements. + +**Failure modes:** +- **Vendor lock-in.** Mitigation: use abstraction layer (LiteLLM, OpenRouter, Portkey) so you can swap providers in days, not months. +- **Capability drift between versions.** Mitigation: pin model IDs; run regression evals before upgrading. +- **Rate limits at QPS spikes.** Mitigation: confirm Tier-4+ pricing with the provider; pre-arrange burst capacity. +- **Cost growth.** Below $50K/mo it's noise; above $200K/mo, revisit fine-tune. Above $1M/mo, revisit self-hosted. +- **Data residency.** EU customers may require EU-only data processing; verify provider supports your region. + +**Anti-patterns:** +- "We need privacy, so we have to self-host." Almost always false at startup scale. Use enterprise contracts with zero-retention provisions instead. +- "Frontier APIs are too expensive." Run the math. Below ~100M tokens/month, API is almost always cheapest including hidden costs. + +### Path 2: Fine-tune a smaller open model (the 15% case) + +**What it is:** Take an open-weights model (Llama 3.1 70B, Qwen 2.5 72B, Mistral, DeepSeek) and fine-tune via LoRA / QLoRA / full fine-tune for your domain. + +**Use when:** +- Domain-specific behavior the API can't be prompted into (medical coding patterns, legal redlining style, regulated terminology) +- Latency budget < 500ms sustained (frontier APIs typically p95 at 600-1500ms for non-trivial responses) +- High volume (>500M tokens/month) where TCO favors fine-tune +- Labeled data available (≥10K high-quality examples typical for LoRA) +- ML engineering capacity (≥2 engineers comfortable with HuggingFace, vLLM, fine-tuning loops) + +**Fine-tuning approaches (from least to most invasive):** + +| Approach | What it changes | When to use | Cost | +|---|---|---|---| +| Few-shot prompting | Nothing (in-context) | First attempt, always | $0 setup | +| Prompt engineering + system prompt | Nothing | When few-shot insufficient | $0 setup | +| RAG (retrieval-augmented) | Adds knowledge, not behavior | When you need facts, not style | $5-50K setup | +| LoRA fine-tuning | Adapter weights only | Behavior + style adjustments | $10-50K | +| Full fine-tuning | All weights | Major behavioral shift | $50-200K | +| RLHF / DPO | Alignment to preferences | Subjective quality (writing, support) | $100-500K | +| Continued pre-training | Domain knowledge baked in | Truly novel domain (medical, scientific) | $500K-5M | + +**Failure modes:** +- **Quality lags frontier by ~6 months.** Frontier model improvements outpace your fine-tune cycle. Plan for refresh every 12-18 months. +- **Retraining cadence is a recurring engineering cost.** Quarterly retraining typical; budget 30% of one ML engineer. +- **Without an eval set, fine-tune drift is invisible.** You won't know quality degraded until a customer complains. +- **Inference is your problem now.** Fine-tuned models often run via hosted inference (Together, Fireworks, Replicate) for $0.50-2.00/M tokens; self-host adds operational complexity. + +**Anti-patterns:** +- "Fine-tune to get better results." If frontier API is already at 90%+ accuracy, fine-tune to a smaller model usually drops it to 80-85%. The "better results" framing is backwards. +- "Fine-tune to save money." Only economically valid at high volume (>500M tokens/mo); below that, API wins even at frontier-premium pricing. + +### Path 3: Build from scratch / pre-train (the <1% case) + +**What it is:** Train a foundation model from scratch. + +**Use when:** Almost never. Only: +- You are a foundation-model company (Anthropic, OpenAI, Cohere, Mistral, DeepSeek, etc.). +- You have a uniquely valuable corpus + $50M+ funding + 18-month patience. +- Your moat IS the model. + +**Why it rarely makes sense:** +- Frontier models have caught up to specialized models in most domains within 18 months (medical, legal, code). +- By the time you ship, frontier capability has advanced 2 generations. +- Pre-training cost: $5M-50M+ depending on model size and data. +- Hidden cost: continued pre-training and alignment to keep up. + +**Failure modes:** +- **Sunk cost trap.** Once you've spent $20M pre-training, sunk cost bias prevents switching to frontier APIs even when they're better. +- **Talent dependency.** Pre-training requires research scientists who can leave for $1M+ TC at frontier labs. +- **Compute access.** H100 / B200 supply remains constrained; access depends on hyperscaler relationships. + +## Decision Tree (use the calculator for the full version) + +1. **Is this well-served by frontier capability?** (YES → API, unless...) +2. **Do you have data residency / sovereignty constraints?** (YES → fine-tune self-hosted) +3. **Do you have domain-specific behavior the API can't be prompted into?** (YES + labeled data + team → fine-tune) +4. **Latency budget < 500ms?** (YES → fine-tune at high volume; API + streaming may suffice at lower volume) +5. **Volume > 500M tokens/month + multi-year stable workload?** (YES → run breakeven, consider fine-tune) +6. **All above NO + need maximum capability?** → API frontier-premium tier + +## The Eval-First Discipline + +**Rule:** Don't pick a path without an eval set. Without measurement, all three paths look the same. + +Minimum eval set: +- 50-100 representative inputs covering your use case +- Expected outputs OR rubric for human grading +- Edge cases: ambiguous inputs, adversarial inputs, format edge cases +- Run on every path you consider; the scores determine the decision + +Tools: `engineering/self-eval/`, `promptfoo`, `Inspect-AI`, internal eval harnesses. + +## When This Reference Doesn't Help + +- **RAG architecture choices.** See `engineering/rag-architect/`. +- **Agent design patterns.** See `engineering/agent-designer/`. +- **Prompt engineering technique.** See `engineering/prompt-governance/`. +- **Eval harness implementation.** See `engineering/self-eval/`. +- **Inference cost optimization tactics.** See `engineering/llm-cost-optimizer/`. + +This reference is about the strategic choice between API / fine-tune / build, not how to implement any of them. + +--- + +**Source authorities (non-exhaustive):** + +- Anthropic, "Model Cards for Claude 3.5 Sonnet, Claude 4 family" — published model performance and capability disclosures +- OpenAI, "GPT-4 Technical Report" (arXiv:2303.08774, 2023) and subsequent model spec releases +- Google DeepMind, "Gemini: A Family of Highly Capable Multimodal Models" (2023, updated 2024-2026) +- Meta AI, "Llama 3.1: Open Foundation and Instruction Models" (2024) +- Hu et al., "LoRA: Low-Rank Adaptation of Large Language Models" (arXiv:2106.09685, 2021) +- Ouyang et al., "Training Language Models to Follow Instructions with Human Feedback" (RLHF, 2022) +- Rafailov et al., "Direct Preference Optimization: Your Language Model is Secretly a Reward Model" (DPO, 2023) +- Stanford CRFM, "On the Opportunities and Risks of Foundation Models" (2021) +- Henderson et al., "Foundation Models and Fair Use" (2023) diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py new file mode 100644 index 00000000..04141cef --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_cost_economics.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""ai_cost_economics.py — API vs self-hosted inference breakeven analysis. + +Stdlib-only. Takes a workload profile and outputs: + - Monthly API cost at three tiers (frontier-premium, frontier-economy, open-hosted) + - Monthly self-hosted cost (GPU rental + ops, at chosen model size) + - Breakeven point: where API and self-hosted cross + - Sensitivity: low/mid/high GPU rate scenarios + - Recommended path with explicit caveats + +Deterministic logic derived from the profile. + +Input schema (JSON): +{ + "workload_name": "Customer support generation", + "monthly_input_tokens_m": 600, # millions of input tokens per month + "monthly_output_tokens_m": 150, + "quality_tier_required": "frontier-economy", # frontier-premium | frontier-economy | open-hosted + "model_size_class_self_host": "70b-class", # 7b-13b | 70b-class + "latency_p95_target_ms": 1500, + "utilization_assumed_pct": 70, # realistic GPU utilization for self-hosting + "include_ops_attribution": true # 30% of an engineer attributed to self-hosted ops +} + +Usage: + python ai_cost_economics.py # uses embedded 5M tokens/day sample + python ai_cost_economics.py path/to/workload.json + python ai_cost_economics.py workload.json --output json +""" + +import argparse +import json +import sys +from typing import Any, Dict, List + + +SAMPLE: Dict[str, Any] = { + "workload_name": "B2B SaaS customer-support generation (5M tokens/day)", + "monthly_input_tokens_m": 600, + "monthly_output_tokens_m": 150, + "quality_tier_required": "frontier-economy", + "model_size_class_self_host": "70b-class", + "latency_p95_target_ms": 1500, + "utilization_assumed_pct": 70, + "include_ops_attribution": True, +} + + +# 2026 API pricing per million tokens, $USD (input / output) +API_PRICING = { + "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"}, + "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"}, + "open-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"}, +} + +# GPU spot pricing 2026 ($/hour). Mid-range; varies by provider and commitment. +GPU_PRICING = { + "A100-spot-low": 1.50, + "A100-spot-mid": 2.50, + "A100-spot-high": 3.50, + "H100-spot-low": 3.50, + "H100-spot-mid": 5.00, + "H100-spot-high": 8.00, +} + +# Tokens per second per GPU at 70% utilization (rough) +TOKENS_PER_GPU_PER_SEC = { + "7b-13b": {"A100": 1500, "H100": 3500}, + "70b-class": {"A100": 200, "H100": 600}, +} + +# Number of GPUs needed for model (minimum, with KV cache) +GPUS_PER_MODEL = { + "7b-13b": 1, + "70b-class": 4, # 70B at FP16 needs ~140GB; 4xA100-40GB or 2xH100-80GB +} + +# Engineer fully-loaded cost (annual) +ENGINEER_FULLY_LOADED = 250_000 +OPS_ATTRIBUTION_PCT = 0.30 # 30% of an engineer attributed to self-hosted ops + + +def api_monthly_cost(profile: Dict[str, Any], tier: str) -> float: + pricing = API_PRICING.get(tier, API_PRICING["frontier-economy"]) + return ( + profile.get("monthly_input_tokens_m", 0) * pricing["input"] + + profile.get("monthly_output_tokens_m", 0) * pricing["output"] + ) + + +def self_hosted_monthly_cost(profile: Dict[str, Any], gpu_type: str, gpu_pricing_tier: str) -> Dict[str, Any]: + """Compute self-hosted monthly cost for given GPU type and pricing tier.""" + model_class = profile.get("model_size_class_self_host", "70b-class") + utilization = profile.get("utilization_assumed_pct", 70) / 100 + monthly_tokens_total_m = profile.get("monthly_input_tokens_m", 0) + profile.get("monthly_output_tokens_m", 0) + monthly_tokens_total = monthly_tokens_total_m * 1_000_000 + + gpus_needed = GPUS_PER_MODEL[model_class] + tokens_per_sec_per_gpu = TOKENS_PER_GPU_PER_SEC[model_class][gpu_type] + effective_tokens_per_sec = gpus_needed * tokens_per_sec_per_gpu * utilization + + # Hours of GPU time needed per month + seconds_per_month = monthly_tokens_total / effective_tokens_per_sec + hours_per_month = seconds_per_month / 3600 + + # But minimum: GPUs must be warm 24/7 if we want consistent latency + # So actual hours = max(hours_per_month, 24 * 30 * gpus_needed) + hours_warm = 24 * 30 * gpus_needed + hours_billable = max(hours_per_month, hours_warm) + + gpu_pricing_key = f"{gpu_type}-spot-{gpu_pricing_tier}" + rate = GPU_PRICING[gpu_pricing_key] + gpu_cost = hours_billable * rate / gpus_needed * gpus_needed # already per GPU + + ops_cost = (ENGINEER_FULLY_LOADED * OPS_ATTRIBUTION_PCT) / 12 if profile.get("include_ops_attribution", True) else 0 + + return { + "gpu_cost": round(gpu_cost, 0), + "ops_cost": round(ops_cost, 0), + "total": round(gpu_cost + ops_cost, 0), + "hours_warm_required": int(hours_warm), + "hours_compute_required": int(hours_per_month), + "gpus_needed": gpus_needed, + "gpu_rate_per_hr": rate, + } + + +def find_breakeven(profile: Dict[str, Any], api_tier: str, gpu_type: str, gpu_pricing_tier: str) -> Dict[str, Any]: + """Find the monthly token volume where API and self-hosted cost cross.""" + # API cost is linear in tokens; self-hosted has fixed (warm GPU) + linear component + model_class = profile.get("model_size_class_self_host", "70b-class") + utilization = profile.get("utilization_assumed_pct", 70) / 100 + gpus_needed = GPUS_PER_MODEL[model_class] + tokens_per_sec_per_gpu = TOKENS_PER_GPU_PER_SEC[model_class][gpu_type] + effective_tokens_per_sec = gpus_needed * tokens_per_sec_per_gpu * utilization + + gpu_pricing_key = f"{gpu_type}-spot-{gpu_pricing_tier}" + rate = GPU_PRICING[gpu_pricing_key] + + # Self-hosted: warm 24/7 fixed cost, plus ops + monthly_fixed = 24 * 30 * gpus_needed * rate + ops_cost = (ENGINEER_FULLY_LOADED * OPS_ATTRIBUTION_PCT) / 12 if profile.get("include_ops_attribution", True) else 0 + self_hosted_floor = monthly_fixed + ops_cost # cost even at zero tokens (because warm) + + # When tokens exceed warm capacity, additional cost is more GPU hours + # But up to warm capacity, total cost is just monthly_fixed + ops_cost + warm_capacity_tokens_per_month = effective_tokens_per_sec * 24 * 30 * 3600 + + # API cost per million tokens (weighted by I/O ratio) + monthly_in = profile.get("monthly_input_tokens_m", 1) + monthly_out = profile.get("monthly_output_tokens_m", 1) + total_m = monthly_in + monthly_out + in_ratio = monthly_in / total_m if total_m else 0.8 + out_ratio = monthly_out / total_m if total_m else 0.2 + + api_per_m = API_PRICING[api_tier]["input"] * in_ratio + API_PRICING[api_tier]["output"] * out_ratio + + # Breakeven: api_per_m * tokens_m = self_hosted_floor + if api_per_m > 0: + breakeven_tokens_m = self_hosted_floor / api_per_m + else: + breakeven_tokens_m = None + + return { + "breakeven_monthly_tokens_m": round(breakeven_tokens_m, 0) if breakeven_tokens_m else None, + "self_hosted_floor_monthly": round(self_hosted_floor, 0), + "warm_capacity_monthly_tokens_m": round(warm_capacity_tokens_per_month / 1_000_000, 0), + "api_per_m_blended": round(api_per_m, 2), + } + + +def analyze(profile: Dict[str, Any]) -> Dict[str, Any]: + api_tier = profile.get("quality_tier_required", "frontier-economy") + monthly_tokens_total_m = profile.get("monthly_input_tokens_m", 0) + profile.get("monthly_output_tokens_m", 0) + + # API costs at all 3 tiers + api_costs = {tier: round(api_monthly_cost(profile, tier), 0) for tier in API_PRICING} + + # Self-hosted at chosen GPU type, 3 pricing tiers + gpu_type = "A100" if profile.get("latency_p95_target_ms", 2000) > 1000 else "H100" + self_hosted_low = self_hosted_monthly_cost(profile, gpu_type, "low") + self_hosted_mid = self_hosted_monthly_cost(profile, gpu_type, "mid") + self_hosted_high = self_hosted_monthly_cost(profile, gpu_type, "high") + + # Breakeven analysis at mid pricing + breakeven = find_breakeven(profile, api_tier, gpu_type, "mid") + + # Recommendation + api_chosen_cost = api_costs[api_tier] + self_hosted_chosen_cost = self_hosted_mid["total"] + + if monthly_tokens_total_m < breakeven["breakeven_monthly_tokens_m"]: + rec = "API" + reasoning = ( + f"Current volume ({monthly_tokens_total_m:.0f}M tokens/mo) is BELOW breakeven " + f"({breakeven['breakeven_monthly_tokens_m']:.0f}M tokens/mo). API tier '{api_tier}' is cheaper " + f"({_fmt_money(api_chosen_cost)}/mo) than self-hosted " + f"({_fmt_money(self_hosted_chosen_cost)}/mo at mid GPU rates)." + ) + caveats = [ + "API costs scale linearly with token volume; revisit when volume doubles", + "Build multi-vendor abstraction (LiteLLM / OpenRouter) for failover", + "Pin model IDs; run regression evals on every model upgrade", + ] + elif self_hosted_high["total"] < api_chosen_cost: + rec = "SELF_HOSTED" + reasoning = ( + f"Current volume ({monthly_tokens_total_m:.0f}M tokens/mo) is well above breakeven. " + f"Self-hosted at {_fmt_money(self_hosted_chosen_cost)}/mo (mid GPU rates) is cheaper than API " + f"at {_fmt_money(api_chosen_cost)}/mo across all GPU pricing scenarios." + ) + caveats = [ + "Quality lags frontier by ~6 months; budget refresh cycle", + "24/7 on-call required; 30% engineer attribution may underestimate at scale", + "GPU spot pricing volatile; negotiate reserved capacity at this scale", + "Eval discipline non-negotiable for self-hosted; without it you cannot detect quality degradation", + ] + else: + rec = "HYBRID" + reasoning = ( + f"Current volume ({monthly_tokens_total_m:.0f}M tokens/mo) is above breakeven but self-hosted " + f"cost ({_fmt_money(self_hosted_chosen_cost)}/mo) is close to API ({_fmt_money(api_chosen_cost)}/mo). " + "Consider hybrid: API for tail / low-volume use cases, self-hosted for high-volume / latency-sensitive paths." + ) + caveats = [ + "Migration to self-hosted typically takes 3-6 months of engineering time — model in TCO", + "Hybrid increases operational complexity; ensure routing logic is testable", + "At this margin, capability differences between API and 70B-class may matter more than cost", + ] + + return { + "recommendation": rec, + "reasoning": reasoning, + "caveats": caveats, + "monthly_costs": { + "api_frontier_premium": api_costs["frontier-premium"], + "api_frontier_economy": api_costs["frontier-economy"], + "api_open_hosted": api_costs["open-hosted"], + "self_hosted_low_gpu_rate": self_hosted_low, + "self_hosted_mid_gpu_rate": self_hosted_mid, + "self_hosted_high_gpu_rate": self_hosted_high, + }, + "breakeven_analysis": breakeven, + "gpu_type_recommended": gpu_type, + "current_monthly_tokens_m": monthly_tokens_total_m, + } + + +def render_text(result: Dict[str, Any], profile: Dict[str, Any], source: str) -> str: + lines = [] + lines.append("=" * 72) + lines.append("AI COST ECONOMICS — API vs SELF-HOSTED") + lines.append(f"Source: {source}") + lines.append("=" * 72) + lines.append("") + lines.append(f"Workload: {profile.get('workload_name')}") + lines.append(f" Volume: {profile.get('monthly_input_tokens_m')}M input + {profile.get('monthly_output_tokens_m')}M output tokens/mo") + lines.append(f" Quality tier required: {profile.get('quality_tier_required')}") + lines.append(f" Model size for self-host: {profile.get('model_size_class_self_host')}") + lines.append(f" Latency p95 target: {profile.get('latency_p95_target_ms')}ms") + lines.append(f" Utilization assumed: {profile.get('utilization_assumed_pct')}%") + lines.append("") + lines.append("-" * 72) + lines.append(f"RECOMMENDATION: {result['recommendation']}") + lines.append("") + for line in _wrap(result["reasoning"], 2): + lines.append(line) + lines.append("") + lines.append("Caveats:") + for c in result["caveats"]: + lines.append(f" • {c}") + lines.append("") + lines.append("-" * 72) + lines.append("MONTHLY COST COMPARISON:") + lines.append("") + mc = result["monthly_costs"] + lines.append(f" API frontier-premium: {_fmt_money(mc['api_frontier_premium']):>15} ({API_PRICING['frontier-premium']['label']})") + lines.append(f" API frontier-economy: {_fmt_money(mc['api_frontier_economy']):>15} ({API_PRICING['frontier-economy']['label']})") + lines.append(f" API open-hosted: {_fmt_money(mc['api_open_hosted']):>15} ({API_PRICING['open-hosted']['label']})") + lines.append("") + lines.append(f" Self-hosted ({result['gpu_type_recommended']}), low GPU rates: {_fmt_money(mc['self_hosted_low_gpu_rate']['total']):>15} (GPU @ ${mc['self_hosted_low_gpu_rate']['gpu_rate_per_hr']}/hr × {mc['self_hosted_low_gpu_rate']['gpus_needed']} GPUs)") + lines.append(f" Self-hosted ({result['gpu_type_recommended']}), mid GPU rates: {_fmt_money(mc['self_hosted_mid_gpu_rate']['total']):>15} (GPU @ ${mc['self_hosted_mid_gpu_rate']['gpu_rate_per_hr']}/hr × {mc['self_hosted_mid_gpu_rate']['gpus_needed']} GPUs)") + lines.append(f" Self-hosted ({result['gpu_type_recommended']}), high GPU rates: {_fmt_money(mc['self_hosted_high_gpu_rate']['total']):>15} (GPU @ ${mc['self_hosted_high_gpu_rate']['gpu_rate_per_hr']}/hr × {mc['self_hosted_high_gpu_rate']['gpus_needed']} GPUs)") + lines.append("") + lines.append(f" Self-hosted ops attribution: {_fmt_money(mc['self_hosted_mid_gpu_rate']['ops_cost'])}/mo (30% of one engineer)") + lines.append("") + lines.append("-" * 72) + be = result["breakeven_analysis"] + lines.append("BREAKEVEN ANALYSIS:") + lines.append("") + if be["breakeven_monthly_tokens_m"]: + lines.append(f" API '{profile.get('quality_tier_required')}' vs self-hosted at mid GPU rates:") + lines.append(f" Breakeven: ~{be['breakeven_monthly_tokens_m']:,.0f}M tokens/month") + lines.append(f" Current volume: {result['current_monthly_tokens_m']:,.0f}M tokens/month") + lines.append(f" Self-hosted floor (warm GPUs + ops, even at zero tokens): {_fmt_money(be['self_hosted_floor_monthly'])}/mo") + lines.append(f" Self-hosted warm capacity ceiling: ~{be['warm_capacity_monthly_tokens_m']:,.0f}M tokens/month") + lines.append(f" API blended cost: ${be['api_per_m_blended']}/M tokens") + lines.append("") + lines.append("-" * 72) + lines.append("REMINDER: This analysis uses 2026 pricing. Pricing changes; re-run quarterly.") + lines.append("Migration to self-hosted is 3-6 months of engineering work — model that in your TCO.") + return "\n".join(lines) + + +def _fmt_money(amount: float) -> str: + return f"${amount:,.0f}" + + +def _wrap(text: str, indent: int, width: int = 70) -> List[str]: + import textwrap + return textwrap.wrap(text, width=width, initial_indent=" " * indent, subsequent_indent=" " * indent) or [" " * indent + text] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="API vs self-hosted inference breakeven + sensitivity analysis.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("path", nargs="?", help="Path to workload JSON (uses embedded sample if omitted)") + parser.add_argument("--output", choices=("text", "json"), default="text", help="Output format") + args = parser.parse_args() + + if args.path: + try: + with open(args.path, "r", encoding="utf-8") as f: + profile = json.load(f) + source = args.path + except (IOError, OSError) as e: + print(f"error: could not read {args.path}: {e}", file=sys.stderr) + return 1 + except json.JSONDecodeError as e: + print(f"error: invalid JSON in {args.path}: {e}", file=sys.stderr) + return 1 + else: + profile = SAMPLE + source = "" + + result = analyze(profile) + + if args.output == "json": + print(json.dumps({"source": source, "profile": profile, **result}, indent=2)) + else: + print(render_text(result, profile, source)) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py new file mode 100644 index 00000000..ba033dc0 --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/ai_risk_classifier.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +"""ai_risk_classifier.py — Classify an AI use case under EU AI Act + US state laws. + +Stdlib-only. Takes a use case profile and outputs: + - Risk tier (PROHIBITED / HIGH / LIMITED / MINIMAL) under EU AI Act + - US state law triggers (NYC LL 144, CO SB 21-169 successor, IL HB 53, CA SB 1001) + - Industry-specific overlays (FDA, NYDFS, NAIC) + - Required controls + conformity assessment trigger + - Citations to specific articles / regulations + +NOT legal advice — surfaces classification for qualified AI counsel. + +Input schema (JSON): +{ + "use_case": "AI screening of job applications", + "domain": "employment", # employment | credit | education | healthcare | critical-infra | + # law-enforcement | biometric | content-moderation | b2b-general | + # consumer-general + "deploys_in_eu": true, + "deploys_in_us_states": ["NY", "CO", "IL", "CA"], + "decisions_affected": "consequential", # consequential | informational | internal-only + "automation_level": "automated", # automated | human-in-loop | advisory + "user_facing": true, + "biometric_data_processed": false, + "children_under_16": false +} + +Usage: + python ai_risk_classifier.py # uses embedded hiring-AI sample + python ai_risk_classifier.py path/to/use_case.json + python ai_risk_classifier.py use_case.json --output json +""" + +import argparse +import json +import sys +from typing import Any, Dict, List + + +SAMPLE: Dict[str, Any] = { + "use_case": "AI-assisted screening of job applications (resume ranking)", + "domain": "employment", + "deploys_in_eu": True, + "deploys_in_us_states": ["NY", "CO", "IL", "CA"], + "decisions_affected": "consequential", + "automation_level": "automated", + "user_facing": False, + "biometric_data_processed": False, + "children_under_16": False, +} + + +# EU AI Act Annex III "high-risk" domains (Article 6(2)) +HIGH_RISK_DOMAINS = { + "employment", + "credit", + "education", + "critical-infra", + "law-enforcement", + "biometric", + "migration", + "justice", + "essential-services", # insurance, public benefits +} + +# EU AI Act Article 5 prohibited practices +PROHIBITED_TRIGGERS = { + "social-scoring", + "real-time-biometric-surveillance", + "subliminal-manipulation", + "exploitation-of-vulnerability", + "predictive-policing-from-profiling", + "emotion-recognition-workplace-or-education", + "biometric-categorization-by-protected-traits", +} + + +def classify_eu(profile: Dict[str, Any]) -> Dict[str, Any]: + """Return EU AI Act classification + reasoning.""" + deploys_eu = profile.get("deploys_in_eu", False) + if not deploys_eu: + return { + "tier": "NOT_APPLICABLE", + "reasoning": "Does not deploy in EU. EU AI Act not triggered.", + "obligations": [], + "citations": [], + } + + domain = profile.get("domain", "") + decisions = profile.get("decisions_affected", "informational") + biometric = profile.get("biometric_data_processed", False) + automation = profile.get("automation_level", "advisory") + use_case = profile.get("use_case", "").lower() + + # Article 5 prohibited check (heuristic match) + for prohibited in PROHIBITED_TRIGGERS: + if any(kw in use_case for kw in prohibited.split("-")): + # Conservative: match only if multiple keywords hit + keywords = prohibited.split("-") + hits = sum(1 for kw in keywords if kw in use_case) + if hits >= 2: + return { + "tier": "PROHIBITED", + "reasoning": ( + f"Use case description appears to match Article 5 prohibited practice ({prohibited}). " + "Cannot deploy in EU regardless of safeguards. Re-scope the product or exclude EU market." + ), + "obligations": ["Cease deployment in EU"], + "citations": ["EU AI Act Art. 5"], + } + + # Special prohibited: biometric in public spaces by law enforcement (real-time) + if biometric and domain == "law-enforcement" and automation == "automated": + return { + "tier": "PROHIBITED", + "reasoning": ( + "Real-time biometric identification by law enforcement in publicly accessible spaces is " + "Art. 5(1)(h) prohibited (narrow exceptions for serious crimes only)." + ), + "obligations": ["Cease deployment unless narrow exception applies, in which case Annex III high-risk obligations also apply"], + "citations": ["EU AI Act Art. 5(1)(h)"], + } + + # High-risk Annex III check + if domain in HIGH_RISK_DOMAINS and decisions == "consequential": + return { + "tier": "HIGH", + "reasoning": ( + f"Annex III high-risk domain ({domain}) with consequential decisions. " + "Conformity assessment + registration + post-market monitoring required before deployment." + ), + "obligations": [ + "Conformity assessment (Art. 43)", + "Registration in EU AI database (Art. 49)", + "Risk management system (Art. 9)", + "Data governance: representative, accurate, complete training data (Art. 10)", + "Technical documentation maintained throughout lifecycle (Art. 11)", + "Logging / record-keeping (Art. 12)", + "Transparency and instructions for use (Art. 13)", + "Human oversight (Art. 14)", + "Accuracy, robustness, cybersecurity (Art. 15)", + "Post-market monitoring + incident reporting (Art. 72)", + ], + "citations": ["EU AI Act Art. 6", "Annex III", "Art. 8-15", "Art. 43", "Art. 49", "Art. 72"], + } + + # Biometric data: special category — usually high-risk + if biometric: + return { + "tier": "HIGH", + "reasoning": ( + "Biometric data processing triggers Annex III obligations even outside the listed domains " + "(special category under GDPR Art. 9 + AI Act overlay)." + ), + "obligations": [ + "Conformity assessment + Annex III high-risk obligations", + "GDPR Art. 9(2) explicit consent or other Art. 9 lawful basis", + "DPIA mandatory (GDPR Art. 35)", + ], + "citations": ["EU AI Act Annex III §1", "GDPR Art. 9", "GDPR Art. 35"], + } + + # Limited risk: chatbots, deepfakes, emotion recognition (outside workplace/edu), generative AI + if "chatbot" in use_case or "deepfake" in use_case or "image generation" in use_case or "video generation" in use_case: + return { + "tier": "LIMITED", + "reasoning": ( + "Limited risk: transparency obligations apply — users must be informed they are interacting with AI " + "or that content is AI-generated." + ), + "obligations": [ + "Inform users they are interacting with AI (Art. 50(1))", + "Mark AI-generated / manipulated content (Art. 50(2))", + "If general-purpose AI model: model card with capabilities, limitations, training-data summary (Art. 53)", + ], + "citations": ["EU AI Act Art. 50", "Art. 53"], + } + + # Minimal risk default + return { + "tier": "MINIMAL", + "reasoning": ( + "Does not fall under prohibited, Annex III high-risk, or limited-risk categories. " + "No specific AI Act obligations beyond general product safety; voluntary codes of conduct recommended." + ), + "obligations": [ + "Voluntary alignment with NIST AI RMF / EU codes of conduct (recommended)", + "GDPR obligations still apply if personal data is processed", + ], + "citations": ["EU AI Act recital 27", "NIST AI RMF 1.0"], + } + + +def us_state_triggers(profile: Dict[str, Any]) -> List[Dict[str, str]]: + """Return list of triggered US state-level obligations.""" + states = set(s.upper() for s in profile.get("deploys_in_us_states", [])) + domain = profile.get("domain", "") + user_facing = profile.get("user_facing", False) + triggers = [] + + # NYC LL 144 — AEDTs in employment + if "NY" in states and domain == "employment": + triggers.append({ + "law": "NYC Local Law 144 (AEDT)", + "trigger": "Automated Employment Decision Tool used in hiring or promotion for NYC employees", + "obligations": ( + "Annual independent bias audit; candidate notice (10+ business days before use); " + "publication of audit summary on company website." + ), + "citation": "NYC Local Law 144 of 2021; 6 RCNY § 5-300", + }) + + # Colorado AI Act / SB 21-169 successor + if "CO" in states and domain in {"employment", "credit", "education", "insurance", "essential-services"}: + triggers.append({ + "law": "Colorado AI Act (SB 21-169 / 2024 amendments)", + "trigger": f"High-risk AI system in consumer decisions ({domain})", + "obligations": ( + "Reasonable care to protect from algorithmic discrimination; impact assessment; " + "consumer notice; right to opt-out of profiling; risk management policy." + ), + "citation": "Colorado SB 21-169 (as amended)", + }) + + # Illinois HB 53 — AI in employment interviews + if "IL" in states and domain == "employment": + triggers.append({ + "law": "Illinois HB 53 (AI Video Interview Act)", + "trigger": "AI analyzes video interviews of Illinois applicants", + "obligations": ( + "Candidate notice + consent before recording; explanation of how AI is used; " + "deletion within 30 days of request; restrictions on sharing data." + ), + "citation": "Illinois 820 ILCS 42/", + }) + + # California SB 1001 — Bot disclosure + if "CA" in states and user_facing: + triggers.append({ + "law": "California SB 1001 (B.O.T. Act)", + "trigger": "User-facing AI bot in commercial transactions or California elections", + "obligations": "Disclose to user that they are interacting with a bot (not a human).", + "citation": "California Business & Professions Code § 17940", + }) + + # Illinois BIPA — biometric data + if "IL" in states and profile.get("biometric_data_processed", False): + triggers.append({ + "law": "Illinois Biometric Information Privacy Act (BIPA)", + "trigger": "Biometric identifier or biometric information capture", + "obligations": ( + "Written informed consent; published retention/destruction policy; cannot sell biometric data; " + "private right of action with statutory damages ($1K-$5K per violation)." + ), + "citation": "Illinois 740 ILCS 14/", + }) + + return triggers + + +def industry_overlays(profile: Dict[str, Any]) -> List[Dict[str, str]]: + """Return industry-specific regulatory overlays.""" + domain = profile.get("domain", "") + overlays = [] + + if domain == "healthcare": + overlays.append({ + "framework": "FDA AI/ML guidance + Software as Medical Device (SaMD)", + "trigger": "AI in clinical decisions, diagnostic, or therapeutic use", + "obligations": ( + "510(k) or De Novo or PMA pathway depending on risk class; Predetermined Change Control Plan " + "for adaptive models; Good Machine Learning Practices (GMLP)." + ), + "citation": "FDA Guidance on AI/ML SaMD (2023); 21 CFR Part 820", + }) + elif domain == "credit": + overlays.append({ + "framework": "ECOA + FCRA + CFPB Circular 2023-03", + "trigger": "AI used in credit underwriting or adverse action", + "obligations": ( + "Specific reason for adverse action (not 'algorithm said no'); model risk management " + "consistent with SR 11-7 if a bank; explainability sufficient for FCRA adverse action notice." + ), + "citation": "15 USC §1691 (ECOA); CFPB Circular 2023-03; Fed SR 11-7", + }) + elif domain == "essential-services": + overlays.append({ + "framework": "NAIC Model Bulletin on AI in Insurance", + "trigger": "AI in insurance underwriting, pricing, claims, fraud", + "obligations": ( + "AI program governance, risk management, third-party AI oversight; " + "documented testing for unfair discrimination." + ), + "citation": "NAIC Model Bulletin on the Use of AI by Insurers (2023)", + }) + + return overlays + + +def required_controls(profile: Dict[str, Any], eu_classification: Dict[str, Any]) -> List[str]: + """Return the required-controls checklist based on tier + profile.""" + tier = eu_classification.get("tier", "") + controls = [] + + if tier in ("HIGH", "LIMITED", "MINIMAL"): + controls.extend([ + "Eval set with documented success criteria before deployment", + "Monitoring of model output in production (drift, bias, hallucination)", + "Fallback behavior defined for model failure modes", + "Human-in-loop review for high-stakes outputs", + ]) + + if tier == "HIGH": + controls.extend([ + "Conformity assessment completed and documented (EU AI Act Art. 43)", + "Registration in EU AI database before deployment (Art. 49)", + "Risk management system documented and maintained (Art. 9)", + "Training data governance: representativeness, accuracy, bias mitigation (Art. 10)", + "Technical documentation per Annex IV maintained throughout lifecycle (Art. 11)", + "Comprehensive logging for traceability (Art. 12)", + "Human oversight design (e.g., stop button, override capability) (Art. 14)", + "Post-market monitoring plan + serious incident reporting (Art. 72)", + "DPIA under GDPR Art. 35 if personal data processed", + ]) + + if tier == "LIMITED": + controls.extend([ + "User notification: 'You are interacting with AI' or 'This content is AI-generated'", + "If general-purpose model: publish model card per Art. 53", + ]) + + if profile.get("user_facing"): + controls.append("Public-facing disclosure of AI usage in customer-facing communications") + + if profile.get("automation_level") == "automated" and profile.get("decisions_affected") == "consequential": + controls.append("Right-to-explanation / contestation mechanism for affected individuals (GDPR Art. 22)") + + return controls + + +def analyze(profile: Dict[str, Any]) -> Dict[str, Any]: + eu = classify_eu(profile) + us = us_state_triggers(profile) + overlays = industry_overlays(profile) + controls = required_controls(profile, eu) + + conformity_required = eu.get("tier") == "HIGH" + + return { + "eu_classification": eu, + "us_state_triggers": us, + "industry_overlays": overlays, + "required_controls": controls, + "conformity_assessment_required": conformity_required, + } + + +def render_text(result: Dict[str, Any], profile: Dict[str, Any], source: str) -> str: + lines = [] + lines.append("=" * 72) + lines.append("AI RISK CLASSIFICATION") + lines.append(f"Source: {source}") + lines.append("=" * 72) + lines.append("") + lines.append(f"Use case: {profile.get('use_case')}") + lines.append(f" Domain: {profile.get('domain')} | Automation: {profile.get('automation_level')} | Decisions: {profile.get('decisions_affected')}") + lines.append(f" Deploys in EU: {profile.get('deploys_in_eu')} | US states: {', '.join(profile.get('deploys_in_us_states', []))}") + lines.append(f" User-facing: {profile.get('user_facing')} | Biometric: {profile.get('biometric_data_processed')}") + lines.append("") + lines.append("-" * 72) + eu = result["eu_classification"] + tier_marker = { + "PROHIBITED": "🔴", + "HIGH": "🟠", + "LIMITED": "🟡", + "MINIMAL": "🟢", + "NOT_APPLICABLE": "⚪", + }.get(eu["tier"], "•") + lines.append(f"EU AI ACT TIER: {tier_marker} {eu['tier']}") + lines.append("") + for line in _wrap(eu["reasoning"], 2): + lines.append(line) + lines.append("") + if eu["citations"]: + lines.append(f" Citations: {', '.join(eu['citations'])}") + lines.append("") + if eu["obligations"]: + lines.append(" EU obligations:") + for o in eu["obligations"]: + lines.append(f" • {o}") + lines.append("") + lines.append("-" * 72) + + lines.append(f"CONFORMITY ASSESSMENT REQUIRED: {'YES' if result['conformity_assessment_required'] else 'no'}") + lines.append("") + lines.append("-" * 72) + + us = result["us_state_triggers"] + if us: + lines.append(f"US STATE LAW TRIGGERS ({len(us)}):") + lines.append("") + for t in us: + lines.append(f" • {t['law']}") + lines.append(f" Trigger: {t['trigger']}") + for line in _wrap(t["obligations"], 4): + lines.append(line) + lines.append(f" Citation: {t['citation']}") + lines.append("") + else: + lines.append("US STATE LAW TRIGGERS: none for the listed states + domain.") + lines.append("") + lines.append("-" * 72) + + overlays = result["industry_overlays"] + if overlays: + lines.append(f"INDUSTRY OVERLAYS ({len(overlays)}):") + lines.append("") + for o in overlays: + lines.append(f" • {o['framework']}") + lines.append(f" Trigger: {o['trigger']}") + for line in _wrap(o["obligations"], 4): + lines.append(line) + lines.append(f" Citation: {o['citation']}") + lines.append("") + lines.append("-" * 72) + + lines.append(f"REQUIRED CONTROLS ({len(result['required_controls'])}):") + for c in result["required_controls"]: + lines.append(f" ☐ {c}") + lines.append("") + lines.append("-" * 72) + lines.append("REMINDER: This is triage, not legal advice. EU AI Act conformity assessment requires qualified") + lines.append("AI counsel and may require Notified Body involvement. Re-run quarterly as regulations evolve.") + return "\n".join(lines) + + +def _wrap(text: str, indent: int, width: int = 70) -> List[str]: + import textwrap + return textwrap.wrap(text, width=width, initial_indent=" " * indent, subsequent_indent=" " * indent) or [" " * indent + text] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Classify an AI use case under EU AI Act + US state laws.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("path", nargs="?", help="Path to use_case JSON (uses embedded sample if omitted)") + parser.add_argument("--output", choices=("text", "json"), default="text", help="Output format") + args = parser.parse_args() + + if args.path: + try: + with open(args.path, "r", encoding="utf-8") as f: + profile = json.load(f) + source = args.path + except (IOError, OSError) as e: + print(f"error: could not read {args.path}: {e}", file=sys.stderr) + return 1 + except json.JSONDecodeError as e: + print(f"error: invalid JSON in {args.path}: {e}", file=sys.stderr) + return 1 + else: + profile = SAMPLE + source = "" + + result = analyze(profile) + + if args.output == "json": + print(json.dumps({"source": source, "profile": profile, **result}, indent=2)) + else: + print(render_text(result, profile, source)) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py new file mode 100644 index 00000000..da0d537f --- /dev/null +++ b/c-level-advisor/skills/chief-ai-officer-advisor/scripts/model_buildvsbuy_calculator.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""model_buildvsbuy_calculator.py — Decide API vs fine-tune vs build for a use case. + +Stdlib-only. Takes a use case profile and outputs: + - Recommendation (API / FINE_TUNE / BUILD) with reasoning + - 3-year TCO comparison across all 3 paths + - Breakeven analysis (where API stops being cheapest) + - Failure modes for the chosen path + +Deterministic logic derived from the profile. + +Input schema (JSON): +{ + "use_case": "Customer support response generation", + "expected_qps": 5, # queries per second peak + "monthly_volume_queries": 4000000, # queries per month + "avg_tokens_in": 800, + "avg_tokens_out": 200, + "latency_budget_ms": 2000, + "accuracy_required": "frontier", # frontier | high | acceptable + "domain_specific": false, # need specific vocabulary / format / behavior + "data_for_finetune_available": false, # do we have labeled data for fine-tune? + "team_ml_capacity_engineers": 1, + "compliance_requires_self_host": false # data residency / sovereignty constraint +} + +Usage: + python model_buildvsbuy_calculator.py # uses embedded customer-support sample + python model_buildvsbuy_calculator.py path/to/use_case.json + python model_buildvsbuy_calculator.py use_case.json --output json +""" + +import argparse +import json +import sys +from typing import Any, Dict, List, Tuple + + +SAMPLE: Dict[str, Any] = { + "use_case": "Customer support response generation (B2B SaaS)", + "expected_qps": 5, + "monthly_volume_queries": 4_000_000, + "avg_tokens_in": 800, + "avg_tokens_out": 200, + "latency_budget_ms": 2000, + "accuracy_required": "high", + "domain_specific": True, + "data_for_finetune_available": False, + "team_ml_capacity_engineers": 1, + "compliance_requires_self_host": False, +} + + +# 2026 API pricing per million tokens, $USD (input / output). These are illustrative; +# real pricing changes; rerun this calculator quarterly. +API_PRICING = { + "frontier-premium": {"input": 3.00, "output": 15.00, "label": "Claude Sonnet 4.6 / GPT-4o-tier"}, + "frontier-economy": {"input": 1.25, "output": 5.00, "label": "Gemini 2.5 Flash / Claude Haiku 4.5-tier"}, + "open-router-hosted": {"input": 0.50, "output": 1.50, "label": "Llama 3.1 70B / Qwen 2.5 72B via hosted endpoint"}, +} + +# Fine-tune cost (one-time + ongoing) +FINETUNE_ONE_TIME = 25_000 # data prep + initial training + eval harness +FINETUNE_ANNUAL_RETRAIN = 15_000 # quarterly retraining + ops +FINETUNE_INFERENCE_PER_M = 0.40 # cost per M tokens at moderate scale on hosted endpoint + +# Self-hosted inference cost (per million tokens, including GPU + ops at 70% utilization) +SELF_HOSTED_PER_M = { + "7b-13b": 0.15, + "70b-class": 1.50, + "frontier-class": 12.00, # very expensive without massive scale; included for completeness +} + +# Build-from-scratch cost (one-time + ongoing) — illustrative; usually NOT recommended +BUILD_FROM_SCRATCH_ONE_TIME = 8_000_000 +BUILD_FROM_SCRATCH_ANNUAL = 3_000_000 + + +def compute_api_cost_3yr(profile: Dict[str, Any], tier: str) -> float: + """3-year API cost given workload.""" + monthly_queries = profile.get("monthly_volume_queries", 0) + tokens_in = profile.get("avg_tokens_in", 0) + tokens_out = profile.get("avg_tokens_out", 0) + + monthly_input_tokens_m = (monthly_queries * tokens_in) / 1_000_000 + monthly_output_tokens_m = (monthly_queries * tokens_out) / 1_000_000 + + pricing = API_PRICING.get(tier, API_PRICING["frontier-premium"]) + monthly_cost = ( + monthly_input_tokens_m * pricing["input"] + + monthly_output_tokens_m * pricing["output"] + ) + return monthly_cost * 36 # 3 years + + +def compute_finetune_cost_3yr(profile: Dict[str, Any]) -> float: + monthly_queries = profile.get("monthly_volume_queries", 0) + tokens_total = profile.get("avg_tokens_in", 0) + profile.get("avg_tokens_out", 0) + monthly_tokens_m = (monthly_queries * tokens_total) / 1_000_000 + + monthly_inference = monthly_tokens_m * FINETUNE_INFERENCE_PER_M + annual_inference = monthly_inference * 12 + return FINETUNE_ONE_TIME + (annual_inference + FINETUNE_ANNUAL_RETRAIN) * 3 + + +def compute_self_hosted_cost_3yr(profile: Dict[str, Any], model_class: str) -> float: + """3-year self-hosted cost including GPU + ops.""" + monthly_queries = profile.get("monthly_volume_queries", 0) + tokens_total = profile.get("avg_tokens_in", 0) + profile.get("avg_tokens_out", 0) + monthly_tokens_m = (monthly_queries * tokens_total) / 1_000_000 + + per_m = SELF_HOSTED_PER_M.get(model_class, SELF_HOSTED_PER_M["70b-class"]) + monthly_inference = monthly_tokens_m * per_m + + # Add fixed ops cost: 1 engineer * 30% load * fully-loaded $250K/yr = $75K/yr ops attribution + annual_ops = 75_000 + return (monthly_inference * 36) + (annual_ops * 3) + + +def compute_build_cost_3yr() -> float: + return BUILD_FROM_SCRATCH_ONE_TIME + (BUILD_FROM_SCRATCH_ANNUAL * 3) + + +def pick_recommendation(profile: Dict[str, Any], costs: Dict[str, float]) -> Tuple[str, str, List[str]]: + """Pick API / FINE_TUNE / BUILD with reasoning and failure modes.""" + accuracy = profile.get("accuracy_required", "high") + domain_specific = profile.get("domain_specific", False) + finetune_data = profile.get("data_for_finetune_available", False) + ml_capacity = profile.get("team_ml_capacity_engineers", 0) + self_host_required = profile.get("compliance_requires_self_host", False) + latency_ms = profile.get("latency_budget_ms", 2000) + monthly_q = profile.get("monthly_volume_queries", 0) + + # Special case: compliance forces self-host + if self_host_required: + return ( + "FINE_TUNE", + ( + "Compliance / data residency forces self-host. Fine-tune a 70B-class open model " + f"({_fmt_money(costs['finetune_3yr'])}/3yr) rather than build from scratch " + f"({_fmt_money(costs['build_3yr'])}/3yr) — the gap is two orders of magnitude with " + "comparable quality for most use cases." + ), + [ + "Quality lags frontier by ~6 months; budget for refresh every 12-18mo", + "Self-hosting requires 24/7 on-call; budget 30%+ of an engineer FTE", + "Eval discipline becomes non-negotiable; without an eval set you cannot tell when retraining is needed", + ], + ) + + # Build from scratch — almost never + if accuracy == "frontier" and monthly_q > 1_000_000_000 and ml_capacity >= 20: + return ( + "BUILD", + ( + "Edge case where frontier accuracy + extreme volume + large ML team justify pre-training. " + "Cost still extreme. Most companies here are foundation-model startups, not application companies." + ), + [ + "By the time you ship, frontier models have caught up — sunk cost risk", + "Requires sustained $50M+ investment over 18+ months", + "Unless model IS your product, do not build", + ], + ) + + # Fine-tune cases + if domain_specific and finetune_data and ml_capacity >= 2: + return ( + "FINE_TUNE", + ( + "Domain-specific behavior + labeled data + ML engineering capacity available. " + f"Fine-tune cost ({_fmt_money(costs['finetune_3yr'])}) competes with API at this volume." + ), + [ + "Fine-tuned model lags frontier by ~6 months; quality drift is inevitable", + "Retraining cadence (quarterly typical) is a recurring engineering cost", + "Without eval set, fine-tune drift is invisible until customer complains", + ], + ) + + # Latency-driven fine-tune (sub-500ms with 70B-class) + if latency_ms < 500 and monthly_q > 1_000_000: + return ( + "FINE_TUNE", + ( + f"Latency budget {latency_ms}ms below frontier-API median (~600-1500ms). " + "Fine-tuned 70B-class on dedicated infra is the path to sub-500ms at scale." + ), + [ + "Sub-500ms requires GPU co-location and warm pools (idle time penalty)", + "Quality must be re-verified at every model swap", + "Streaming responses can buy headroom on latency budget; consider before committing to fine-tune", + ], + ) + + # Default to API for everything else + economy_acceptable = accuracy in ("acceptable", "high") + if economy_acceptable and costs["api_economy_3yr"] < costs["finetune_3yr"]: + return ( + "API", + ( + f"Frontier-economy API tier ({API_PRICING['frontier-economy']['label']}) at " + f"{_fmt_money(costs['api_economy_3yr'])}/3yr beats fine-tune ({_fmt_money(costs['finetune_3yr'])}/3yr). " + "Iterate on prompt engineering and eval discipline before committing to fine-tune." + ), + [ + "Vendor lock-in: build abstraction layer (LiteLLM, OpenRouter) for multi-vendor failover", + "Capability drift between model versions: pin model IDs and run regression evals on upgrades", + "Rate limits at QPS spikes: confirm Tier-4+ pricing with provider", + ], + ) + return ( + "API", + ( + f"Frontier-premium API at {_fmt_money(costs['api_premium_3yr'])}/3yr is the right starting point. " + "Revisit fine-tune at ≥10M queries/month OR domain-specific behavior the API can't be prompted into." + ), + [ + "Vendor lock-in: build abstraction layer for multi-vendor failover", + "Capability drift between model versions; pin model IDs", + "Rate limits at QPS spikes; confirm pricing tier with provider", + ], + ) + + +def analyze(profile: Dict[str, Any]) -> Dict[str, Any]: + costs = { + "api_premium_3yr": compute_api_cost_3yr(profile, "frontier-premium"), + "api_economy_3yr": compute_api_cost_3yr(profile, "frontier-economy"), + "api_open_hosted_3yr": compute_api_cost_3yr(profile, "open-router-hosted"), + "finetune_3yr": compute_finetune_cost_3yr(profile), + "self_hosted_70b_3yr": compute_self_hosted_cost_3yr(profile, "70b-class"), + "build_3yr": compute_build_cost_3yr(), + } + + recommendation, reasoning, failure_modes = pick_recommendation(profile, costs) + + # Compute breakeven volume where API and fine-tune cross + monthly_q = profile.get("monthly_volume_queries", 1) + tokens_per_q = profile.get("avg_tokens_in", 0) + profile.get("avg_tokens_out", 0) + annual_q = monthly_q * 12 + + # Find breakeven where API economy total == fine-tune total over 3 years + if tokens_per_q and annual_q: + api_economy_per_query = costs["api_economy_3yr"] / (annual_q * 3) if annual_q else 0 + # finetune_cost = ONE_TIME + (queries * tokens * inference_per_m / 1M + ANNUAL_RETRAIN) * 3 + # Solve for queries where api_cost == finetune_cost + # api_economy_per_query * Q = FINETUNE_ONE_TIME + (Q * tokens_per_q * FINETUNE_INFERENCE_PER_M / 1M + RETRAIN) * 3 + # api_economy_per_query * Q - 3 * Q * tokens_per_q * FINETUNE_INFERENCE_PER_M / 1M = FINETUNE_ONE_TIME + 3 * RETRAIN + # Q * (api_economy_per_query - 3 * tokens_per_q * FINETUNE_INFERENCE_PER_M / 1M) = ONE_TIME + 3 * RETRAIN + coefficient = ( + api_economy_per_query + - 3 * tokens_per_q * FINETUNE_INFERENCE_PER_M / 1_000_000 + ) + rhs = FINETUNE_ONE_TIME + 3 * FINETUNE_ANNUAL_RETRAIN + breakeven_3yr_queries = int(rhs / coefficient) if coefficient > 0 else None + breakeven_monthly_queries = int(breakeven_3yr_queries / 36) if breakeven_3yr_queries else None + else: + breakeven_monthly_queries = None + + return { + "recommendation": recommendation, + "reasoning": reasoning, + "failure_modes": failure_modes, + "costs_3yr_usd": {k: round(v, 0) for k, v in costs.items()}, + "breakeven_monthly_queries_api_vs_finetune": breakeven_monthly_queries, + "current_monthly_volume": profile.get("monthly_volume_queries", 0), + } + + +def render_text(result: Dict[str, Any], profile: Dict[str, Any], source: str) -> str: + lines = [] + lines.append("=" * 72) + lines.append("MODEL BUILD-VS-BUY ANALYSIS") + lines.append(f"Source: {source}") + lines.append("=" * 72) + lines.append("") + lines.append(f"Use case: {profile.get('use_case')}") + lines.append(f" Volume: {profile.get('monthly_volume_queries'):,} queries/mo @ {profile.get('expected_qps')} QPS peak") + lines.append(f" Tokens: {profile.get('avg_tokens_in')} in / {profile.get('avg_tokens_out')} out per query") + lines.append(f" Latency budget: {profile.get('latency_budget_ms')}ms | Accuracy: {profile.get('accuracy_required')}") + lines.append(f" Domain-specific: {profile.get('domain_specific')} | Fine-tune data available: {profile.get('data_for_finetune_available')}") + lines.append(f" ML capacity: {profile.get('team_ml_capacity_engineers')} engineers | Compliance forces self-host: {profile.get('compliance_requires_self_host')}") + lines.append("") + lines.append("-" * 72) + lines.append(f"RECOMMENDATION: {result['recommendation']}") + lines.append("") + for line in _wrap(result["reasoning"], 2): + lines.append(line) + lines.append("") + lines.append("Failure modes to plan for:") + for fm in result["failure_modes"]: + lines.append(f" • {fm}") + lines.append("") + lines.append("-" * 72) + lines.append("3-YEAR TCO COMPARISON ($ USD):") + lines.append("") + costs = result["costs_3yr_usd"] + lines.append(f" API (frontier-premium, {API_PRICING['frontier-premium']['label']}): {_fmt_money(costs['api_premium_3yr']):>15}") + lines.append(f" API (frontier-economy, {API_PRICING['frontier-economy']['label']}): {_fmt_money(costs['api_economy_3yr']):>15}") + lines.append(f" API (open-router-hosted, {API_PRICING['open-router-hosted']['label']}): {_fmt_money(costs['api_open_hosted_3yr']):>15}") + lines.append(f" Fine-tune (70B-class, hosted inference): {_fmt_money(costs['finetune_3yr']):>15}") + lines.append(f" Self-hosted (70B-class on rented H100/A100): {_fmt_money(costs['self_hosted_70b_3yr']):>15}") + lines.append(f" Build from scratch (pre-train + ops): {_fmt_money(costs['build_3yr']):>15}") + lines.append("") + if result["breakeven_monthly_queries_api_vs_finetune"]: + lines.append(f"Breakeven: API (economy) vs fine-tune crosses at ~{result['breakeven_monthly_queries_api_vs_finetune']:,} queries/month") + if result["current_monthly_volume"] < result["breakeven_monthly_queries_api_vs_finetune"]: + lines.append(f" Current volume ({result['current_monthly_volume']:,}/mo) is BELOW breakeven → API still cheaper.") + else: + lines.append(f" Current volume ({result['current_monthly_volume']:,}/mo) is ABOVE breakeven → fine-tune economics favorable.") + lines.append("") + lines.append("-" * 72) + lines.append("REMINDER: TCO does not capture quality cost. Fine-tune quality lags frontier by ~6 months;") + lines.append("self-hosted requires eval discipline you may not have. Re-run quarterly with updated pricing.") + return "\n".join(lines) + + +def _fmt_money(amount: float) -> str: + return f"${amount:,.0f}" + + +def _wrap(text: str, indent: int, width: int = 70) -> List[str]: + import textwrap + return textwrap.wrap(text, width=width, initial_indent=" " * indent, subsequent_indent=" " * indent) or [" " * indent + text] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Decide API vs fine-tune vs build with 3-year TCO comparison.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("path", nargs="?", help="Path to use_case JSON (uses embedded sample if omitted)") + parser.add_argument("--output", choices=("text", "json"), default="text", help="Output format") + args = parser.parse_args() + + if args.path: + try: + with open(args.path, "r", encoding="utf-8") as f: + profile = json.load(f) + source = args.path + except (IOError, OSError) as e: + print(f"error: could not read {args.path}: {e}", file=sys.stderr) + return 1 + except json.JSONDecodeError as e: + print(f"error: invalid JSON in {args.path}: {e}", file=sys.stderr) + return 1 + else: + profile = SAMPLE + source = "" + + result = analyze(profile) + + if args.output == "json": + print(json.dumps({"source": source, "profile": profile, **result}, indent=2)) + else: + print(render_text(result, profile, source)) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())