mirror of
https://github.com/alirezarezvani/claude-skills.git
synced 2026-09-10 22:41:16 +00:00
fa686a0835
skill_validator.py required frontmatter fields ["Name", "Tier", "Category",
"Dependencies", "Author", "Version"] and sections ["Name", "Description",
"Features", "Usage", "Examples"]. That is the bold key/value convention used
by its own assets/sample-skill fixture, not YAML frontmatter and not a schema
any real skill has followed. All 362 skills failed both checks identically, so
skill-quality-review.yml has been printing the same two errors on every run
and nobody has acted on them.
- frontmatter now requires name + description, what Claude Code actually
reads, and lists the current optional fields for reference
- the fixed section list becomes a scored recommendation drawn from measured
usage. No heading appears in even 30% of the 361 real SKILL.md files, so a
required list cannot be justified; a miss is now a warning, not an error
- _check_external_imports uses sys.stdlib_module_names instead of a
hand-maintained set. The old set omitted __future__, so every script using
`from __future__ import annotations` was reported as carrying an external
dependency. Real third-party imports are still caught
- the sample-skill fixture gets valid YAML frontmatter, so the reference
implementation stops teaching the schema that caused this
cfo-advisor goes from 86.4 with 2 bogus errors to 95.5 with none. The one
remaining error class, "SKILL.md too short: minimum 100 lines", is left alone:
it contradicts skill_review_checklist_runner.py's under-100-lines check and
SKILL-AUTHORING-STANDARD.md's 10KB cap, and picking a winner belongs with the
wider CONVENTIONS.md reconciliation.
Assisted-by: Claude Code:claude-opus-5
697 lines
30 KiB
Python
697 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Skill Validator - Validates skill directories against quality standards
|
|
|
|
This script validates a skill directory structure, documentation, and Python scripts
|
|
against the claude-skills ecosystem standards. It checks for required files, proper
|
|
formatting, and compliance with tier-specific requirements.
|
|
|
|
Usage:
|
|
python skill_validator.py <skill_path> [--tier TIER] [--json] [--verbose]
|
|
|
|
Author: Claude Skills Engineering Team
|
|
Version: 1.0.0
|
|
Dependencies: Python Standard Library Only
|
|
"""
|
|
|
|
import argparse
|
|
import ast
|
|
import json
|
|
import re
|
|
import sys
|
|
try:
|
|
import yaml
|
|
except ImportError:
|
|
# Minimal YAML subset: parse simple key: value frontmatter without pyyaml
|
|
class _YamlStub:
|
|
class YAMLError(Exception):
|
|
pass
|
|
@staticmethod
|
|
def safe_load(text):
|
|
result = {}
|
|
for line in text.strip().splitlines():
|
|
if ':' in line:
|
|
key, _, value = line.partition(':')
|
|
result[key.strip()] = value.strip()
|
|
return result if result else None
|
|
yaml = _YamlStub()
|
|
import datetime as dt
|
|
from pathlib import Path
|
|
from typing import Dict, List, Any, Optional, Tuple
|
|
|
|
|
|
class ValidationError(Exception):
|
|
"""Custom exception for validation errors"""
|
|
pass
|
|
|
|
|
|
class ValidationReport:
|
|
"""Container for validation results"""
|
|
|
|
def __init__(self, skill_path: str):
|
|
self.skill_path = skill_path
|
|
self.timestamp = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
self.checks = {}
|
|
self.warnings = []
|
|
self.errors = []
|
|
self.suggestions = []
|
|
self.overall_score = 0.0
|
|
self.compliance_level = "FAIL"
|
|
|
|
def add_check(self, check_name: str, passed: bool, message: str = "", score: float = 0.0):
|
|
"""Add a validation check result"""
|
|
self.checks[check_name] = {
|
|
"passed": passed,
|
|
"message": message,
|
|
"score": score
|
|
}
|
|
|
|
def add_warning(self, message: str):
|
|
"""Add a warning message"""
|
|
self.warnings.append(message)
|
|
|
|
def add_error(self, message: str):
|
|
"""Add an error message"""
|
|
self.errors.append(message)
|
|
|
|
def add_suggestion(self, message: str):
|
|
"""Add an improvement suggestion"""
|
|
self.suggestions.append(message)
|
|
|
|
def calculate_overall_score(self):
|
|
"""Calculate overall compliance score"""
|
|
if not self.checks:
|
|
self.overall_score = 0.0
|
|
return
|
|
|
|
total_score = sum(check["score"] for check in self.checks.values())
|
|
max_score = len(self.checks) * 1.0
|
|
self.overall_score = (total_score / max_score) * 100 if max_score > 0 else 0.0
|
|
|
|
# Determine compliance level
|
|
if self.overall_score >= 90:
|
|
self.compliance_level = "EXCELLENT"
|
|
elif self.overall_score >= 75:
|
|
self.compliance_level = "GOOD"
|
|
elif self.overall_score >= 60:
|
|
self.compliance_level = "ACCEPTABLE"
|
|
elif self.overall_score >= 40:
|
|
self.compliance_level = "NEEDS_IMPROVEMENT"
|
|
else:
|
|
self.compliance_level = "POOR"
|
|
|
|
|
|
class SkillValidator:
|
|
"""Main skill validation engine"""
|
|
|
|
# Tier requirements
|
|
TIER_REQUIREMENTS = {
|
|
"BASIC": {
|
|
"min_skill_md_lines": 100,
|
|
"min_scripts": 1,
|
|
"script_size_range": (100, 300),
|
|
"required_dirs": ["scripts"],
|
|
"optional_dirs": ["assets", "references", "expected_outputs"],
|
|
"features_required": ["argparse", "main_guard"]
|
|
},
|
|
"STANDARD": {
|
|
"min_skill_md_lines": 200,
|
|
"min_scripts": 1,
|
|
"script_size_range": (300, 500),
|
|
"required_dirs": ["scripts", "assets", "references"],
|
|
"optional_dirs": ["expected_outputs"],
|
|
"features_required": ["argparse", "main_guard", "json_output", "help_text"]
|
|
},
|
|
"POWERFUL": {
|
|
"min_skill_md_lines": 300,
|
|
"min_scripts": 2,
|
|
"script_size_range": (500, 800),
|
|
"required_dirs": ["scripts", "assets", "references", "expected_outputs"],
|
|
"optional_dirs": [],
|
|
"features_required": ["argparse", "main_guard", "json_output", "help_text", "error_handling"]
|
|
}
|
|
}
|
|
|
|
# Sections the repo actually uses. Measured across all 361 real SKILL.md
|
|
# files: no single heading appears in even 30% of them, so requiring a
|
|
# fixed list is not defensible. These are scored as a recommendation
|
|
# (RECOMMENDED_SECTIONS_THRESHOLD of them present earns full marks) and
|
|
# never raise an error.
|
|
RECOMMENDED_SKILL_MD_SECTIONS = [
|
|
"References", "Quick Start", "Related Skills", "Workflow", "Workflows",
|
|
"When to Use", "Proactive Triggers", "Output Artifacts",
|
|
"Anti-Patterns", "Output Format", "Overview", "Core Capabilities",
|
|
]
|
|
RECOMMENDED_SECTIONS_THRESHOLD = 2
|
|
|
|
# What Claude Code actually reads from SKILL.md frontmatter. `description`
|
|
# is what the skill listing matches against, so a skill without one is
|
|
# invisible to model invocation.
|
|
#
|
|
# This list previously read ["Name", "Tier", "Category", "Dependencies",
|
|
# "Author", "Version"] — the bold key/value convention used by the
|
|
# assets/sample-skill fixture, not YAML frontmatter and not a schema any
|
|
# real skill has ever followed. Every one of the 362 skills failed both
|
|
# checks identically, so the output was noise nobody acted on.
|
|
FRONTMATTER_REQUIRED_FIELDS = ["name", "description"]
|
|
|
|
# Present on many skills and harmless, but no runtime reads them.
|
|
FRONTMATTER_OPTIONAL_FIELDS = [
|
|
"when_to_use", "argument-hint", "arguments", "allowed-tools",
|
|
"disallowed-tools", "disable-model-invocation", "user-invocable",
|
|
"model", "effort", "context", "agent", "background", "hooks",
|
|
"paths", "shell",
|
|
]
|
|
|
|
def __init__(self, skill_path: str, target_tier: Optional[str] = None, verbose: bool = False):
|
|
self.skill_path = Path(skill_path).resolve()
|
|
self.target_tier = target_tier
|
|
self.verbose = verbose
|
|
self.report = ValidationReport(str(self.skill_path))
|
|
|
|
def log_verbose(self, message: str):
|
|
"""Log verbose message if verbose mode enabled"""
|
|
if self.verbose:
|
|
print(f"[VERBOSE] {message}", file=sys.stderr)
|
|
|
|
def validate_skill_structure(self) -> ValidationReport:
|
|
"""Main validation entry point"""
|
|
try:
|
|
self.log_verbose(f"Starting validation of {self.skill_path}")
|
|
|
|
# Check if path exists
|
|
if not self.skill_path.exists():
|
|
self.report.add_error(f"Skill path does not exist: {self.skill_path}")
|
|
return self.report
|
|
|
|
if not self.skill_path.is_dir():
|
|
self.report.add_error(f"Skill path is not a directory: {self.skill_path}")
|
|
return self.report
|
|
|
|
# Run all validation checks
|
|
self._validate_required_files()
|
|
self._validate_skill_md()
|
|
self._validate_readme()
|
|
self._validate_directory_structure()
|
|
self._validate_python_scripts()
|
|
self._validate_tier_compliance()
|
|
|
|
# Calculate overall score
|
|
self.report.calculate_overall_score()
|
|
|
|
self.log_verbose(f"Validation completed. Score: {self.report.overall_score:.1f}")
|
|
|
|
except Exception as e:
|
|
self.report.add_error(f"Validation failed with exception: {str(e)}")
|
|
|
|
return self.report
|
|
|
|
def _validate_required_files(self):
|
|
"""Validate presence of required files"""
|
|
self.log_verbose("Checking required files...")
|
|
|
|
# Check SKILL.md
|
|
skill_md_path = self.skill_path / "SKILL.md"
|
|
if skill_md_path.exists():
|
|
self.report.add_check("skill_md_exists", True, "SKILL.md found", 1.0)
|
|
else:
|
|
self.report.add_check("skill_md_exists", False, "SKILL.md missing", 0.0)
|
|
self.report.add_error("SKILL.md is required but missing")
|
|
|
|
# Check README.md
|
|
readme_path = self.skill_path / "README.md"
|
|
if readme_path.exists():
|
|
self.report.add_check("readme_exists", True, "README.md found", 1.0)
|
|
else:
|
|
self.report.add_check("readme_exists", False, "README.md missing", 0.0)
|
|
self.report.add_warning("README.md is recommended but missing")
|
|
self.report.add_suggestion("Add README.md with usage instructions and examples")
|
|
|
|
def _validate_skill_md(self):
|
|
"""Validate SKILL.md content and format"""
|
|
self.log_verbose("Validating SKILL.md...")
|
|
|
|
skill_md_path = self.skill_path / "SKILL.md"
|
|
if not skill_md_path.exists():
|
|
return
|
|
|
|
try:
|
|
content = skill_md_path.read_text(encoding='utf-8')
|
|
lines = content.split('\n')
|
|
line_count = len([line for line in lines if line.strip()])
|
|
|
|
# Check line count
|
|
min_lines = self._get_tier_requirement("min_skill_md_lines", 100)
|
|
if line_count >= min_lines:
|
|
self.report.add_check("skill_md_length", True,
|
|
f"SKILL.md has {line_count} lines (≥{min_lines})", 1.0)
|
|
else:
|
|
self.report.add_check("skill_md_length", False,
|
|
f"SKILL.md has {line_count} lines (<{min_lines})", 0.0)
|
|
self.report.add_error(f"SKILL.md too short: {line_count} lines, minimum {min_lines}")
|
|
|
|
# Validate frontmatter
|
|
self._validate_frontmatter(content)
|
|
|
|
# Check required sections
|
|
self._validate_required_sections(content)
|
|
|
|
except Exception as e:
|
|
self.report.add_check("skill_md_readable", False, f"Error reading SKILL.md: {str(e)}", 0.0)
|
|
self.report.add_error(f"Cannot read SKILL.md: {str(e)}")
|
|
|
|
def _validate_frontmatter(self, content: str):
|
|
"""Validate SKILL.md frontmatter"""
|
|
self.log_verbose("Validating frontmatter...")
|
|
|
|
# Extract frontmatter
|
|
if content.startswith('---'):
|
|
try:
|
|
end_marker = content.find('---', 3)
|
|
if end_marker == -1:
|
|
self.report.add_check("frontmatter_format", False,
|
|
"Frontmatter closing marker not found", 0.0)
|
|
return
|
|
|
|
frontmatter_text = content[3:end_marker].strip()
|
|
frontmatter = yaml.safe_load(frontmatter_text)
|
|
|
|
if not isinstance(frontmatter, dict):
|
|
self.report.add_check("frontmatter_format", False,
|
|
"Frontmatter is not a valid dictionary", 0.0)
|
|
return
|
|
|
|
# Check required fields
|
|
missing_fields = []
|
|
for field in self.FRONTMATTER_REQUIRED_FIELDS:
|
|
if field not in frontmatter:
|
|
missing_fields.append(field)
|
|
|
|
if not missing_fields:
|
|
self.report.add_check("frontmatter_complete", True,
|
|
"All required frontmatter fields present", 1.0)
|
|
else:
|
|
self.report.add_check("frontmatter_complete", False,
|
|
f"Missing fields: {', '.join(missing_fields)}", 0.0)
|
|
self.report.add_error(f"Missing frontmatter fields: {', '.join(missing_fields)}")
|
|
|
|
except yaml.YAMLError as e:
|
|
self.report.add_check("frontmatter_format", False,
|
|
f"Invalid YAML frontmatter: {str(e)}", 0.0)
|
|
self.report.add_error(f"Invalid YAML frontmatter: {str(e)}")
|
|
|
|
else:
|
|
self.report.add_check("frontmatter_exists", False,
|
|
"No frontmatter found", 0.0)
|
|
self.report.add_error("SKILL.md must start with YAML frontmatter")
|
|
|
|
def _validate_required_sections(self, content: str):
|
|
"""Score SKILL.md against the repo's common section conventions.
|
|
|
|
Advisory only: the repo has no universal section schema, so a miss is
|
|
a hint to the author, not a validation error.
|
|
"""
|
|
self.log_verbose("Checking recommended sections...")
|
|
|
|
present = []
|
|
for section in self.RECOMMENDED_SKILL_MD_SECTIONS:
|
|
pattern = rf'^#+\s*{re.escape(section)}\s*$'
|
|
if re.search(pattern, content, re.MULTILINE | re.IGNORECASE):
|
|
present.append(section)
|
|
|
|
threshold = self.RECOMMENDED_SECTIONS_THRESHOLD
|
|
if len(present) >= threshold:
|
|
self.report.add_check("recommended_sections", True,
|
|
f"{len(present)} recommended section(s) present: "
|
|
f"{', '.join(present)}", 1.0)
|
|
else:
|
|
self.report.add_check("recommended_sections", False,
|
|
f"Only {len(present)} recommended section(s) present "
|
|
f"(suggest at least {threshold} of: "
|
|
f"{', '.join(self.RECOMMENDED_SKILL_MD_SECTIONS)})", 0.0)
|
|
self.report.add_warning(
|
|
f"SKILL.md has {len(present)} of the repo's common sections; "
|
|
f"consider adding at least {threshold}")
|
|
|
|
def _validate_readme(self):
|
|
"""Validate README.md content"""
|
|
self.log_verbose("Validating README.md...")
|
|
|
|
readme_path = self.skill_path / "README.md"
|
|
if not readme_path.exists():
|
|
return
|
|
|
|
try:
|
|
content = readme_path.read_text(encoding='utf-8')
|
|
|
|
# Check minimum content length
|
|
if len(content.strip()) >= 200:
|
|
self.report.add_check("readme_substantial", True,
|
|
"README.md has substantial content", 1.0)
|
|
else:
|
|
self.report.add_check("readme_substantial", False,
|
|
"README.md content is too brief", 0.5)
|
|
self.report.add_suggestion("Expand README.md with more detailed usage instructions")
|
|
|
|
except Exception as e:
|
|
self.report.add_check("readme_readable", False,
|
|
f"Error reading README.md: {str(e)}", 0.0)
|
|
|
|
def _validate_directory_structure(self):
|
|
"""Validate directory structure against tier requirements"""
|
|
self.log_verbose("Validating directory structure...")
|
|
|
|
required_dirs = self._get_tier_requirement("required_dirs", ["scripts"])
|
|
optional_dirs = self._get_tier_requirement("optional_dirs", [])
|
|
|
|
# Check required directories
|
|
missing_required = []
|
|
for dir_name in required_dirs:
|
|
dir_path = self.skill_path / dir_name
|
|
if dir_path.exists() and dir_path.is_dir():
|
|
self.report.add_check(f"dir_{dir_name}_exists", True,
|
|
f"{dir_name}/ directory found", 1.0)
|
|
else:
|
|
missing_required.append(dir_name)
|
|
self.report.add_check(f"dir_{dir_name}_exists", False,
|
|
f"{dir_name}/ directory missing", 0.0)
|
|
|
|
if missing_required:
|
|
self.report.add_error(f"Missing required directories: {', '.join(missing_required)}")
|
|
|
|
# Check optional directories and provide suggestions
|
|
missing_optional = []
|
|
for dir_name in optional_dirs:
|
|
dir_path = self.skill_path / dir_name
|
|
if not (dir_path.exists() and dir_path.is_dir()):
|
|
missing_optional.append(dir_name)
|
|
|
|
if missing_optional:
|
|
self.report.add_suggestion(f"Consider adding optional directories: {', '.join(missing_optional)}")
|
|
|
|
def _validate_python_scripts(self):
|
|
"""Validate Python scripts in the scripts directory"""
|
|
self.log_verbose("Validating Python scripts...")
|
|
|
|
scripts_dir = self.skill_path / "scripts"
|
|
if not scripts_dir.exists():
|
|
return
|
|
|
|
python_files = list(scripts_dir.glob("*.py"))
|
|
min_scripts = self._get_tier_requirement("min_scripts", 1)
|
|
|
|
# Check minimum number of scripts
|
|
if len(python_files) >= min_scripts:
|
|
self.report.add_check("min_scripts_count", True,
|
|
f"Found {len(python_files)} Python scripts (≥{min_scripts})", 1.0)
|
|
else:
|
|
self.report.add_check("min_scripts_count", False,
|
|
f"Found {len(python_files)} Python scripts (<{min_scripts})", 0.0)
|
|
self.report.add_error(f"Insufficient scripts: {len(python_files)}, minimum {min_scripts}")
|
|
|
|
# Validate each script
|
|
for script_path in python_files:
|
|
self._validate_single_script(script_path)
|
|
|
|
def _validate_single_script(self, script_path: Path):
|
|
"""Validate a single Python script"""
|
|
script_name = script_path.name
|
|
self.log_verbose(f"Validating script: {script_name}")
|
|
|
|
try:
|
|
content = script_path.read_text(encoding='utf-8')
|
|
|
|
# Count lines of code (excluding empty lines and comments)
|
|
lines = content.split('\n')
|
|
loc = len([line for line in lines if line.strip() and not line.strip().startswith('#')])
|
|
|
|
# Check script size against tier requirements
|
|
size_range = self._get_tier_requirement("script_size_range", (100, 1000))
|
|
min_size, max_size = size_range
|
|
|
|
if min_size <= loc <= max_size:
|
|
self.report.add_check(f"script_size_{script_name}", True,
|
|
f"{script_name} has {loc} LOC (within {min_size}-{max_size})", 1.0)
|
|
else:
|
|
self.report.add_check(f"script_size_{script_name}", False,
|
|
f"{script_name} has {loc} LOC (outside {min_size}-{max_size})", 0.5)
|
|
if loc < min_size:
|
|
self.report.add_suggestion(f"Consider expanding {script_name} (currently {loc} LOC)")
|
|
else:
|
|
self.report.add_suggestion(f"Consider refactoring {script_name} (currently {loc} LOC)")
|
|
|
|
# Parse and validate Python syntax
|
|
try:
|
|
tree = ast.parse(content)
|
|
self.report.add_check(f"script_syntax_{script_name}", True,
|
|
f"{script_name} has valid Python syntax", 1.0)
|
|
|
|
# Check for required features
|
|
self._validate_script_features(tree, script_name, content)
|
|
|
|
except SyntaxError as e:
|
|
self.report.add_check(f"script_syntax_{script_name}", False,
|
|
f"{script_name} has syntax error: {str(e)}", 0.0)
|
|
self.report.add_error(f"Syntax error in {script_name}: {str(e)}")
|
|
|
|
except Exception as e:
|
|
self.report.add_check(f"script_readable_{script_name}", False,
|
|
f"Cannot read {script_name}: {str(e)}", 0.0)
|
|
self.report.add_error(f"Cannot read {script_name}: {str(e)}")
|
|
|
|
def _validate_script_features(self, tree: ast.AST, script_name: str, content: str):
|
|
"""Validate required script features"""
|
|
required_features = self._get_tier_requirement("features_required", ["argparse", "main_guard"])
|
|
|
|
# Check for argparse usage
|
|
if "argparse" in required_features:
|
|
has_argparse = self._check_argparse_usage(tree)
|
|
self.report.add_check(f"script_argparse_{script_name}", has_argparse,
|
|
f"{'Uses' if has_argparse else 'Missing'} argparse in {script_name}", 1.0 if has_argparse else 0.0)
|
|
if not has_argparse:
|
|
self.report.add_error(f"{script_name} must use argparse for command-line arguments")
|
|
|
|
# Check for main guard
|
|
if "main_guard" in required_features:
|
|
has_main_guard = 'if __name__ == "__main__"' in content
|
|
self.report.add_check(f"script_main_guard_{script_name}", has_main_guard,
|
|
f"{'Has' if has_main_guard else 'Missing'} main guard in {script_name}", 1.0 if has_main_guard else 0.0)
|
|
if not has_main_guard:
|
|
self.report.add_error(f"{script_name} must have 'if __name__ == \"__main__\"' guard")
|
|
|
|
# Check for external imports (should only use stdlib)
|
|
external_imports = self._check_external_imports(tree)
|
|
if not external_imports:
|
|
self.report.add_check(f"script_imports_{script_name}", True,
|
|
f"{script_name} uses only standard library", 1.0)
|
|
else:
|
|
self.report.add_check(f"script_imports_{script_name}", False,
|
|
f"{script_name} uses external imports: {', '.join(external_imports)}", 0.0)
|
|
self.report.add_error(f"{script_name} uses external imports: {', '.join(external_imports)}")
|
|
|
|
def _check_argparse_usage(self, tree: ast.AST) -> bool:
|
|
"""Check if the script uses argparse"""
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name == 'argparse':
|
|
return True
|
|
elif isinstance(node, ast.ImportFrom):
|
|
if node.module == 'argparse':
|
|
return True
|
|
return False
|
|
|
|
def _check_external_imports(self, tree: ast.AST) -> List[str]:
|
|
"""Check for external (non-stdlib) imports.
|
|
|
|
Uses the interpreter's own module list rather than a hand-maintained
|
|
set, which is what this function's original comment asked for. The old
|
|
set omitted `__future__`, so every script using
|
|
`from __future__ import annotations` was reported as carrying an
|
|
external dependency.
|
|
"""
|
|
stdlib_modules = set(getattr(sys, "stdlib_module_names", ()))
|
|
if not stdlib_modules: # Python < 3.10 has no sys.stdlib_module_names
|
|
stdlib_modules = set(sys.builtin_module_names)
|
|
# A compiler directive, not a dependency.
|
|
stdlib_modules.add('__future__')
|
|
|
|
external_imports = []
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
module_name = alias.name.split('.')[0]
|
|
if module_name not in stdlib_modules:
|
|
external_imports.append(alias.name)
|
|
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
module_name = node.module.split('.')[0]
|
|
if module_name not in stdlib_modules:
|
|
external_imports.append(node.module)
|
|
|
|
return list(set(external_imports))
|
|
|
|
def _validate_tier_compliance(self):
|
|
"""Validate overall tier compliance"""
|
|
if not self.target_tier:
|
|
return
|
|
|
|
self.log_verbose(f"Validating {self.target_tier} tier compliance...")
|
|
|
|
# This is a summary check - individual checks are done in other methods
|
|
critical_checks = ["skill_md_exists", "min_scripts_count", "skill_md_length"]
|
|
failed_critical = [check for check in critical_checks
|
|
if check in self.report.checks and not self.report.checks[check]["passed"]]
|
|
|
|
if not failed_critical:
|
|
self.report.add_check("tier_compliance", True,
|
|
f"Meets {self.target_tier} tier requirements", 1.0)
|
|
else:
|
|
self.report.add_check("tier_compliance", False,
|
|
f"Does not meet {self.target_tier} tier requirements", 0.0)
|
|
self.report.add_error(f"Failed critical checks for {self.target_tier} tier: {', '.join(failed_critical)}")
|
|
|
|
def _get_tier_requirement(self, requirement: str, default: Any) -> Any:
|
|
"""Get tier-specific requirement value"""
|
|
if self.target_tier and self.target_tier in self.TIER_REQUIREMENTS:
|
|
return self.TIER_REQUIREMENTS[self.target_tier].get(requirement, default)
|
|
return default
|
|
|
|
|
|
class ReportFormatter:
|
|
"""Formats validation reports for output"""
|
|
|
|
@staticmethod
|
|
def format_json(report: ValidationReport) -> str:
|
|
"""Format report as JSON"""
|
|
return json.dumps({
|
|
"skill_path": report.skill_path,
|
|
"timestamp": report.timestamp,
|
|
"overall_score": round(report.overall_score, 1),
|
|
"compliance_level": report.compliance_level,
|
|
"checks": report.checks,
|
|
"warnings": report.warnings,
|
|
"errors": report.errors,
|
|
"suggestions": report.suggestions
|
|
}, indent=2)
|
|
|
|
@staticmethod
|
|
def format_human_readable(report: ValidationReport) -> str:
|
|
"""Format report as human-readable text"""
|
|
lines = []
|
|
lines.append("=" * 60)
|
|
lines.append("SKILL VALIDATION REPORT")
|
|
lines.append("=" * 60)
|
|
lines.append(f"Skill: {report.skill_path}")
|
|
lines.append(f"Timestamp: {report.timestamp}")
|
|
lines.append(f"Overall Score: {report.overall_score:.1f}/100 ({report.compliance_level})")
|
|
lines.append("")
|
|
|
|
# Group checks by category
|
|
structure_checks = {k: v for k, v in report.checks.items() if k.startswith(('skill_md', 'readme', 'dir_'))}
|
|
script_checks = {k: v for k, v in report.checks.items() if k.startswith('script_')}
|
|
other_checks = {k: v for k, v in report.checks.items() if k not in structure_checks and k not in script_checks}
|
|
|
|
if structure_checks:
|
|
lines.append("STRUCTURE VALIDATION:")
|
|
for check_name, result in structure_checks.items():
|
|
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
|
lines.append(f" {status}: {result['message']}")
|
|
lines.append("")
|
|
|
|
if script_checks:
|
|
lines.append("SCRIPT VALIDATION:")
|
|
for check_name, result in script_checks.items():
|
|
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
|
lines.append(f" {status}: {result['message']}")
|
|
lines.append("")
|
|
|
|
if other_checks:
|
|
lines.append("OTHER CHECKS:")
|
|
for check_name, result in other_checks.items():
|
|
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
|
lines.append(f" {status}: {result['message']}")
|
|
lines.append("")
|
|
|
|
if report.errors:
|
|
lines.append("ERRORS:")
|
|
for error in report.errors:
|
|
lines.append(f" • {error}")
|
|
lines.append("")
|
|
|
|
if report.warnings:
|
|
lines.append("WARNINGS:")
|
|
for warning in report.warnings:
|
|
lines.append(f" • {warning}")
|
|
lines.append("")
|
|
|
|
if report.suggestions:
|
|
lines.append("SUGGESTIONS:")
|
|
for suggestion in report.suggestions:
|
|
lines.append(f" • {suggestion}")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def main():
|
|
"""Main entry point"""
|
|
parser = argparse.ArgumentParser(
|
|
description="Validate skill directories against quality standards",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
python skill_validator.py engineering/my-skill
|
|
python skill_validator.py engineering/my-skill --tier POWERFUL --json
|
|
python skill_validator.py engineering/my-skill --verbose
|
|
|
|
Tier Options:
|
|
BASIC - Basic skill requirements (100+ lines SKILL.md, 1+ script)
|
|
STANDARD - Standard skill requirements (200+ lines, advanced features)
|
|
POWERFUL - Powerful skill requirements (300+ lines, comprehensive features)
|
|
"""
|
|
)
|
|
|
|
parser.add_argument("skill_path",
|
|
help="Path to the skill directory to validate")
|
|
parser.add_argument("--tier",
|
|
choices=["BASIC", "STANDARD", "POWERFUL"],
|
|
help="Target tier for validation (optional)")
|
|
parser.add_argument("--json",
|
|
action="store_true",
|
|
help="Output results in JSON format")
|
|
parser.add_argument("--verbose",
|
|
action="store_true",
|
|
help="Enable verbose logging")
|
|
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
# Create validator and run validation
|
|
validator = SkillValidator(args.skill_path, args.tier, args.verbose)
|
|
report = validator.validate_skill_structure()
|
|
|
|
# Format and output report
|
|
if args.json:
|
|
print(ReportFormatter.format_json(report))
|
|
else:
|
|
print(ReportFormatter.format_human_readable(report))
|
|
|
|
# Exit with error code if validation failed
|
|
if report.errors or report.overall_score < 60:
|
|
sys.exit(1)
|
|
else:
|
|
sys.exit(0)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\nValidation interrupted by user", file=sys.stderr)
|
|
sys.exit(130)
|
|
except Exception as e:
|
|
print(f"Validation failed: {str(e)}", file=sys.stderr)
|
|
if args.verbose:
|
|
import traceback
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|