diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 9382d933..3329a267 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -50,6 +50,8 @@ jobs: build-mode: manual - language: javascript-typescript build-mode: none + - language: python + build-mode: none steps: - name: Check out repository diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 681a1b40..a58d031d 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -63,6 +63,8 @@ cp secret.yaml.example secret.yaml | oauth2-github-client-id | GitHub OAuth ID | 否 | | oauth2-github-client-secret | GitHub OAuth 密钥 | 否 | | skill-scanner-llm-api-key | LLM API 密钥 | 否 | +| skill-scanner-llm-base-url | 本地/自定义 LLM 服务地址 | 否 | +| skill-scanner-llm-model | Scanner 使用的 LLM 模型名 | 否 | ### 3. 选择部署方式 @@ -192,6 +194,7 @@ kubectl apply -k overlays/with-infra/ # 或 overlays/external/ | oauth2-github-client-id | GitHub OAuth ID | 否 | | oauth2-github-client-secret | GitHub OAuth 密钥 | 否 | | skill-scanner-llm-api-key | LLM API 密钥 | 否 | +| skill-scanner-llm-base-url | 本地/自定义 LLM 服务地址 | 否 | | skill-scanner-llm-model | LLM 模型名称 | 否 | ### 存储配置 diff --git a/deploy/k8s/base/scanner-deployment.yaml b/deploy/k8s/base/scanner-deployment.yaml index 9cff8b93..91c7f3e3 100644 --- a/deploy/k8s/base/scanner-deployment.yaml +++ b/deploy/k8s/base/scanner-deployment.yaml @@ -28,6 +28,12 @@ spec: name: skillhub-secret key: skill-scanner-llm-api-key optional: true + - name: SKILL_SCANNER_LLM_BASE_URL + valueFrom: + secretKeyRef: + name: skillhub-secret + key: skill-scanner-llm-base-url + optional: true - name: SKILL_SCANNER_LLM_MODEL valueFrom: secretKeyRef: diff --git a/deploy/k8s/base/secret.yaml.example b/deploy/k8s/base/secret.yaml.example index 41b9ea5c..5ff967cc 100644 --- a/deploy/k8s/base/secret.yaml.example +++ b/deploy/k8s/base/secret.yaml.example @@ -24,6 +24,7 @@ stringData: # LLM 配置（可选，用于技能扫描） skill-scanner-llm-api-key: "" + skill-scanner-llm-base-url: "" skill-scanner-llm-model: "" # S3 存储配置（可选，使用 S3/OSS 时配置） diff --git a/docs/security-scanning.md b/docs/security-scanning.md index 2fab9717..bd8d80fc 100644 --- a/docs/security-scanning.md +++ b/docs/security-scanning.md @@ -61,6 +61,7 @@ Important environment variables: Scanner-side optional environment variables: - `SKILL_SCANNER_LLM_API_KEY` +- `SKILL_SCANNER_LLM_BASE_URL` - `SKILL_SCANNER_LLM_MODEL` If the LLM variables are absent, the scanner should still run with non-LLM analyzers. diff --git a/docs/skillhub/en/guide/kubernetes.md b/docs/skillhub/en/guide/kubernetes.md index 73eb2cee..c56caf9a 100644 --- a/docs/skillhub/en/guide/kubernetes.md +++ b/docs/skillhub/en/guide/kubernetes.md @@ -63,6 +63,8 @@ cp secret.yaml.example secret.yaml | oauth2-github-client-id | GitHub OAuth ID | No | | oauth2-github-client-secret | GitHub OAuth secret | No | | skill-scanner-llm-api-key | LLM API key | No | +| skill-scanner-llm-base-url | Local/custom LLM service base URL | No | +| skill-scanner-llm-model | LLM model name used by the scanner | No | ### 3. Choose Deployment Method diff --git a/docs/skillhub/en/guide/scanner.md b/docs/skillhub/en/guide/scanner.md index 20f48139..5c11902d 100644 --- a/docs/skillhub/en/guide/scanner.md +++ b/docs/skillhub/en/guide/scanner.md @@ -79,6 +79,8 @@ Enabling the LLM analysis engine can improve the accuracy of security detection: | `SKILLHUB_SCANNER_USE_LLM` | Enable LLM analysis | `false` | | `SKILLHUB_SCANNER_LLM_PROVIDER` | LLM provider (anthropic / openai / azure) | `anthropic` | | `SKILL_SCANNER_LLM_API_KEY` | LLM API key | - | +| `SKILL_SCANNER_LLM_BASE_URL` | Local/custom LLM service base URL | - | +| `SKILL_SCANNER_LLM_MODEL` | LLM model name | - | ### Deployment Notes diff --git a/docs/skillhub/guide/kubernetes.md b/docs/skillhub/guide/kubernetes.md index de8f505e..9a4b3a8d 100644 --- a/docs/skillhub/guide/kubernetes.md +++ b/docs/skillhub/guide/kubernetes.md @@ -63,6 +63,8 @@ cp secret.yaml.example secret.yaml | oauth2-github-client-id | GitHub OAuth ID | 否 | | oauth2-github-client-secret | GitHub OAuth 密钥 | 否 | | skill-scanner-llm-api-key | LLM API 密钥 | 否 | +| skill-scanner-llm-base-url | 本地/自定义 LLM 服务地址 | 否 | +| skill-scanner-llm-model | Scanner 使用的 LLM 模型名 | 否 | ### 3. 选择部署方式 diff --git a/docs/skillhub/guide/scanner.md b/docs/skillhub/guide/scanner.md index 8cbdc134..33837cbb 100644 --- a/docs/skillhub/guide/scanner.md +++ b/docs/skillhub/guide/scanner.md @@ -79,6 +79,8 @@ Skill Scanner 执行多引擎分析 | `SKILLHUB_SCANNER_USE_LLM` | 启用 LLM 分析 | `false` | | `SKILLHUB_SCANNER_LLM_PROVIDER` | LLM 提供商（anthropic / openai / azure） | `anthropic` | | `SKILL_SCANNER_LLM_API_KEY` | LLM API 密钥 | - | +| `SKILL_SCANNER_LLM_BASE_URL` | 本地/自定义 LLM 服务地址 | - | +| `SKILL_SCANNER_LLM_MODEL` | LLM 模型名称 | - | ### 部署说明 diff --git a/scanner/Dockerfile b/scanner/Dockerfile index cb0c82c8..90eac126 100644 --- a/scanner/Dockerfile +++ b/scanner/Dockerfile @@ -1,10 +1,14 @@ FROM python:3.11-alpine +ARG SKILL_SCANNER_VERSION=1.0.2 + WORKDIR /app -RUN apk add --no-cache --virtual .build-deps gcc musl-dev libffi-dev && \ - pip install --no-cache-dir cisco-ai-skill-scanner && \ - apk del .build-deps && \ +COPY backports/apply_1_0_2_llm_base_url_backport.py /tmp/apply_1_0_2_llm_base_url_backport.py + +RUN pip install --no-cache-dir "cisco-ai-skill-scanner==${SKILL_SCANNER_VERSION}" && \ + python /tmp/apply_1_0_2_llm_base_url_backport.py /usr/local/lib/python3.11/site-packages && \ + rm /tmp/apply_1_0_2_llm_base_url_backport.py && \ addgroup -S app && \ adduser -S app -G app && \ mkdir -p /tmp/skillhub-scans && \ diff --git a/scanner/backports/apply_1_0_2_llm_base_url_backport.py b/scanner/backports/apply_1_0_2_llm_base_url_backport.py new file mode 100644 index 00000000..7ea6bbd6 --- /dev/null +++ b/scanner/backports/apply_1_0_2_llm_base_url_backport.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Backport SKILL_SCANNER_LLM_BASE_URL support into cisco-ai-skill-scanner 1.0.2.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +EXPECTED_DIST_INFO = "cisco_ai_skill_scanner-1.0.2.dist-info" +ROUTER_RELATIVE_PATH = Path("skill_scanner/api/router.py") + +def replace_exact(content: str, old: str, new: str, expected_count: int, label: str) -> str: + actual_count = content.count(old) + if actual_count != expected_count: + raise SystemExit(f"Expected {expected_count} occurrences of {label}, found {actual_count}.") + return content.replace(old, new, expected_count) + + +def replace_regex(content: str, pattern: str, replacement: str, expected_count: int, label: str) -> str: + updated, actual_count = re.subn(pattern, replacement, content, count=expected_count, flags=re.MULTILINE) + if actual_count != expected_count: + raise SystemExit(f"Expected {expected_count} regex replacements for {label}, found {actual_count}.") + return updated + + +def main() -> int: + site_packages = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/usr/local/lib/python3.11/site-packages") + dist_info = site_packages / EXPECTED_DIST_INFO + if not dist_info.exists(): + raise SystemExit(f"Expected {EXPECTED_DIST_INFO} under {site_packages}, but it was not found.") + + router_path = site_packages / ROUTER_RELATIVE_PATH + content = router_path.read_text(encoding="utf-8") + content = replace_regex( + content, + r'^(?P\s*)llm_model = os.getenv\("SKILL_SCANNER_LLM_MODEL"\)$', + r'\g<0>\n\gllm_base_url = os.getenv("SKILL_SCANNER_LLM_BASE_URL")', + 2, + "llm_model environment lookup", + ) + content = replace_exact( + content, + "LLMAnalyzer(model=llm_model)", + "LLMAnalyzer(model=llm_model, base_url=llm_base_url)", + 2, + "LLMAnalyzer model constructor", + ) + content = replace_exact( + content, + "LLMAnalyzer(provider=provider_str)", + "LLMAnalyzer(provider=provider_str, base_url=llm_base_url)", + 2, + "LLMAnalyzer provider constructor", + ) + + router_path.write_text(content, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/scanner-llm-base-url-test.sh b/scripts/tests/scanner-llm-base-url-test.sh new file mode 100755 index 00000000..79378b61 --- /dev/null +++ b/scripts/tests/scanner-llm-base-url-test.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCANNER_DIR="$REPO_ROOT/scanner" +TMP_DIRS=() + +cleanup() { + local status=$? + local d + for d in "${TMP_DIRS[@]+"${TMP_DIRS[@]}"}"; do + rm -rf "$d" + done + exit "$status" +} +trap cleanup EXIT + +new_tmp() { + local d + d="$(mktemp -d)" + TMP_DIRS+=("$d") + echo "$d" +} + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +tmp="$(new_tmp)" +skill_dir="$tmp/skill" +mkdir -p "$skill_dir/demo-skill" + +cat >"$skill_dir/demo-skill/SKILL.md" <<'EOF' +--- +name: demo-skill +description: Minimal valid skill used for scanner integration coverage. +license: Apache-2.0 +--- + +This is a harmless demo skill used for scanner integration testing. +EOF + +cat >"$skill_dir/demo-skill/run.sh" <<'EOF' +#!/usr/bin/env sh +echo "demo" +EOF +chmod +x "$skill_dir/demo-skill/run.sh" + +IMAGE_TAG="skillhub-scanner-llm-base-url-test:$(date +%s)" +docker build --no-cache -t "$IMAGE_TAG" "$SCANNER_DIR" >/dev/null + +docker run --rm -i \ + -v "$skill_dir:/work/skill:ro" \ + --entrypoint python \ + "$IMAGE_TAG" - <<'PY' +import asyncio +from datetime import datetime, timezone +import http.server +import io +import inspect +import json +import os +from pathlib import Path +import threading +import urllib.request +import zipfile + +from fastapi.params import Query +from skill_scanner.core.models import ScanResult +import skill_scanner.api.router as router + +signature = inspect.signature(router.scan_uploaded_skill) +if not isinstance(signature.parameters["use_llm"].default, Query): + raise SystemExit("scan-upload use_llm should remain a Query parameter") +if not isinstance(signature.parameters["llm_provider"].default, Query): + raise SystemExit("scan-upload llm_provider should remain a Query parameter") + +state = {"base_urls": [], "paths": []} + + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, format, *args): # noqa: A003 + return + + def do_POST(self): # noqa: N802 + length = int(self.headers.get("content-length", "0")) + self.rfile.read(length) + state["paths"].append(self.path) + + payload = json.dumps( + { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": int(datetime.now(timezone.utc).timestamp()), + "model": "local-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "No findings."}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +server = http.server.HTTPServer(("127.0.0.1", 0), Handler) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() + +target_base_url = f"http://127.0.0.1:{server.server_port}/v1" +os.environ["SKILL_SCANNER_LLM_BASE_URL"] = target_base_url +os.environ["SKILL_SCANNER_LLM_MODEL"] = "test-model" + + +class FakeStaticAnalyzer: + pass + + +class FakeLLMAnalyzer: + def __init__(self, model=None, provider=None, base_url=None): + self.model = model + self.provider = provider + self.base_url = base_url + state["base_urls"].append(base_url) + + def analyze(self, skill_path): + request = urllib.request.Request( + self.base_url + "/chat/completions", + data=b"{}", + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + response.read() + + +class FakeSkillScanner: + def __init__(self, analyzers): + self.analyzers = analyzers + + def scan_skill(self, skill_path): + for analyzer in self.analyzers: + analyze = getattr(analyzer, "analyze", None) + if callable(analyze): + analyze(skill_path) + + return ScanResult( + skill_name="demo-skill", + skill_directory=str(skill_path), + findings=[], + scan_duration_seconds=0.05, + analyzers_used=["fake-llm"], + timestamp=datetime.now(timezone.utc), + ) + + +router.StaticAnalyzer = FakeStaticAnalyzer +router.LLMAnalyzer = FakeLLMAnalyzer +router.SkillScanner = FakeSkillScanner +router.LLM_AVAILABLE = True + +request = router.ScanRequest( + skill_directory="/work/skill/demo-skill", + use_llm=True, + llm_provider="openai", + use_behavioral=False, + use_aidefense=False, + aidefense_api_key=None, +) + +def build_skill_archive_bytes(skill_root: str) -> bytes: + skill_path = Path(skill_root) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in skill_path.rglob("*"): + if path.is_file(): + archive.writestr(str(path.relative_to(skill_path.parent)), path.read_bytes()) + return buffer.getvalue() + +class FakeUploadFile: + def __init__(self, filename: str, payload: bytes): + self.filename = filename + self._payload = payload + + async def read(self) -> bytes: + return self._payload + +try: + direct_response = asyncio.run(router.scan_skill(request)) + + upload_response = asyncio.run( + router.scan_uploaded_skill( + file=FakeUploadFile("demo-skill.zip", build_skill_archive_bytes("/work/skill/demo-skill")), + use_llm=True, + llm_provider="openai", + use_behavioral=False, + use_aidefense=False, + aidefense_api_key=None, + ) + ) +finally: + server.shutdown() + thread.join(timeout=5) + +if not getattr(direct_response, "scan_id", None): + raise SystemExit("scan_skill should still return a scan response") +if not getattr(upload_response, "scan_id", None): + raise SystemExit("scan_uploaded_skill should still return a scan response") +if len(state["base_urls"]) != 2: + raise SystemExit(f"expected two LLM analyzer constructions, got {len(state['base_urls'])}") +if any(base_url != target_base_url for base_url in state["base_urls"]): + raise SystemExit(f"expected every base_url to be {target_base_url}, got {state['base_urls']}") +if len(state["paths"]) != 2: + raise SystemExit(f"expected two LLM requests, got {state['paths']}") +if not all(path.startswith("/v1/") for path in state["paths"]): + raise SystemExit(f"expected every request path to start with /v1/, got {state['paths']}") +PY + +grep -Fq "name: SKILL_SCANNER_LLM_BASE_URL" "$REPO_ROOT/deploy/k8s/base/scanner-deployment.yaml" \ + || fail "Kubernetes scanner deployment must expose SKILL_SCANNER_LLM_BASE_URL" +grep -Fq "skill-scanner-llm-base-url" "$REPO_ROOT/deploy/k8s/base/secret.yaml.example" \ + || fail "Kubernetes secret example must document skill-scanner-llm-base-url" + +echo "scanner-llm-base-url-test passed"