From 7f934e63aba46f8b4b05104e8872502e747ac1c0 Mon Sep 17 00:00:00 2001 From: wowo-zZ Date: Thu, 30 Jul 2026 15:51:43 +0800 Subject: [PATCH] feat(bootstrap): verify built-in skill artifacts Signed-off-by: wowo-zZ --- .env.release.draft | 3 + .env.release.example | 3 + builtin-skills/README.md | 5 +- compose.release.yml | 1 + docs/20-cloud-url-builtin-skills-setup.md | 38 +++-- docs/21-official-starter-skills-plan.md | 3 +- docs/22-builtin-skills-candidate-pool.md | 8 +- scripts/tests/validate-release-config-test.sh | 11 ++ scripts/validate-release-config.sh | 1 + .../bootstrap/BuiltinSkillInitializer.java | 16 +- .../bootstrap/BuiltinSkillManifestLoader.java | 24 ++- .../resources/builtin-skills/manifest.json | 6 +- .../BuiltinSkillInitializerTest.java | 36 ++++- .../BuiltinSkillManifestLoaderTest.java | 141 ++++++++++++++++-- 14 files changed, 250 insertions(+), 46 deletions(-) diff --git a/.env.release.draft b/.env.release.draft index c994b08e..ea870991 100644 --- a/.env.release.draft +++ b/.env.release.draft @@ -58,6 +58,9 @@ WEB_PORT=80 # Must stay true when the public site is behind HTTPS. SESSION_COOKIE_SECURE=true +# Built-in starter skills are installed by default. Set to false to skip initialization. +SKILLHUB_BUILTIN_SKILLS_ENABLED=true + # External object storage. Production should use s3. SKILLHUB_STORAGE_PROVIDER=s3 diff --git a/.env.release.example b/.env.release.example index 59387e0f..f8792aff 100644 --- a/.env.release.example +++ b/.env.release.example @@ -129,6 +129,9 @@ SKILLHUB_AUTH_PASSWORD_RESET_FROM_NAME=SkillHub # Security scanner is enabled by default. Set to false to disable scanning. SKILLHUB_SECURITY_SCANNER_ENABLED=true +# Built-in starter skills are installed by default. Set to false to skip initialization. +SKILLHUB_BUILTIN_SKILLS_ENABLED=true + # Required for signing anonymous download rate-limit cookies. Use a unique random value per deployment. # runtime.sh generates and persists one automatically when this placeholder is still present. SKILLHUB_DOWNLOAD_ANON_COOKIE_SECRET=replace-with-random-download-secret-32-bytes diff --git a/builtin-skills/README.md b/builtin-skills/README.md index 1ebd7a69..31302b4c 100644 --- a/builtin-skills/README.md +++ b/builtin-skills/README.md @@ -20,7 +20,10 @@ make test-builtin-skills The build writes deterministic, uncompressed ZIPs and `artifacts.json` to `builtin-skills/dist/`. The artifact index records each ZIP's SHA-256 for the release step; runtime -manifest integration is maintained separately from the reviewed source collection. +manifest integration is maintained separately from the reviewed source collection. A package is +added to the runtime manifest only after its immutable CDN URL is available; the manifest records +the matching SHA-256 so the backend can reject changed or incorrectly uploaded bytes before +extraction. ## Share a Skill with the Community diff --git a/compose.release.yml b/compose.release.yml index 73318768..b7cf400b 100644 --- a/compose.release.yml +++ b/compose.release.yml @@ -87,6 +87,7 @@ services: SKILLHUB_SECURITY_SCANNER_ENABLED: ${SKILLHUB_SECURITY_SCANNER_ENABLED:-true} SKILLHUB_SECURITY_SCANNER_URL: http://skill-scanner:8000 SKILLHUB_SECURITY_SCANNER_MODE: upload + SKILLHUB_BUILTIN_SKILLS_ENABLED: ${SKILLHUB_BUILTIN_SKILLS_ENABLED:-true} SKILLHUB_AUTH_DIRECT_ENABLED: ${SKILLHUB_AUTH_DIRECT_ENABLED:-false} BOOTSTRAP_ADMIN_ENABLED: ${BOOTSTRAP_ADMIN_ENABLED:-false} BOOTSTRAP_ADMIN_USER_ID: ${BOOTSTRAP_ADMIN_USER_ID:-docker-admin} diff --git a/docs/20-cloud-url-builtin-skills-setup.md b/docs/20-cloud-url-builtin-skills-setup.md index 186c1ec5..3d2a20f4 100644 --- a/docs/20-cloud-url-builtin-skills-setup.md +++ b/docs/20-cloud-url-builtin-skills-setup.md @@ -30,11 +30,12 @@ scripts/build-builtin-skills.py server/skillhub-app/src/main/resources/builtin-skills/manifest.json ``` -首版 manifest 只需要维护三个字段： +manifest 需要维护四个字段： - `slug`：Skill 在 `@global` 下的 slug。 - `version`：期望同步的 Skill 版本。 - `url`：Skill zip 包的云存储 HTTPS 链接。 +- `sha256`：发布制品的 SHA-256，小写 64 位十六进制字符串。 ## 2. Manifest 配置 @@ -46,7 +47,8 @@ manifest 文件格式如下： { "slug": "skillhub-hello", "version": "1.0.0", - "url": "https://bjcdn.openstorage.cn//skillhub-hello-1.0.0.zip" + "url": "https://bjcdn.openstorage.cn/aicontest/2026-06-11/f8a59af3-30d4-4031-80f6-ebff74b05195.zip", + "sha256": "acb591ed0891e735c364b955f5b94b2b9ce567c1d9e347312cebfbfde2d93f57" } ] } @@ -60,17 +62,20 @@ manifest 文件格式如下： { "slug": "skillhub-hello", "version": "1.0.0", - "url": "https://bjcdn.openstorage.cn//skillhub-hello-1.0.0.zip" + "url": "https://bjcdn.openstorage.cn//skillhub-hello-1.0.0.zip", + "sha256": "" }, { "slug": "skillhub-hello", "version": "1.1.0", - "url": "https://bjcdn.openstorage.cn//skillhub-hello-1.1.0.zip" + "url": "https://bjcdn.openstorage.cn//skillhub-hello-1.1.0.zip", + "sha256": "" }, { "slug": "skillhub-guide", "version": "1.0.0", - "url": "https://bjcdn.openstorage.cn//skillhub-guide-1.0.0.zip" + "url": "https://bjcdn.openstorage.cn//skillhub-guide-1.0.0.zip", + "sha256": "" } ] } @@ -79,8 +84,9 @@ manifest 文件格式如下： 配置要求： - `skills` 必须是数组。 -- 每一项必须同时填写 `slug`、`version`、`url`。 +- 每一项必须同时填写 `slug`、`version`、`url`、`sha256`。 - `slug` 必须符合 SkillHub slug 规则。 +- `sha256` 必须是小写 64 位十六进制字符串，并与 URL 返回的原始 zip 字节一致。 - 同一个 `slug + version` 重复出现时，只处理第一条，后续重复项会被跳过。 - manifest 最多处理前 100 条 entries。 - 同一个 `slug` 的多个版本建议按从旧到新的顺序排列；运行时按 manifest 文件顺序处理，不做自动版本排序。 @@ -155,11 +161,12 @@ skillhub-hello-1.0.0.zip 7. 按 manifest 顺序处理每一个 item。 8. 下载前先检查 `@global/{slug}` 和目标版本是否已经存在；如果已经确定应跳过，则不发起远程下载。 9. 只有需要发布新 Skill 或新版本时，才下载对应 zip 包。 -10. 解包并校验 Skill 入口 `SKILL.md`。 -11. 校验 manifest 中的 `slug`、`version` 与包内元数据一致。 -12. 发布前再次检查是否已存在同名 Skill 或同版本，处理并发启动场景。 -13. 需要发布时调用现有 `SkillPublishService.publishFromEntries(...)`。 -14. 发布完成后，该 Skill 位于 `@global/{slug}`，可见性为 `PUBLIC`。 +10. 对下载到的原始 zip 字节计算 SHA-256，并与 manifest 的 `sha256` 比较；不一致时停止处理该项。 +11. 解包并校验 Skill 入口 `SKILL.md`。 +12. 校验 manifest 中的 `slug`、`version` 与包内元数据一致。 +13. 发布前再次检查是否已存在同名 Skill 或同版本，处理并发启动场景。 +14. 需要发布时调用现有 `SkillPublishService.publishFromEntries(...)`。 +15. 发布完成后，该 Skill 位于 `@global/{slug}`，可见性为 `PUBLIC`。 同步逻辑不会直接写数据库 seed 数据。它复用现有发布服务，因此会保留现有的包校验、对象存储写入、版本记录、latest version 更新、事件和搜索索引同步。 @@ -223,7 +230,7 @@ SKILLHUB_BUILTIN_SKILLS_ENABLED=false 3. 运行 `make test-builtin-skills`，确认包结构、来源、许可证和确定性构建门禁通过。 4. 运行 `make build-builtin-skills`，从 `builtin-skills/dist/artifacts.json` 读取制品哈希。 5. 上传 zip 到 `bjcdn.openstorage.cn` 或其子域名下的不可变路径。 -6. 在 `server/skillhub-app/src/main/resources/builtin-skills/manifest.json` 中新增一项。 +6. 在 `server/skillhub-app/src/main/resources/builtin-skills/manifest.json` 中新增一项，同时填写制品 URL 和 `artifacts.json` 中对应的 SHA-256。 7. 本地或测试环境启动 SkillHub，查看后端日志确认同步结果。 8. 在 Web UI 或 API 中确认 `@global/{slug}` 已公开可见。 @@ -251,10 +258,12 @@ SKILLHUB_BUILTIN_SKILLS_ENABLED=false |---|---| | manifest not found | 确认 `builtin-skills/manifest.json` 是否被打进 classpath | | publisher account id already exists but is not a system account | `builtin-skill-publisher` 已被普通账号占用；需要人工处理账号冲突后再启用内置同步 | -| slug, version, and url are required | 检查 manifest item 是否缺字段或字段不是字符串 | +| slug, version, url, and sha256 are required | 检查 manifest item 是否缺字段或字段不是字符串 | | slug is invalid | 检查 slug 是否符合 SkillHub slug 规则 | +| sha256 must be 64 lowercase hexadecimal characters | 使用 `builtin-skills/dist/artifacts.json` 中对应制品的 SHA-256 | | URL is not allowed | 检查 URL 是否为 HTTPS、host 是否为 `bjcdn.openstorage.cn` 或其子域名 | | package download failed | 检查云存储对象是否存在、是否返回 HTTP 200、是否超时 | +| package checksum mismatch | 云端对象与 manifest 固定的制品不一致；不要继续解包或发布，检查是否上传错误或对象被覆盖 | | package must contain SKILL.md | 检查 zip 是否存在唯一可识别的 `SKILL.md` 入口 | | manifest version does not match package version | 检查 manifest `version` 和 `SKILL.md version` 是否一致 | | slug already belongs to another user | 说明 `@global/{slug}` 已被非内置发布者创建或发布，内置同步不会覆盖 | @@ -268,7 +277,8 @@ SKILLHUB_BUILTIN_SKILLS_ENABLED=false - `make test-builtin-skills` 通过，且 15 个回归用例都有对应包。 - manifest JSON 格式合法。 -- 每个 item 都包含 `slug`、`version`、`url`。 +- 每个 item 都包含 `slug`、`version`、`url`、`sha256`。 +- 每个 `sha256` 都与 URL 下载到的原始 zip 字节一致。 - URL 使用 `https://bjcdn.openstorage.cn/...` 或可信子域名。 - zip 根目录直接包含 `SKILL.md`，或只有一个顶层 Skill 目录且该目录包含 `SKILL.md`。 - `SKILL.md name` 归一化后的 slug 与 manifest `slug` 一致。 diff --git a/docs/21-official-starter-skills-plan.md b/docs/21-official-starter-skills-plan.md index 691c8d90..9d03e346 100644 --- a/docs/21-official-starter-skills-plan.md +++ b/docs/21-official-starter-skills-plan.md @@ -19,7 +19,8 @@ SkillHub 已经支持通过 manifest 将远程 Skill 包同步到 `@global`。 2. 完成协议、许可证、安全和可用性检查。 3. 将审查后源码、来源和回归用例纳入开源仓库。 4. 确定性打包并上传官方云存储。 -5. 加入 `builtin-skills/manifest.json`。 +5. 将不可变 URL 和制品 SHA-256 加入 + `server/skillhub-app/src/main/resources/builtin-skills/manifest.json`。 6. 随 SkillHub 默认部署同步到 `@global`。 目标总量为 30～50 个。当前已有的 `skillhub-hello` 和 `agentguard` 也需要重新检查，通过质量 diff --git a/docs/22-builtin-skills-candidate-pool.md b/docs/22-builtin-skills-candidate-pool.md index 5c98bd48..9be85d60 100644 --- a/docs/22-builtin-skills-candidate-pool.md +++ b/docs/22-builtin-skills-candidate-pool.md @@ -206,7 +206,9 @@ B25 的核心行程生成实测失败，并会把敏感旅行画像写入宿主 4. 在声明支持的 Agent 中完成至少一个真实任务，并记录输入、预期结果和实际结果。 5. 运行 SkillHub 包校验、安全扫描和人工审查。 6. 使用新版本号打包，上传不可变的官方 CDN URL。 -7. 更新 `builtin-skills/manifest.json`，在干净部署中验证同步、下载和安装。 +7. 将不可变 URL 和制品 SHA-256 写入 + `server/skillhub-app/src/main/resources/builtin-skills/manifest.json`，在干净部署中验证同步、下载和安装。 -当前不修改 manifest：候选包尚未完成以上检查，也没有可用的官方不可变 URL。把 GitHub -分支下载地址直接写进 manifest，会导致上游内容变化后无法审计和复现。 +第一批 15 个适配包已经完成内容、安全和确定性构建检查；在官方不可变 CDN URL 可用前，暂不把 +它们写入运行时 manifest。把 GitHub 分支下载地址直接写进 manifest，会导致内容变化后无法审计 +和复现。 diff --git a/scripts/tests/validate-release-config-test.sh b/scripts/tests/validate-release-config-test.sh index bee780fb..af876a9c 100755 --- a/scripts/tests/validate-release-config-test.sh +++ b/scripts/tests/validate-release-config-test.sh @@ -37,6 +37,7 @@ POSTGRES_PASSWORD=strong-postgres-password SESSION_COOKIE_SECURE=true BOOTSTRAP_ADMIN_ENABLED=false SKILLHUB_TRUST_FORWARDED_PROTO=false +SKILLHUB_BUILTIN_SKILLS_ENABLED=true SKILLHUB_STORAGE_PROVIDER=s3 SKILLHUB_STORAGE_S3_ENDPOINT=https://storage.example.com SKILLHUB_STORAGE_S3_BUCKET=skillhub @@ -69,6 +70,11 @@ valid_env="$tmp/valid.env" write_env "$valid_env" "release-download-secret-32-bytes-minimum" "$SCRIPT" "$valid_env" >/dev/null +disabled_builtin_skills_env="$tmp/disabled-builtin-skills.env" +write_env "$disabled_builtin_skills_env" "release-download-secret-32-bytes-minimum" +printf '%s\n' "SKILLHUB_BUILTIN_SKILLS_ENABLED=false" >>"$disabled_builtin_skills_env" +"$SCRIPT" "$disabled_builtin_skills_env" >/dev/null + missing_env="$tmp/missing.env" write_env "$missing_env" "" no expect_fail "$missing_env" "SKILLHUB_DOWNLOAD_ANON_COOKIE_SECRET is required" @@ -86,6 +92,11 @@ write_env "$invalid_forwarded_proto_env" "release-download-secret-32-bytes-minim printf '%s\n' "SKILLHUB_TRUST_FORWARDED_PROTO=yes" >>"$invalid_forwarded_proto_env" expect_fail "$invalid_forwarded_proto_env" "SKILLHUB_TRUST_FORWARDED_PROTO must be true or false" +invalid_builtin_skills_env="$tmp/invalid-builtin-skills.env" +write_env "$invalid_builtin_skills_env" "release-download-secret-32-bytes-minimum" +printf '%s\n' "SKILLHUB_BUILTIN_SKILLS_ENABLED=yes" >>"$invalid_builtin_skills_env" +expect_fail "$invalid_builtin_skills_env" "SKILLHUB_BUILTIN_SKILLS_ENABLED must be true or false" + valid_redis_cluster_env="$tmp/valid-redis-cluster.env" write_env "$valid_redis_cluster_env" "release-download-secret-32-bytes-minimum" cat >>"$valid_redis_cluster_env" <<'EOF' diff --git a/scripts/validate-release-config.sh b/scripts/validate-release-config.sh index 7c7940a3..7549a782 100755 --- a/scripts/validate-release-config.sh +++ b/scripts/validate-release-config.sh @@ -220,6 +220,7 @@ reject_patterns SPRING_MAIL_PASSWORD "TODO_*" "todo_*" "replace*" validate_boolean SESSION_COOKIE_SECURE validate_boolean BOOTSTRAP_ADMIN_ENABLED validate_boolean SKILLHUB_TRUST_FORWARDED_PROTO +validate_boolean SKILLHUB_BUILTIN_SKILLS_ENABLED validate_boolean SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE validate_boolean SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET validate_boolean SPRING_DATA_REDIS_SSL_ENABLED diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializer.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializer.java index 7ed8c00d..4689176f 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializer.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializer.java @@ -209,7 +209,21 @@ public class BuiltinSkillInitializer { return SyncOutcome.FAILED; } - SkillPackageArchiveExtractor.ExtractionResult extractionResult = extractor.extract(packageBytes.get()); + byte[] downloadedBytes = packageBytes.get(); + String actualSha256 = sha256(downloadedBytes); + if (!item.sha256().equals(actualSha256)) { + log.warn( + "Skipping built-in skill slug={} version={} because package checksum mismatch: " + + "expectedSha256={}, actualSha256={}", + item.slug(), + item.version(), + item.sha256(), + actualSha256 + ); + return SyncOutcome.FAILED; + } + + SkillPackageArchiveExtractor.ExtractionResult extractionResult = extractor.extract(downloadedBytes); List entries = extractionResult.entries(); SkillMetadata metadata = parseSkillMetadata(entries); String packageSlug = SlugValidator.slugify(metadata.name()); diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoader.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoader.java index 487179a4..e5f2a708 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoader.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoader.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.regex.Pattern; @Component public class BuiltinSkillManifestLoader { @@ -24,6 +25,7 @@ public class BuiltinSkillManifestLoader { static final int MAX_ITEMS = 100; private static final Logger log = LoggerFactory.getLogger(BuiltinSkillManifestLoader.class); + private static final Pattern SHA256_PATTERN = Pattern.compile("[0-9a-f]{64}"); private final ObjectMapper objectMapper; private final ResourceLoader resourceLoader; @@ -72,8 +74,22 @@ public class BuiltinSkillManifestLoader { String slug = text(itemNode, "slug"); String version = text(itemNode, "version"); String url = text(itemNode, "url"); - if (!StringUtils.hasText(slug) || !StringUtils.hasText(version) || !StringUtils.hasText(url)) { - log.warn("Skipping built-in skill manifest item {} because slug, version, and url are required", index); + String sha256 = text(itemNode, "sha256"); + if (!StringUtils.hasText(slug) + || !StringUtils.hasText(version) + || !StringUtils.hasText(url) + || !StringUtils.hasText(sha256)) { + log.warn( + "Skipping built-in skill manifest item {} because slug, version, url, and sha256 are required", + index + ); + continue; + } + if (!SHA256_PATTERN.matcher(sha256).matches()) { + log.warn( + "Skipping built-in skill manifest item {} because sha256 must be 64 lowercase hexadecimal characters", + index + ); continue; } try { @@ -90,7 +106,7 @@ public class BuiltinSkillManifestLoader { continue; } - items.add(new ManifestItem(slug, version, url)); + items.add(new ManifestItem(slug, version, url, sha256)); } return List.copyOf(items); } @@ -103,6 +119,6 @@ public class BuiltinSkillManifestLoader { return value.asText().trim(); } - public record ManifestItem(String slug, String version, String url) { + public record ManifestItem(String slug, String version, String url, String sha256) { } } diff --git a/server/skillhub-app/src/main/resources/builtin-skills/manifest.json b/server/skillhub-app/src/main/resources/builtin-skills/manifest.json index a9fc8cac..24ea0a81 100644 --- a/server/skillhub-app/src/main/resources/builtin-skills/manifest.json +++ b/server/skillhub-app/src/main/resources/builtin-skills/manifest.json @@ -3,12 +3,14 @@ { "slug": "skillhub-hello", "version": "1.0.0", - "url": "https://bjcdn.openstorage.cn/aicontest/2026-06-11/f8a59af3-30d4-4031-80f6-ebff74b05195.zip" + "url": "https://bjcdn.openstorage.cn/aicontest/2026-06-11/f8a59af3-30d4-4031-80f6-ebff74b05195.zip", + "sha256": "acb591ed0891e735c364b955f5b94b2b9ce567c1d9e347312cebfbfde2d93f57" }, { "slug": "agentguard", "version": "1.1", - "url": "https://bjcdn.openstorage.cn/aicontest/2026-06-12/9d063bc7-223a-4762-adeb-305c268aa29e.zip" + "url": "https://bjcdn.openstorage.cn/aicontest/2026-06-12/9d063bc7-223a-4762-adeb-305c268aa29e.zip", + "sha256": "77b16d62c1adaf8c1d5ca06b9bff26005bc77da345e6036cc5f1f91bdee1a149" } ] } diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializerTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializerTest.java index bc5602a0..e9e6dda5 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializerTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillInitializerTest.java @@ -44,9 +44,9 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Async; import org.springframework.test.util.ReflectionTestUtils; +import java.lang.reflect.Method; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.lang.reflect.Method; import java.security.MessageDigest; import java.util.HexFormat; import java.util.List; @@ -58,10 +58,14 @@ class BuiltinSkillInitializerTest { private static final String GLOBAL = "global"; private static final String PUBLISHER = "builtin-skill-publisher"; + private static final byte[] PACKAGE_BYTES = "zip".getBytes(StandardCharsets.UTF_8); + private static final String PACKAGE_SHA256 = + "4a70fe9aa6436e02c2dea340fbd1e352e4ef2d8ce6ca52ad25d4b95471fc8bf2"; private static final ManifestItem ITEM = new ManifestItem( "skillhub-hello", "1.0.0", - "https://bjcdn.openstorage.cn/skills/skillhub-hello.zip" + "https://bjcdn.openstorage.cn/skills/skillhub-hello.zip", + PACKAGE_SHA256 ); @Mock private BuiltinSkillManifestLoader manifestLoader; @@ -215,7 +219,8 @@ class BuiltinSkillInitializerTest { ManifestItem malformed = new ManifestItem( "skillhub-hello", "1.0.0", - "https://bjcdn.openstorage.cn/skills/%zz.zip" + "https://bjcdn.openstorage.cn/skills/%zz.zip", + PACKAGE_SHA256 ); givenManifestAndSystemPublisher(List.of(malformed)); when(skillRepository.findByNamespaceIdAndSlug(1L, "skillhub-hello")).thenReturn(List.of()); @@ -227,6 +232,25 @@ class BuiltinSkillInitializerTest { assertThat(output).doesNotContain("Failed to synchronize built-in skill slug=skillhub-hello"); } + @Test + void skipsPackageWithMismatchedSha256BeforeExtraction() throws Exception { + ManifestItem mismatched = new ManifestItem( + "skillhub-hello", + "1.0.0", + ITEM.url(), + "0000000000000000000000000000000000000000000000000000000000000000" + ); + givenManifestAndSystemPublisher(List.of(mismatched)); + when(skillRepository.findByNamespaceIdAndSlug(1L, "skillhub-hello")).thenReturn(List.of()); + when(downloader.download(URI.create(mismatched.url()))).thenReturn(Optional.of(PACKAGE_BYTES)); + + runInitializer(); + + verify(downloader).download(URI.create(mismatched.url())); + verify(extractor, never()).extract(any()); + verify(skillPublishService, never()).publishFromEntries(any(), any(), any(), any(), any(), anyBoolean()); + } + @Test void skipsWhenManifestSlugDoesNotMatchPackageMetadata() throws Exception { givenExtractedPackage(packageEntries("other-skill", "1.0.0", "same")); @@ -340,14 +364,14 @@ class BuiltinSkillInitializerTest { } private void givenExtractedPackage(List entries) throws Exception { - byte[] bytes = "zip".getBytes(StandardCharsets.UTF_8); when(namespaceRepository.findBySlug(GLOBAL)).thenReturn(Optional.of(globalNamespace)); when(manifestLoader.load()).thenReturn(List.of(ITEM)); lenient().when(userAccountRepository.findById(PUBLISHER)).thenReturn(Optional.of(systemPublisher())); lenient().when(namespaceMemberRepository.findByNamespaceIdAndUserId(1L, PUBLISHER)) .thenReturn(Optional.of(new NamespaceMember(1L, PUBLISHER, NamespaceRole.OWNER))); - when(downloader.download(URI.create(ITEM.url()))).thenReturn(Optional.of(bytes)); - when(extractor.extract(bytes)).thenReturn(new SkillPackageArchiveExtractor.ExtractionResult(entries, List.of())); + when(downloader.download(URI.create(ITEM.url()))).thenReturn(Optional.of(PACKAGE_BYTES)); + when(extractor.extract(PACKAGE_BYTES)) + .thenReturn(new SkillPackageArchiveExtractor.ExtractionResult(entries, List.of())); } private void givenManifestAndSystemPublisher() { diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoaderTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoaderTest.java index 66bc128f..71229f07 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoaderTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/bootstrap/BuiltinSkillManifestLoaderTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.ResourceLoader; import java.nio.charset.StandardCharsets; @@ -12,22 +13,53 @@ import java.util.List; class BuiltinSkillManifestLoaderTest { + private static final String SHA256_A = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + private static final String SHA256_B = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + @Test void loadsManifestItemsInOrder() { BuiltinSkillManifestLoader loader = loaderWith(""" { "skills": [ - {"slug": "skillhub-hello", "version": "1.0.0", "url": "https://bjcdn.openstorage.cn/skillhub-hello.zip"}, - {"slug": "skillhub-hello", "version": "1.1.0", "url": "https://cdn.bjcdn.openstorage.cn/skillhub-hello.zip"} + { + "slug": "skillhub-hello", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/skillhub-hello.zip", + "sha256": "%s" + }, + { + "slug": "skillhub-hello", + "version": "1.1.0", + "url": "https://cdn.bjcdn.openstorage.cn/skillhub-hello.zip", + "sha256": "%s" + } ] } - """); + """.formatted(SHA256_A, SHA256_B)); List items = loader.load(); assertThat(items) .extracting(BuiltinSkillManifestLoader.ManifestItem::version) .containsExactly("1.0.0", "1.1.0"); + assertThat(items) + .extracting(BuiltinSkillManifestLoader.ManifestItem::sha256) + .containsExactly(SHA256_A, SHA256_B); + } + + @Test + void loadsBundledManifestFromClasspath() { + BuiltinSkillManifestLoader loader = new BuiltinSkillManifestLoader( + new ObjectMapper(), + new DefaultResourceLoader() + ); + + List items = loader.load(); + + assertThat(items).isNotEmpty(); + assertThat(items).allSatisfy(item -> + assertThat(item.sha256()).matches("[0-9a-f]{64}") + ); } @Test @@ -65,20 +97,61 @@ class BuiltinSkillManifestLoaderTest { } @Test - void skipsItemsWithMissingHumanFieldsAndDuplicateSlugVersion() { + void skipsItemsWithMissingRequiredFieldsAndDuplicateSlugVersion() { BuiltinSkillManifestLoader loader = loaderWith(""" { "skills": [ - {"slug": "skillhub-hello", "version": "1.0.0", "url": "https://bjcdn.openstorage.cn/first.zip"}, - {"slug": "skillhub-hello", "version": "1.0.0", "url": "https://bjcdn.openstorage.cn/second.zip"}, - {"slug": "InvalidUppercase", "version": "1.0.0", "url": "https://bjcdn.openstorage.cn/invalid.zip"}, - {"slug": "", "version": "1.0.0", "url": "https://bjcdn.openstorage.cn/blank.zip"}, - {"slug": "missing-version", "url": "https://bjcdn.openstorage.cn/missing-version.zip"}, - {"slug": "missing-url", "version": "1.0.0"}, - {"slug": "valid-after-invalid", "version": "1.0.0", "url": "https://bjcdn.openstorage.cn/valid.zip"} + { + "slug": "skillhub-hello", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/first.zip", + "sha256": "%s" + }, + { + "slug": "skillhub-hello", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/second.zip", + "sha256": "%s" + }, + { + "slug": "InvalidUppercase", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/invalid.zip", + "sha256": "%s" + }, + { + "slug": "", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/blank.zip", + "sha256": "%s" + }, + { + "slug": "missing-version", + "url": "https://bjcdn.openstorage.cn/missing-version.zip", + "sha256": "%s" + }, + { + "slug": "missing-url", + "version": "1.0.0", + "sha256": "%s" + }, + { + "slug": "valid-after-invalid", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/valid.zip", + "sha256": "%s" + } ] } - """); + """.formatted( + SHA256_A, + SHA256_B, + SHA256_A, + SHA256_A, + SHA256_A, + SHA256_A, + SHA256_B + )); List items = loader.load(); @@ -90,6 +163,45 @@ class BuiltinSkillManifestLoaderTest { ); } + @Test + void skipsItemsWithMissingOrInvalidSha256() { + BuiltinSkillManifestLoader loader = loaderWith(""" + { + "skills": [ + { + "slug": "missing-sha", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/missing.zip" + }, + { + "slug": "short-sha", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/short.zip", + "sha256": "abc123" + }, + { + "slug": "uppercase-sha", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/uppercase.zip", + "sha256": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + }, + { + "slug": "valid-sha", + "version": "1.0.0", + "url": "https://bjcdn.openstorage.cn/valid.zip", + "sha256": "%s" + } + ] + } + """.formatted(SHA256_A)); + + List items = loader.load(); + + assertThat(items) + .extracting(BuiltinSkillManifestLoader.ManifestItem::slug) + .containsExactly("valid-sha"); + } + @Test void capsManifestEntriesAtOneHundredRawEntries() { StringBuilder json = new StringBuilder("{\"skills\":["); @@ -98,12 +210,13 @@ class BuiltinSkillManifestLoaderTest { json.append(','); } if (i == 0) { - json.append("{\"slug\":\"\",\"version\":\"1.0.0\",\"url\":\"https://bjcdn.openstorage.cn/blank.zip\"}"); + json.append("{\"slug\":\"\",\"version\":\"1.0.0\",\"url\":\"https://bjcdn.openstorage.cn/blank.zip\",") + .append("\"sha256\":\"").append(SHA256_A).append("\"}"); } else { json.append("{\"slug\":\"skill-").append(i) .append("\",\"version\":\"1.0.0\",\"url\":\"https://bjcdn.openstorage.cn/skill-") .append(i) - .append(".zip\"}"); + .append(".zip\",\"sha256\":\"").append(SHA256_A).append("\"}"); } } json.append("]}");