From 8a150e34d37e265fdfb57172aa3f90514e74ab39 Mon Sep 17 00:00:00 2001 From: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:31:29 +0800 Subject: [PATCH 1/4] feat(storage): add S3-compatible deployment options Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> --- .github/workflows/pr-helm-chart.yml | 1 + .github/workflows/pr-scripts.yml | 7 + .../skillhub/tests/configuration-contracts.sh | 15 ++ compose.storage.rustfs.yml | 49 ++++ compose.storage.silo.yml | 33 +++ deploy/runtime-mirror-images.txt | 3 + deploy/storage/application-policy.json | 15 ++ deploy/storage/helm-values.example.yaml | 17 ++ deploy/storage/rustfs.env.example | 24 ++ deploy/storage/silo.env.example | 24 ++ docs/09-deployment.md | 2 + docs/github-sync-operations.md | 2 + docs/object-storage.md | 226 ++++++++++++++++++ docs/skillhub/guide/kubernetes.md | 2 + .../.openspec.yaml | 2 + .../design.md | 183 ++++++++++++++ .../proposal.md | 27 +++ .../specs/object-storage-deployment/spec.md | 49 ++++ .../specs/s3-compatible-storage/spec.md | 57 +++++ .../tasks.md | 42 ++++ scripts/storage-compose.sh | 45 ++++ scripts/storage-diagnostics.sh | 33 +++ scripts/tests/mirror-storage-images-test.sh | 26 ++ scripts/tests/storage-compose-render-test.sh | 52 ++++ scripts/tests/storage-compose-test.sh | 33 +++ scripts/tests/storage-diagnostics-test.sh | 39 +++ scripts/tests/validate-release-config-test.sh | 5 + scripts/validate-release-config.sh | 1 + .../skillhub/storage/S3StorageService.java | 37 ++- .../storage/S3StorageContractTest.java | 98 ++++++++ .../storage/S3StorageProtocolTest.java | 65 +++++ .../storage/S3StorageServiceTest.java | 85 ++++++- .../storage/StorageContractProbe.java | 147 ++++++++++++ .../storage/StorageContractProbeTest.java | 80 +++++++ 34 files changed, 1513 insertions(+), 13 deletions(-) create mode 100644 compose.storage.rustfs.yml create mode 100644 compose.storage.silo.yml create mode 100644 deploy/storage/application-policy.json create mode 100644 deploy/storage/helm-values.example.yaml create mode 100644 deploy/storage/rustfs.env.example create mode 100644 deploy/storage/silo.env.example create mode 100644 docs/object-storage.md create mode 100644 openspec/changes/add-s3-compatible-storage-deployments/.openspec.yaml create mode 100644 openspec/changes/add-s3-compatible-storage-deployments/design.md create mode 100644 openspec/changes/add-s3-compatible-storage-deployments/proposal.md create mode 100644 openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md create mode 100644 openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md create mode 100644 openspec/changes/add-s3-compatible-storage-deployments/tasks.md create mode 100755 scripts/storage-compose.sh create mode 100755 scripts/storage-diagnostics.sh create mode 100755 scripts/tests/mirror-storage-images-test.sh create mode 100755 scripts/tests/storage-compose-render-test.sh create mode 100755 scripts/tests/storage-compose-test.sh create mode 100755 scripts/tests/storage-diagnostics-test.sh create mode 100644 server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageContractTest.java create mode 100644 server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageProtocolTest.java create mode 100644 server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbe.java create mode 100644 server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbeTest.java diff --git a/.github/workflows/pr-helm-chart.yml b/.github/workflows/pr-helm-chart.yml index 20359be3f..b9d919741 100644 --- a/.github/workflows/pr-helm-chart.yml +++ b/.github/workflows/pr-helm-chart.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - charts/skillhub/** + - deploy/storage/helm-values.example.yaml - .github/workflows/pr-helm-chart.yml - .github/workflows/publish-chart.yml types: diff --git a/.github/workflows/pr-scripts.yml b/.github/workflows/pr-scripts.yml index c47bf96f3..9fbd44ac7 100644 --- a/.github/workflows/pr-scripts.yml +++ b/.github/workflows/pr-scripts.yml @@ -9,6 +9,9 @@ on: - '.env.release.example' - '.env.release.draft' - 'compose.release.yml' + - 'compose.storage.*.yml' + - 'deploy/storage/**' + - 'deploy/runtime-mirror-images.txt' - 'deploy/k8s/base/configmap.yaml' - 'deploy/k8s/base/scanner-deployment.yaml' - 'charts/skillhub/values.yaml' @@ -51,6 +54,10 @@ jobs: - run: bash scripts/tests/publish-cli-test.sh - run: bash scripts/tests/runtime-secret-test.sh - run: bash scripts/tests/validate-release-config-test.sh + - run: bash scripts/tests/storage-compose-test.sh + - run: bash scripts/tests/storage-compose-render-test.sh + - run: bash scripts/tests/storage-diagnostics-test.sh + - run: bash scripts/tests/mirror-storage-images-test.sh - run: bash scripts/tests/nginx-forwarded-proto-test.sh - run: bash scripts/tests/smoke-test-admin-mode-test.sh - run: bash scripts/tests/web-base-path-routing-test.sh diff --git a/charts/skillhub/tests/configuration-contracts.sh b/charts/skillhub/tests/configuration-contracts.sh index d9dd8b140..311989971 100755 --- a/charts/skillhub/tests/configuration-contracts.sh +++ b/charts/skillhub/tests/configuration-contracts.sh @@ -468,4 +468,19 @@ assert_rejected public-base-url-query --set-string publicBaseUrl=https://skills. assert_rejected public-base-url-fragment --set-string publicBaseUrl=https://skills.example.com#frag assert_rejected public-base-url-no-host --set-string publicBaseUrl=https:// +# Example values must reuse the external application Secret and existing S3 config keys. +helm template storage-contract "$CHART_DIR" -f "$TEST_VALUES" \ + -f "$CHART_DIR/../../deploy/storage/helm-values.example.yaml" \ + >"$TMP_DIR/storage-example.yaml" +grep -Fq 'skillhub-storage-provider: "s3"' "$TMP_DIR/storage-example.yaml" \ + || fail "Storage example must select the existing S3 provider" +grep -Fq 's3-force-path-style: "true"' "$TMP_DIR/storage-example.yaml" \ + || fail "Storage example must use path style" +grep -Fq 'name: skillhub-app-secrets' "$TMP_DIR/storage-example.yaml" \ + || fail "Storage example must reference the external application Secret" +grep -Fq 'key: skillhub-storage-s3-access-key' "$TMP_DIR/storage-example.yaml" \ + || fail "Storage access key Secret mapping changed" +grep -Fq 'key: skillhub-storage-s3-secret-key' "$TMP_DIR/storage-example.yaml" \ + || fail "Storage secret key Secret mapping changed" + echo "Helm configuration contract tests passed" diff --git a/compose.storage.rustfs.yml b/compose.storage.rustfs.yml new file mode 100644 index 000000000..2b3713602 --- /dev/null +++ b/compose.storage.rustfs.yml @@ -0,0 +1,49 @@ +# Optional single-node storage. Select exactly one overlay with scripts/storage-compose.sh. +services: + object-storage-permissions: + image: ${RUSTFS_IMAGE:-docker.io/rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b} + user: "0:0" + entrypoint: ["chown", "10001:10001", "/data"] + restart: "no" + network_mode: none + read_only: true + cap_drop: [ALL] + cap_add: [CHOWN] + volumes: + - rustfs_data:/data + + object-storage: + image: ${RUSTFS_IMAGE:-docker.io/rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b} + restart: unless-stopped + depends_on: + object-storage-permissions: + condition: service_completed_successfully + environment: + RUSTFS_ACCESS_KEY: ${SKILLHUB_OBJECT_STORAGE_ROOT_USER:?Set storage administrator user} + RUSTFS_SECRET_KEY: ${SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD:?Set storage administrator password} + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ADDRESS: ":9001" + ports: + - "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_API_PORT:-9000}:9000" + - "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_CONSOLE_PORT:-9001}:9001" + volumes: + - rustfs_data:/data + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "--max-time", "3", "http://127.0.0.1:9000/health"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + + server: + environment: + SKILLHUB_STORAGE_PROVIDER: s3 + SKILLHUB_STORAGE_S3_ENDPOINT: http://object-storage:9000 + SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE: "true" + SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: "false" + depends_on: + object-storage: + condition: service_healthy + +volumes: + rustfs_data: diff --git a/compose.storage.silo.yml b/compose.storage.silo.yml new file mode 100644 index 000000000..535a79387 --- /dev/null +++ b/compose.storage.silo.yml @@ -0,0 +1,33 @@ +# Optional single-node storage. Select exactly one overlay with scripts/storage-compose.sh. +services: + object-storage: + image: ${SILO_IMAGE:-docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7} + restart: unless-stopped + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${SKILLHUB_OBJECT_STORAGE_ROOT_USER:?Set storage administrator user} + MINIO_ROOT_PASSWORD: ${SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD:?Set storage administrator password} + ports: + - "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_API_PORT:-9000}:9000" + - "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_CONSOLE_PORT:-9001}:9001" + volumes: + - silo_data:/data + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "--max-time", "3", "http://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + + server: + environment: + SKILLHUB_STORAGE_PROVIDER: s3 + SKILLHUB_STORAGE_S3_ENDPOINT: http://object-storage:9000 + SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE: "true" + SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: "false" + depends_on: + object-storage: + condition: service_healthy + +volumes: + silo_data: diff --git a/deploy/runtime-mirror-images.txt b/deploy/runtime-mirror-images.txt index e83668f37..b7fde8c2e 100644 --- a/deploy/runtime-mirror-images.txt +++ b/deploy/runtime-mirror-images.txt @@ -5,3 +5,6 @@ docker.io/pgsty/silo:RELEASE.2026-09-16T00-00-00Z minio:latest prom/prometheus:latest prometheus:latest grafana/grafana:latest grafana:latest nginx:alpine nginx:alpine +# Optional single-node storage overlays; do not reuse minio:latest for RustFS. +docker.io/rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b rustfs:1.0.3-9ca9559533b5 +docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7 silo:20261018-2636fc4155d9 diff --git a/deploy/storage/application-policy.json b/deploy/storage/application-policy.json new file mode 100644 index 000000000..c5fa82df4 --- /dev/null +++ b/deploy/storage/application-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::skillhub"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::skillhub/*"] + } + ] +} diff --git a/deploy/storage/helm-values.example.yaml b/deploy/storage/helm-values.example.yaml new file mode 100644 index 000000000..94af67ca8 --- /dev/null +++ b/deploy/storage/helm-values.example.yaml @@ -0,0 +1,17 @@ +# Merge into an existing valid SkillHub values file; this is not a full installation. +# Storage is installed independently. The pre-created Secret must also contain +# all existing application-required keys, not only these two S3 keys: +# skillhub-storage-s3-access-key / skillhub-storage-s3-secret-key. +existingSecret: skillhub-app-secrets +s3: + enabled: true + endpoint: http://object-storage.storage.svc.cluster.local:9000 + publicEndpoint: https://storage.example.com + bucket: skillhub + region: us-east-1 + forcePathStyle: true + disableChunkedEncoding: true + autoCreateBucket: false + presignExpiry: PT10M + accessKey: "" + secretKey: "" diff --git a/deploy/storage/rustfs.env.example b/deploy/storage/rustfs.env.example new file mode 100644 index 000000000..69ae976a4 --- /dev/null +++ b/deploy/storage/rustfs.env.example @@ -0,0 +1,24 @@ +# Merge these values into a protected .env.release based on .env.release.example. +# Do not source an untrusted env file. Replace placeholders before provisioning. +# Select with: scripts/storage-compose.sh --storage rustfs -- COMPOSE_COMMAND +SKILLHUB_STORAGE_PROVIDER=s3 +SKILLHUB_STORAGE_S3_ENDPOINT=http://object-storage:9000 +# Must be reachable by download clients; loopback is only for local validation. +SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000 +SKILLHUB_STORAGE_S3_BUCKET=skillhub +SKILLHUB_STORAGE_S3_REGION=us-east-1 +SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=true +SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true +SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false +SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-with-business-access-key +SKILLHUB_STORAGE_S3_SECRET_KEY=replace-with-business-secret +# Storage administrator only; never pass these credentials to the SkillHub server. +SKILLHUB_OBJECT_STORAGE_ROOT_USER=replace-with-storage-admin +SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD=replace-with-storage-admin-secret +OBJECT_STORAGE_BIND_ADDRESS=127.0.0.1 +OBJECT_STORAGE_API_PORT=9000 +OBJECT_STORAGE_CONSOLE_PORT=9001 +# Enable only with the existing GitHub-sync service and credential configuration. +SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=object-storage:9000 +# Set a positive byte budget appropriate for the deployment before enabling sync. +SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824 diff --git a/deploy/storage/silo.env.example b/deploy/storage/silo.env.example new file mode 100644 index 000000000..4921efbc2 --- /dev/null +++ b/deploy/storage/silo.env.example @@ -0,0 +1,24 @@ +# Merge these values into a protected .env.release based on .env.release.example. +# Do not source an untrusted env file. Replace placeholders before provisioning. +# Select with: scripts/storage-compose.sh --storage silo -- COMPOSE_COMMAND +SKILLHUB_STORAGE_PROVIDER=s3 +SKILLHUB_STORAGE_S3_ENDPOINT=http://object-storage:9000 +# Must be reachable by download clients; loopback is only for local validation. +SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000 +SKILLHUB_STORAGE_S3_BUCKET=skillhub +SKILLHUB_STORAGE_S3_REGION=us-east-1 +SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=true +SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true +SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false +SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-with-business-access-key +SKILLHUB_STORAGE_S3_SECRET_KEY=replace-with-business-secret +# Storage administrator only; never pass these credentials to the SkillHub server. +SKILLHUB_OBJECT_STORAGE_ROOT_USER=replace-with-storage-admin +SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD=replace-with-storage-admin-secret +OBJECT_STORAGE_BIND_ADDRESS=127.0.0.1 +OBJECT_STORAGE_API_PORT=9000 +OBJECT_STORAGE_CONSOLE_PORT=9001 +# Enable only with the existing GitHub-sync service and credential configuration. +SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=object-storage:9000 +# Set a positive byte budget appropriate for the deployment before enabling sync. +SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824 diff --git a/docs/09-deployment.md b/docs/09-deployment.md index ae2958002..dffea4ac7 100644 --- a/docs/09-deployment.md +++ b/docs/09-deployment.md @@ -1,5 +1,7 @@ # skillhub 部署架构与运维 +对象存储可选部署、凭证和迁移说明见[对象存储手册](object-storage.md)。 + ## 1 运行模型 当前仓库只保留两种运行方式： diff --git a/docs/github-sync-operations.md b/docs/github-sync-operations.md index d40241eec..1626d0088 100644 --- a/docs/github-sync-operations.md +++ b/docs/github-sync-operations.md @@ -1,5 +1,7 @@ # GitHub 公开 Skill 同步：管理员运行手册 +对象存储可选部署、凭证和迁移说明见[对象存储手册](object-storage.md)。 + 本功能只采集 GitHub 公开仓库。SkillHub Java 保存配置、来源、任务、审核和发布状态；独立 Go 服务轮询 Java 的内部接口执行发现与采集。Go 停止后，Java 中已保存的任务和检查点仍在；重新启动 Go 后由租约机制继续处理。功能仍在实施验收阶段，生产启用前须完成 [OpenSpec 验收任务](../openspec/changes/add-github-skill-sync/tasks.md) 中未勾选的部署检查。 ## 启用前配置 diff --git a/docs/object-storage.md b/docs/object-storage.md new file mode 100644 index 000000000..516a1a7ed --- /dev/null +++ b/docs/object-storage.md @@ -0,0 +1,226 @@ +# 对象存储部署与迁移 + +SkillHub 通过同一个 S3 接口连接 RustFS、PGSTY Silo 或外部 S3 服务，不需要按存储品牌修改业务代码。`local` 模式保留。本文新增的单机叠加配置不是高可用存储集群；正式支持范围以本文末尾的验证矩阵为准。 + +## 1. 选择部署方式 + +| 方式 | 使用配置 | 数据归属 | +|---|---|---| +| RustFS | `compose.release.yml` + `compose.storage.rustfs.yml` | 独立 `rustfs_data` 卷 | +| Silo | `compose.release.yml` + `compose.storage.silo.yml` | 独立 `silo_data` 卷 | +| 外部 S3 | 仅 `compose.release.yml` | 外部服务的桶 | +| local | 仅 `compose.release.yml`，显式 `SKILLHUB_STORAGE_PROVIDER=local` | 现有应用存储卷 | + +一次只选择一个本地存储。使用 `scripts/storage-compose.sh`，它会拒绝重复选择和额外 `-f` 文件；不要自行同时合并两个存储文件。所有命令在仓库根目录执行，要求 Bash、Docker Compose v2；诊断另外要求 JDK 21+、Maven Wrapper 和 GNU timeout。 + +## 2. 准备受保护配置 + +从 `.env.release.example` 准备 `.env.release`，再把 `deploy/storage/rustfs.env.example` 或 `deploy/storage/silo.env.example` 中的存储配置合入该文件，并替换所有占位值。不要直接用示例部署，也不要把实际配置加入 Git。 + +```sh +# 已有 .env.release 时不要重复复制覆盖 +cp .env.release.example .env.release +chmod 600 .env.release +# 编辑完整应用配置，再合入选定产品的 env 示例 +``` + +配置分为两组： + +- `SKILLHUB_OBJECT_STORAGE_ROOT_USER` / `SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD` 仅供对象存储管理员初始化；叠加文件不会将它们传给 SkillHub server。 +- `SKILLHUB_STORAGE_S3_ACCESS_KEY` / `SKILLHUB_STORAGE_S3_SECRET_KEY` 是单独的应用账号，限定在应用桶范围。不要复用存储 root 账号。 + +`SKILLHUB_STORAGE_S3_ENDPOINT=http://object-storage:9000` 用于后端对象操作及 Worker 的预签名 PUT；`SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT` 用于用户下载签名。后者必须是下载方真实可达的地址；示例中的 `127.0.0.1` 仅适用于当前机器本地验证。生产环境应准备专用存储域名、TLS 与保持签名主机/路径的反向代理，不能签名后替换 URL。 + +新叠加配置明确启用 S3、路径寻址、关闭自动建桶；其值覆盖基础 Compose 的对应配置。其他应用字段仍来自 `.env.release`。更改配置后重建对应服务才能生效，不由后台系统配置页面管理。 + +两个存储 API/Console 默认仅绑定 `127.0.0.1:9000/9001`。端口冲突时调整 `OBJECT_STORAGE_API_PORT` / `OBJECT_STORAGE_CONSOLE_PORT`，同时调整下载公开端点；容器内部端口和 endpoint 仍是 9000。不要为了远程浏览 Console 将管理接口无保护地暴露到公网。 + +## 3. 分步初始化，避免应用先于桶启动 + +下面用 RustFS，Silo 把每条命令的 `--storage rustfs` 改为 `--storage silo`。固定 `--project-name`，后续更新、状态和停机也使用相同参数。 + +```sh +scripts/storage-compose.sh --storage rustfs --env-file .env.release \ + --project-name skillhub -- up -d object-storage +scripts/storage-compose.sh --storage rustfs --env-file .env.release \ + --project-name skillhub -- ps object-storage +``` + +上述命令会创建选定产品的容器和数据卷，但不会启动整个应用栈。RustFS 会先用同一固定镜像的一次性权限容器，将数据卷根目录交给其运行身份 `10001:10001`；它无网络、不持有密钥，完成后退出，不对已有对象递归修改权限。 + +存储健康后，由存储管理员完成以下初始化。使用对应版本的官方 Console 或管理客户端，不使用 SkillHub 的业务进程代为创建账号： + +1. 在本地 Console `http://127.0.0.1:9001` 登录存储管理员账号。 +2. 创建私有桶 `skillhub`；已存在时核对，不清空、不重建桶。 +3. 创建独立业务身份并授予 `deploy/storage/application-policy.json` 中的桶内权限。若修改桶名，同步修改 policy 的两个 Resource。 +4. 使用业务身份的凭证填写应用的 S3 access/secret。确认匿名对象读取被拒绝，业务身份不能操作其他桶或管理账号。 +5. 重复初始化只做存在性和权限核对，不自动轮换密钥、删除用户或变更已有桶的数据。 + +不同产品的管理 API 与 Console 功能不完全一致；不要把 MinIO 管理命令未经验证地套到 RustFS。政策中包含读写删除对象、列举桶和读取桶区域；不授予创建桶、管理用户或公开桶权限。 + +### 可选：用已验证管理客户端初始化 + +本次使用 [PGSTY mcli RELEASE.2026-09-16T00-00-00Z](https://github.com/pgsty/mc/releases/tag/RELEASE.2026-09-16T00-00-00Z) 验证了 RustFS 1.0.3 和 Silo RELEASE.2026-10-18T00-00-00Z 的账号与策略管理。客户端安装和校验按其官方发布说明进行；`mcli` 是部署管理员工具，不是 SkillHub 的运行依赖。 + +先将 `.env.release` 中的 root、业务账号和桶参数安全注入当前管理会话。以下命令仅用于**首次创建账号**；已有账号先查询并核对，不能重复 `user add` 来重置凭证。 + +```sh +umask 077 +export MC_CONFIG_DIR="$PWD/.dev/object-storage-admin" +mkdir -p "$MC_CONFIG_DIR" +# 这里是管理员从宿主机访问的地址，端口需与本部署一致 +mcli alias set storage http://127.0.0.1:9000 \ + "$SKILLHUB_OBJECT_STORAGE_ROOT_USER" "$SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD" +mcli mb --ignore-existing "storage/$SKILLHUB_STORAGE_S3_BUCKET" +# 首次创建之前先查询；存在时跳过 user add，核对已有身份和权限 +mcli admin user info storage "$SKILLHUB_STORAGE_S3_ACCESS_KEY" +# 仅确认不存在且信息查询不是网络/权限错误时执行 +mcli admin user add storage "$SKILLHUB_STORAGE_S3_ACCESS_KEY" "$SKILLHUB_STORAGE_S3_SECRET_KEY" +# 先核对 policy 的桶名和权限；同名已有策略需人工核对，不能盲目覆盖 +mcli admin policy create storage skillhub-app deploy/storage/application-policy.json +mcli admin policy attach storage skillhub-app --user "$SKILLHUB_STORAGE_S3_ACCESS_KEY" +``` + +重复部署时，使用 `mb --ignore-existing`、`admin user info` 和 `admin policy info` 核对状态，不删除桶、不重建账号或轮换凭证。管理客户端的本地配置含管理员凭证，应保持私有；它不挂载到应用容器。 + +## 4. 诊断和启动应用 + +先将上一步的业务 S3 参数注入当前 shell。诊断从宿主机执行时，endpoint 必须改为宿主机可达的 API 地址，而非仅容器网络可见的 `object-storage`。不要把含密钥的整行命令粘贴到共享日志中。 + +以下仅为本机验证示例；生产诊断的公开端点应使用真实下载域名，不能把 loopback 测试当成外网连通性证明。用子 shell 限定临时地址，避免它们覆盖后续 Compose 读取的发布配置。 + +```sh +# 其他 S3 参数和业务凭证应通过当前会话的安全配置方式注入 +( +export SKILLHUB_STORAGE_S3_ENDPOINT=http://127.0.0.1:9000 +export SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000 +export SKILLHUB_STORAGE_S3_BUCKET=skillhub +export SKILLHUB_STORAGE_S3_REGION=us-east-1 +export SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=true +export SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true +scripts/storage-diagnostics.sh +) +``` + +该命令使用实际 `ObjectStorageService`，写入随机 `skillhub-diagnostics/<随机值>/` 对象，检查字节、元数据、GET/PUT 签名、匿名拒绝、签名过期/篡改和删除。完成后仅删除本次创建的精确对象；失败返回非零，原始日志默认保存在 Git 忽略的 `.dev/storage-diagnostics/`，权限受 umask 077 保护。日志可能包含签名 URL，分享前必须脱敏。超时或进程被强制终止时可能有残留，使用私有日志中的前缀核查并精确清理；不要清空整个桶。 + +交付验收时另运行 `scripts/storage-diagnostics.sh --full`，额外创建 1001 个小对象，检查真实翻页、统计上限与跨批次删除。满批删除如遇后端短暂逐项失败，适配层必须明确报错；验收会记录失败并最多尝试 3 次调用方恢复，最终独立确认对象已经不存在，不把 HTTP 200 当成全部删除成功；该模式最多运行 420 秒。普通诊断最多 180 秒，外层强制终止后可能需要按日志核查残留。 + +S3 PUT 会在应用私有临时文件中暂存请求体，以支持文件流签名和重试；部署时应计入并发上传的临时磁盘预算，不能只预算对象存储的数据卷。临时文件在操作成功或失败后清理，不将整个技能包缓存到 JVM 堆中。 + +诊断不是应用 liveness，不要把这个有写入动作的命令接到公开健康检查。 + +```sh +make validate-release-config +scripts/storage-compose.sh --storage rustfs --env-file .env.release \ + --project-name skillhub -- up -d +``` + +`make validate-release-config` 校验应用发布配置，不证明存储可用；同样，存储健康不证明发布、审核和下载业务已通过。启动后还需按本次变更的业务验收矩阵验证。 + +### GitHub 同步 + +沿用[同步运行手册](github-sync-operations.md)的服务密钥和其他配置，另外设置： + +```dotenv +SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=object-storage:9000 +SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824 +``` + +允许列表按完整主机及端口匹配，不能只填 `object-storage`。容量按部署预算调整为正数。Worker 必须可达内部 endpoint，不能把宿主机 `localhost` 作为容器上传地址。 + +```sh +scripts/storage-compose.sh --storage rustfs --env-file .env.release \ + --project-name skillhub -- --profile github-sync up -d +``` + +## 5. Helm / Kubernetes + +存储独立部署和维护，SkillHub Chart 不安装 RustFS/Silo 子 Chart，不在应用卸载时删除对象存储。 + +`deploy/storage/helm-values.example.yaml` 只包含新增接入所需字段，需要合并到已有有效 values 中。使用已有 `existingSecret` 时，Chart 不创建应用 Secret；必须保留其他应用必需键，并加入： + +- `skillhub-storage-s3-access-key` +- `skillhub-storage-s3-secret-key` + +它们只存业务凭证，不存对象存储 root 密钥。启用 `s3.enabled` 后，`s3.*` 决定地址和协议参数，凭证从已有 Secret 读取；`s3.accessKey` / `s3.secretKey` 留空，不把明文写入 values。变更外部 Secret 后按部署策略滚动应用。 + +参见 [Kubernetes 指南](skillhub/guide/kubernetes.md)。模板渲染通过与真实 K8s 连通性、对象操作、升级验收分别记录。 + +## 6. 更新与停机 + +先备份并确认目标版本的迁移要求，再更新固定镜像。通过 `RUSTFS_IMAGE` / `SILO_IMAGE` 覆盖时仍应指定经验证的不可变摘要；覆盖制品需要重新验收。 + +```sh +scripts/storage-compose.sh --storage rustfs --env-file .env.release \ + --project-name skillhub -- ps +# 停止并移除本项目容器，保留命名数据卷；不要增加 -v +scripts/storage-compose.sh --storage rustfs --env-file .env.release \ + --project-name skillhub -- down +``` + +`deploy/runtime-mirror-images.txt` 保留原有 `minio:latest` 别名。新产品采用独立版本标签；已有镜像目标会被同步脚本跳过，修改映射清单不等于更新仓库内容。实际同步属于单独的发布操作，验收应比较目标 manifest digest。 + +## 7. 从已有后端迁移 + +修改 endpoint/provider 不会迁移文件。RustFS、Silo 和 local 不得默认共用原始数据目录。 + +1. 记录现有配置、对象清单、元数据与加密方式，备份并验证恢复。目标使用独立桶/卷和业务账号。 +2. 逻辑复制对象，保留 key、Content-Type 及业务需要的元数据；记录迁移工具版本与选项。加密对象经授权解密读取后写入新服务，不能直接复制内部存储文件。 +3. 用对象数量、大小和独立内容哈希校验；不能把 multipart 或加密对象的 ETag 一律当 MD5。 +4. 进入维护窗口，暂停发布、头像变更/审核、Suite 写入、同步、后台删除/过期清理；排空任务。已签发 PUT 也必须等到过期或撤销旧端写权限，避免最终同步后仍写旧桶。 +5. 在冻结点完成最终同步，同时处理新增、覆盖和删除差异。保留旧源只读；已签发 GET 可在有效期内继续指向旧源，应保留读取或声明短暂中断窗口。 +6. 切换配置、重建受影响服务，检查下载原始字节、头像、Suite 和同步导入/清理，再恢复写入。 +7. 恢复写入前可以回退配置到未变的旧源；恢复写入后，必须重新停写并协调新后端的新增、覆盖、删除差异后再回切，防止丢失新文件或恢复已删除内容。 + +### 已验证的逻辑复制方式 + +本次用上述版本的 `mcli` 在独立测试桶演练 RustFS → Silo → RustFS，包含在线初始复制、冻结后的新增／覆盖／删除、在途 PUT 过期，以及切换后产生新数据再回退。`mirror` 会保存本次验证对象的 Content-Type 和自定义元数据；版本历史、对象锁、加密和存储策略仍需部署方分别评估。 + +下面的 `old`、`new` 是操作者预先配置的管理客户端别名，不是内置端点。先核对方向、桶名和私有权限，再执行；`--remove` 会删除目标中源端不存在的对象，只能在已批准的目标桶范围操作。不能把它指向尚在写入的旧源或其他业务桶。 + +```sh +# 初始复制；此阶段源仍可写，不作为最终一致状态 +mcli mirror --overwrite --max-workers 2 old/skillhub new/skillhub +# 按前述步骤暂停应用、Worker、定时清理等全部写入，并排空已签发 PUT +mcli mirror --dry-run --overwrite --remove --max-workers 2 old/skillhub new/skillhub +# 审核删除差异后，在冻结窗口执行最终同步 +mcli mirror --overwrite --remove --max-workers 2 old/skillhub new/skillhub +# 对应对象逐项核对原始字节和元数据；不要用 ETag 代替哈希 +mcli stat --json old/skillhub/OBJECT_KEY +mcli stat --json new/skillhub/OBJECT_KEY +mcli cat old/skillhub/OBJECT_KEY | sha256sum +mcli cat new/skillhub/OBJECT_KEY | sha256sum +``` + +示例的 `OBJECT_KEY` 必须替换为实际对象键；正式迁移应对完整清单逐项自动比对，不只抽一个文件。未使用 `--preserve`，避免同时复制桶策略和锁配置；目标桶及业务权限单独初始化并保持私有。对额外业务元数据先做代表性验证。 + +恢复写入后的回退采用同一停写和签名排空规则，重新核对方向后先 `new/skillhub → old/skillhub` 逻辑同步及比对，再回切应用，不能直接启动旧源。旧签名 URL 不保证跨产品切换后继续有效，应保留旧读端或明确中断窗口；新签名需在切换后重新申请。 + +旧源按部署方备份保留政策保存，确认恢复能力后再单独授权清理。失败时保留清单与源数据，不通过删除业务数据“修复”验收。 + +## 8. 制品和兼容性边界 + +| 产品 | 固定版本 | 多架构 manifest digest | 许可证 | +|---|---|---|---| +| RustFS | 1.0.3 | `sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b` | Apache-2.0 | +| PGSTY Silo | RELEASE.2026-10-18T00-00-00Z | `sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7` | AGPL-3.0-or-later | + +两者的官方 manifest 均包含 linux/amd64 和 linux/arm64；本机运行结果只证明实际执行的架构。新增 Silo 叠加文件选用上述版本；已有开发服务及 `minio:latest` 别名保持原版本，不自动升级已有实例。 + +本次在 linux/amd64 上使用同一应用构件完成两种产品的真实契约验收；不代表任意 S3 产品、任意版本、arm64 运行或高可用部署已获验证。 + +| 范围 | RustFS 1.0.3 | Silo 2026-10-18 | +|---|---|---| +| 文件流上传、字节和元数据、超过 1000 项分页和删除 | 通过；满批曾出现 `ServiceUnavailable` 逐项失败，必须由调用方处理 | 通过 | +| GET/PUT 签名、所需头、过期/篡改、匿名私有访问 | 通过 | 通过 | +| 发布审核下载、头像待审/批准/移除、Suite | 通过 | 通过 | +| Go Worker 内部端点 PUT、导入及暂存清理 | 通过 | 通过 | + +RustFS 的满批临时错误不是“HTTP 200 即全部成功”；适配层会明确抛错，诊断的有界恢复只证明操作可幂等恢复，不能证明后端从不产生逐项失败。业务调用者应沿用现有补偿，不把诊断脚本的三次恢复误认为生产适配器自动重试。 + +制品核验采用 Trivy 0.76.0：RustFS 镜像系统包和官方发行 SBOM（1157 个组件）均未检出 High/Critical；Silo 镜像尚有 `CVE-2026-39414` 的 legacy MinIO 模块匹配。对这一条告警，已将镜像二进制的 commit `0cd388c666f8211f552604ae73448a066e9c25ce` 与发行标签源码绑定，并核对 CSV 有界读取修复；实际普通 2 MiB 与 gzip 解压后 2 MiB 的无换行记录均被拒绝且服务存活，本地分析裁定该特定漏洞路径已修补（依据：[上游公告](https://github.com/minio/minio/security/advisories/GHSA-h749-fxx7-pwpg)、[镜像对应提交的 CSV 读取实现](https://github.com/pgsty/silo/blob/0cd388c666f8211f552604ae73448a066e9c25ce/internal/s3select/csv/reader.go)）。没有全局忽略 CVE，也不据此声称整体镜像无风险。部署时应使用自己的当前漏洞库再次扫描。 + +未运行真实 Kubernetes 集群、外网 TLS/代理或远端镜像仓库同步；Helm 证据限于 lint 和模板契约。旧 `minio:latest` 仍映射既有版本，其风险与升级另行管理，不能把新增 Silo 的扫描结论套给旧别名。 + +参考：[RustFS 官方部署资料](https://github.com/rustfs/docs.rustfs.com/blob/main/content/en/installation/container/docker.md)、[RustFS 发布](https://github.com/rustfs/rustfs/releases/tag/1.0.3)、[Silo 源仓库](https://github.com/pgsty/silo)。厂商兼容性声明不代替 SkillHub 自身验收。 diff --git a/docs/skillhub/guide/kubernetes.md b/docs/skillhub/guide/kubernetes.md index 2135b9a3b..0be3b5c92 100644 --- a/docs/skillhub/guide/kubernetes.md +++ b/docs/skillhub/guide/kubernetes.md @@ -1,5 +1,7 @@ # Kubernetes 部署指南 +对象存储可选部署、凭证和迁移说明见[对象存储手册（GitHub）](https://github.com/iflytek/skillhub/blob/main/docs/object-storage.md)。 + 本文档说明如何在 Kubernetes 集群中部署 SkillHub。 ## 前置条件 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/.openspec.yaml b/openspec/changes/add-s3-compatible-storage-deployments/.openspec.yaml new file mode 100644 index 000000000..dc36fb1bf --- /dev/null +++ b/openspec/changes/add-s3-compatible-storage-deployments/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-21 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/design.md b/openspec/changes/add-s3-compatible-storage-deployments/design.md new file mode 100644 index 000000000..0e46abd74 --- /dev/null +++ b/openspec/changes/add-s3-compatible-storage-deployments/design.md @@ -0,0 +1,183 @@ +## Context + +动机和范围见 [proposal.md](./proposal.md)。设计基线为 `6c739b838ed543dac9967e60b248fbacad4ad9f0`，源代码核对日期为 2026-10-21。本文件定义目标方案；实际实现与验收状态以 tasks.md 和对应交付记录为准。 + +### 已有能力与实际缺口 + +| 位置 | 已有事实 | 本次处理 | +|---|---|---| +| `server/skillhub-storage/.../ObjectStorageService.java` | 对象读写、删除、元数据、下载签名；上传签名与前缀统计为可选能力 | 保持接口边界，新增共享契约测试 | +| 同模块 `S3StorageService.java` | AWS SDK，支持 endpoint、publicEndpoint、路径寻址、预签名 | 验证协议差异，只有测试证明必要时才局部修复 | +| `compose.release.yml` | 传入 S3 参数，无本地对象存储服务 | 新增可选叠加文件，不改现有部署默认值 | +| `docker-compose.yml` | 开发环境 `minio` 服务实际使用 PGSTY Silo | 保留服务名和旧镜像别名兼容 | +| `.env.release.example` | 显式 local；Compose 缺省 provider 为 s3 | 新示例显式设置 provider，避免依赖不同层默认值 | +| `charts/skillhub/values.yaml` | `s3.enabled`、外部端点、`existingSecret` | 提供接入示例，不增加必装存储子 Chart | +| `GithubSyncUploadAppService` | local 流式上传、s3 预签名 PUT | 保留模式和上传凭证边界，验证 Worker 可达性 | + +S3 当前客户端初始化不等于存储可用性验证。既有单元测试不能证明 RustFS 或 Silo 的真实兼容性。批量删除响应中的逐项错误需要补充失败响应测试，以判断是否需要修复成功判定；不能仅凭 HTTP 200 宣告全部成功。 + +## Goals / Non-Goals + +**Goals:** 以统一 S3 协议保持业务代码稳定，交付可选存储运行方式、可重复验收及迁移指南。 + +**Non-Goals:** 不引入 `rustfs` / `silo` provider、业务模块专用 SDK、插件加载框架、多后端路由、双写、热切换、数据库迁移或后台配置页面。不将本次单机示例包装成高可用集群方案，不自动升级已有 Silo 部署。 + +## Decisions + +### D1. 架构：沿用协议接口，供应商差异留在部署层 + +```mermaid +flowchart TB + App[技能发布、下载、头像、Suite、同步编排] --> SPI[ObjectStorageService] + SPI --> Local[LocalFileStorageService] + SPI --> S3[S3StorageService / AWS SDK] + S3 --> R[RustFS] + S3 --> S[PGSTY Silo] + S3 --> E[外部 S3 兼容服务] + Env[部署环境与 Secret] --> S3 +``` + +`skillhub-storage` 持有接口、实现、属性与协议测试；应用服务只编排业务调用。产品初始化命令和镜像参数留在交付文件及操作手册中。现有 domain 到 storage 的已知依赖偏差不在本次扩大或重构。 + +替代方案：每个品牌实现一个 adapter 会重复签名、重试与错误处理，无法提升本次扩展性；通用插件框架增加没有实际调用需求的机制。只有将来出现实际非 S3 协议需求时再评估新实现。 + +### D2. 技术配置：复用已有键，部署时确定后端 + +下表全部是已有配置键。此处是新本地存储示例的目标取值，不覆盖全局默认值。 + +| 环境变量 | 示例或要求 | +|---|---| +| `SKILLHUB_STORAGE_PROVIDER` | `s3` | +| `SKILLHUB_STORAGE_S3_ENDPOINT` | `http://object-storage:9000`，后端和 Worker 可达 | +| `SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT` | 例如 `https://storage.example.com`，下载方可达；留空则使用内部端点 | +| `SKILLHUB_STORAGE_S3_BUCKET` | `skillhub`，先准备 | +| `SKILLHUB_STORAGE_S3_REGION` | 示例 `us-east-1`，与服务配置匹配 | +| `SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE` | 本地产品示例明确 `true` | +| `SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING` | 按契约测试结果显式设置，不以品牌条件散落在业务代码中 | +| `SKILLHUB_STORAGE_S3_ACCESS_KEY` / `SKILLHUB_STORAGE_S3_SECRET_KEY` | 业务账号，由受保护 env / Secret 注入 | +| `SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET` | 新示例 `false` | +| `SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY` | 沿用现有 `PT10M` 默认值，验收另用短期签名 | + +AWS 默认凭证链的既有接入能力保留，不强制所有外部 S3 使用静态密钥。静态凭证要求成对配置。密钥不进入 Git、镜像、系统配置表或诊断输出。 + +Helm 使用已有 `s3.*` 与 `existingSecret`，实施时核对模板实际 Secret 键名和优先级后提供可执行示例，不在此编造新的 values 字段。 + +### D3. 网络与签名：按实际访问方分流 + +```mermaid +sequenceDiagram + participant User as 浏览器或CLI + participant API as SkillHub + participant Public as 存储公开端点 + participant Internal as 存储内部端点 + participant Worker as GitHub同步Worker + User->>API: 请求下载（现有业务授权） + API-->>User: 公开端点的限时GET签名 + User->>Public: 原样使用签名URL + API->>Internal: 读写对象及元数据 + Worker->>API: 申请同步上传凭证 + API-->>Worker: 内部端点的PUT签名及所需请求头 + Worker->>Internal: 上传staging对象 +``` + +当前 GET presigner 使用 publicEndpoint，未设置时回退 endpoint；PUT presigner 使用 endpoint。公开域名、证书和反向代理由部署者准备，不能在生成签名后替换 host、scheme 或路径。公网下载可达但 Worker 不可达不算验收通过。 + +启用同步时配置已有 `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS`，以及正数的 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES`；沿用现有上传 ticket、lease 和容量检查，不绕过安全主机校验。必要的专用存储域名代理转发应保持签名语义，不把未验证的子路径重写列为支持配置。 + +### D4. Compose 与 Helm 的交付方式 + +交付文件： + +- `compose.storage.rustfs.yml`：RustFS 服务、健康检查、独立 `rustfs_data` 卷。 +- `compose.storage.silo.yml`：Silo 服务、健康检查、独立 `silo_data` 卷。 +- 产品配套 env 示例与初始化操作说明：由实施任务确定文档路径并链接到现有部署手册。 + +两个叠加文件各自提供 `object-storage` 服务，**只能选择一个**。由于 Compose 合并同名服务可能产生混合配置，不能依赖 Compose 自己报错：新增薄包装部署预检入口，检查输入文件列表，在执行 Compose 前拒绝同时选择二者，并将余下参数透传。手册采用该入口；直接调用 Compose 的高级用法需明确互斥条件。 + +部署操作与初始化步骤见 `docs/object-storage.md`；通过包装脚本保证单选： + +```sh +# 先准备私有桶和业务身份，再使用一个选定的叠加文件 +# 仓库根目录；操作会启动并修改相应项目的容器/卷 +# .env.release 由部署者准备，存储初始化步骤另见手册 +scripts/storage-compose.sh --storage rustfs --env-file .env.release --project-name skillhub -- up -d +``` + +Silo 将最后一个文件改为 `compose.storage.silo.yml`；外部 S3 只使用基础 Compose。status、升级和 down 必须沿用相同 project、env 和文件组合，不删除数据卷作为更新步骤。 + +启动顺序：存储健康 → 操作者或显式一次性初始化准备桶/业务账号 → 业务凭证诊断通过 → 启动应用 → 业务冒烟。基础 Compose 的本地存储卷和权限初始化暂不清理，避免扩大改动。 + +初始化优先提供人工可执行的明确步骤，不默认启动持有 root 密钥的常驻容器；如封装一次性初始化，必须幂等、失败阻断且完成后退出。具体产品命令、环境变量、健康接口和 UID 以所选发行版官方资料与运行结果确认，不能照搬另一产品参数。 + +Helm 将存储作为独立服务管理。Chart 的安装、升级和卸载不承担对象存储生命周期，避免卸载应用误删数据。现有测试脚本中使用 Silo 不代表 Chart 默认捆绑它。 + +### D5. 请求体重放与临时文件 + +实际发布通过 `Files.newInputStream` 提供不可 reset 的流。AWS SDK 2.20.26 的签名/重试会重读请求体，直接 `RequestBody.fromInputStream` 在此路径下会消耗流并使后续发送不完整；真实应用和文件流契约已复现超时，内存流测试不能覆盖此缺陷。 + +复用适配层已有暂存 helper，将每次 PUT 的请求体暂存到私有临时文件，用 `RequestBody.fromFile` 提供可重放的源；不在内存缓存整个包。上传前校验实际字节数与声明大小，成功失败均释放暂存文件。接口与调用方保持不变，代价为每次 S3 PUT 增加一次临时磁盘写入，部署需计入并发上传的临时空间预算。 + +### D6. 镜像、诊断与兼容性证据 + +镜像必须固定经验证版本与 digest，记录支持的 CPU 架构、许可证、来源和制品扫描结果。镜像同步使用产品独立且不可变的目标标签；保留已有 `minio:latest` 兼容映射，不用 RustFS 接管这个别名。`scripts/mirror-runtime-images.sh` 遇到已有目标会跳过，若需要刷新仓库必须显式验证目标 digest，不能靠改清单完成。 + +不新增公开可写健康接口。新增显式调用的存储诊断脚本/测试入口，经配置绑定真实 `ObjectStorageService`，在随机专用前缀执行 put/get/head/list/delete 与签名验证，设置整体超时，使用 finally 清理；清理失败也返回非零并报告脱敏的残留键。只删除本次生成的确切对象，不按宽泛前缀清空桶。普通 liveness 不执行远程写入。 + +共享契约测试用同一应用构件，按后端注入配置。真实测试失败不能因凭证或 Docker 缺失被计为通过；默认单元测试可不启动存储，但被选为交付门禁的集成任务缺环境必须失败或明确阻断。负向权限和批量部分失败先用确定性模拟响应验证错误分支，再在实际产品可控权限条件下验收。满批真实删除遇短暂逐项失败时，适配层必须抛错；契约测试记录这条失败证据，最多进行 3 次调用方恢复，最后独立列举确认对象不存在。生产适配器不新增一套隐式重试策略，沿用现有业务补偿。 + +## Risks / Trade-offs + +- [供应商声称 S3 兼容但有操作差异] → 只发布被测版本与契约范围，不能宣称任意版本全兼容。 +- [签名可生成但目标不可达] → 从真实浏览器/CLI 和 Worker 网络分别调用，不只断言 URL 字符串。 +- [最小权限不足或过宽] → 按实际对象、列举和批量删除操作配置桶权限；初始化权限不交给业务进程。 +- [故障时静默回退导致数据分散] → 不增加 fallback，沿用明确存储异常与脱敏日志。 +- [对象公开直读绕过业务权限] → 桶保持私有；检查匿名直读拒绝与签名有效期；临时签名持有者有效期内可读是现有行为。 +- [单机存储失败导致不可用] → 明确持久卷、备份恢复和容量监控，不将本方案宣称为 HA。 +- [更多配置增加维护成本] → 不改核心业务接口，使用共享测试与单一协议操作手册，产品特有内容限于部署附录。 + +## Migration Plan + +1. 清点现有 backend、桶、对象键、元数据和加密方式；备份对象及部署配置，演练恢复。原始数据卷不能直接挂给 RustFS；Silo 即使声称格式兼容，也不默认做原地切换。 +2. 准备独立目标桶和业务账号，在专用前缀跑契约探针；逻辑复制对象，保留 key、Content-Type 及必要元数据。跨加密后端通过授权逻辑读取再写入，不能拷贝内部加密文件。 +3. 记录对象清单、大小及内容校验结果，不能把 ETag 一律当 MD5；迁移工具以实际能力选择，记录版本与命令，不预设某个工具已支持全部元数据。 +4. 进入维护窗口，暂停发布、头像申请/处理、Suite 写入、GitHub 同步、过期清理等所有存储写入和删除；排空在途任务。对已签发 PUT 链接等待过期或以旧存储权限撤销写入，防止最终同步后仍有旧端点写入。 +5. 在明确冻结点执行最终同步，协调新增、覆盖和删除差异；旧源只读保留。GET 旧签名仍可能在有效期内指向旧端点，应保留可读访问或明确短暂中断窗口。 +6. 切换部署配置并重建受影响服务，验收元数据、实际下载、头像审核、Suite、同步上传/入库/清理。结果符合场景矩阵才恢复写入。 +7. 恢复写入前可切回未变更旧源；恢复写入后回退必须再次停写，将新后端新增、覆盖及删除差异协调回旧源并校验，不能仅改 endpoint。保留期由部署方按备份政策确定，确认恢复能力后才单独授权清理旧源。 + +无需 SQL 迁移或 CHECK 约束。文件系统 local → S3 也需逻辑复制，不能只修改 provider。 + +## 验收矩阵与质量门禁 + +下面是验收要求；实际执行结果在 tasks.md 和交付验收记录中逐项关联，未执行项不能借文档校验声明通过。 + +| 需求 | 验收场景和独立断言 | 阶段 / 阻断条件 | +|---|---|---| +| S3-01 | 同一应用对 RustFS/Silo 运行共享套件；local、外部 S3 配置兼容 | 集成；任一新增产品失败不得声明支持 | +| S3-02 | 随机二进制与中文/空格键精确读回；元数据比对；缺失对象；单删/批删/部分失败；分页及上限完整标志 | 单元＋真实集成；字节或错误判定不一致阻断 | +| S3-03 | 实际请求 GET/PUT；必需头；公开/内部网络；过期/篡改拒绝 | 集成；只生成 URL 不算通过 | +| S3-04 | 错密钥、拒绝权限、断网超时；匿名私有对象拒绝；无本地 fallback；日志无凭证 | 负向集成；越权、静默成功或泄密阻断 | +| DEP-01 | 两个文件分别渲染与启动；双选预检拒绝；独立卷；外部 S3 无本地服务 | 配置＋真实部署；配置混合或卷复用阻断 | +| DEP-02 | 初始化幂等、root 不进入应用；Helm Secret 模板；Worker allowed-host 与容量预算 | 配置＋集成；权限越界或同步失败阻断 | +| DEP-03 | 固定 digest/架构与镜像扫描证据；诊断成功清理、失败退出；旧镜像别名回归 | 交付；缺不可变制品或存在未处理阻断漏洞不发布 | +| DEP-04 | 带覆盖/删除/在途 PUT 的小样本迁移与回退，校验逻辑对象差异 | 迁移演练；丢对象/复活已删除内容阻断 | +| 跨能力 | 发布 ZIP→审核→下载字节一致；头像私有待审→批准→移除；Suite 包；GitHub staging PUT→导入→清理 | E2E；只用 SDK 测试不能替代业务路径 | + +需求门禁：本变更行为和范围明确；设计门禁：接口、网络、凭证、停写及回退边界一致；实现门禁：模块测试/配置回归通过；交付门禁：两种产品真实契约、关键业务链和迁移演练有证据。某项失败先修复并重跑受影响集合，不通过不得升级为“可发布”。保留支持矩阵，包含失败、未执行和限制，不只填写通过数量。 + +## Open Questions + +以下项目不改变架构与行为契约，但进入交付前必须补齐证据： + +- 已按官方 manifest 固定 RustFS 1.0.3、Silo RELEASE.2026-10-18T00-00-00Z（含 amd64/arm64），清单见 `docs/object-storage.md`。实际 amd64 镜像与初始化权限、扫描结果已核验，Silo 单条 legacy 模块告警的源码和行为裁定见用户手册；不把 manifest 核验扩大为 arm64 运行支持。 +- 迁移工具在所选产品组合下的元数据与删除差异处理能力；先小样本演练，再补实际操作指南。 + +## 参考与证据边界 + +- [Issue #937](https://github.com/iflytek/skillhub/issues/937)：问题来源；讨论不等于产品已验收。 +- [MinIO 源仓库](https://github.com/minio/minio)：维护状态参考，具体迁移仍需部署方决定。 +- [RustFS 官方 S3 兼容说明](https://docs.rustfs.com/en/reference/s3-compatibility)：厂商协议覆盖说明，不是 SkillHub 实测结果。 +- [RustFS 官方发布](https://github.com/rustfs/rustfs/releases)：实施时选择并核验制品，不能使用可变 latest 作为验收基线。 +- [PGSTY Silo 源仓库](https://github.com/pgsty/silo)：独立社区分支来源；格式兼容声明不替代备份恢复测试。 + +实现采用本次固定镜像及同一工作区应用构件验证，结果见任务状态与交付记录；不继承历史预览、其他应用或厂商测试的通过结论。 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/proposal.md b/openspec/changes/add-s3-compatible-storage-deployments/proposal.md new file mode 100644 index 000000000..c3855433d --- /dev/null +++ b/openspec/changes/add-s3-compatible-storage-deployments/proposal.md @@ -0,0 +1,27 @@ +## Why + +[#937](https://github.com/iflytek/skillhub/issues/937) 希望提供 MinIO 之外的对象存储选择。SkillHub 已有 S3 协议适配层，缺的是可复用的交付配置、真实兼容性证据和迁移操作边界。 + +## What Changes + +- 保持 `local` / `s3` 配置契约，通过同一 S3 接口接入 RustFS、PGSTY Silo 或外部 S3 服务。 +- 为正式 Compose 栈增加可选 RustFS / Silo 叠加文件，使用独立数据卷，不静默替换现有部署。 +- 明确内外网端点、预签名、业务凭证、桶初始化、GitHub 同步 Worker 网络和 Helm 外部存储配置。 +- 增加共享真实契约验收、迁移停写和回退指南，以及镜像供应链说明。 + +## Capabilities + +### New Capabilities + +- `s3-compatible-storage`: 不同 S3 实现下的对象读写、签名、权限和错误行为契约。 +- `object-storage-deployment`: 可选部署、配置诊断、兼容性声明与安全迁移交付契约。 + +### Modified Capabilities + +无。当前 `openspec/specs/` 尚无相应能力正文；新增规格记录本次目标，不代表重写全部已有实现。 + +## Impact + +主要影响 `server/skillhub-storage`、Compose、配置校验、镜像清单、Helm 示例和部署手册。验证发布下载、头像审核对象、Suite 包和 GitHub 同步调用链；业务模块不引入供应商 SDK。 + +不改数据库、公开 API、技能生命周期或权限策略，不增加后台存储配置页面。本轮仅交付设计，代码、镜像、部署和兼容性证明待实施。 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md b/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md new file mode 100644 index 000000000..dd0e88f2a --- /dev/null +++ b/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md @@ -0,0 +1,49 @@ +## Purpose + +为部署者提供可选的 S3 兼容存储交付方式、配置检查及可追溯的验收依据，并明确已有对象迁移的安全要求。新增选项不应使原有部署被自动替换，也不能把配置更新误认为数据已完成迁移。 + +## ADDED Requirements + +### Requirement: DEP-01 存储选项独立部署 +交付物 SHALL 提供 RustFS 和 Silo 可选 Compose 叠加配置，每次仅启用一个本地存储选项，数据卷独立；外部 S3 不要求启动本地存储。Helm SHALL 保留外部存储接入，不强制安装存储子 Chart。 + +#### Scenario: 选择本地存储 +- **WHEN** 部署者按手册选择一种存储叠加配置 +- **THEN** 应用通过稳定的内部服务地址连接所选服务，另一产品不会启动或共用数据目录 + +#### Scenario: 选择冲突 +- **WHEN** 部署者试图同时选择两种本地存储选项 +- **THEN** 交付预检明确拒绝此组合，不能静默选择其中之一 + +### Requirement: DEP-02 部署配置与凭证边界 +端点、桶、凭证和产品 SHALL 由部署配置决定，变更通过重建相关服务生效，不进入系统设置表或后台热切换。示例 SHALL 区分初始化管理员凭证与应用业务凭证，默认不依赖应用自动建桶。 + +#### Scenario: 新实例初始化 +- **WHEN** 部署者按示例准备桶和具备所需对象权限的业务账号 +- **THEN** 应用无需存储 root 凭证即可运行，文档指出前缀统计所需的列举权限 + +#### Scenario: Worker 接入 +- **WHEN** 启用 S3 模式 GitHub 同步 +- **THEN** 手册要求 Worker 可达上传端点、允许目标主机并配置有效 staging 容量预算，完整导入与清理能够验收 + +### Requirement: DEP-03 可验证的交付与兼容声明 +交付 SHALL 记录应用修订、被测产品版本、不可变镜像摘要、架构和验收结果。配置检查与真实对象诊断必须区分，服务存活不能替代真实操作。旧镜像别名 SHALL 保持兼容，映射文件修改不能视为目标镜像已刷新。 + +#### Scenario: 发布支持矩阵 +- **WHEN** 宣布某个产品版本受支持 +- **THEN** 共享契约和关键业务路径有真实结果，未执行项及限制被列出，阻断用例失败时不能声明通过 + +#### Scenario: 运行诊断 +- **WHEN** 操作者执行存储诊断 +- **THEN** 在专用前缀进行有时限的真实对象操作并清理，失败非零退出，不泄露密钥或触碰业务对象 + +### Requirement: DEP-04 逻辑迁移与回退 +迁移指南 SHALL 要求保留对象键和必要元数据、校验内容、暂停相关写入并完成最终同步后切换。不同产品不得默认复用原始数据卷；切换配置不代表迁移完成。 + +#### Scenario: 已有数据切换 +- **WHEN** 实例从现有存储迁往另一产品 +- **THEN** 先备份复制，再停写并处理在途任务，校验最终差异，验证发布下载和同步后才恢复写入 + +#### Scenario: 接受新写入后回退 +- **WHEN** 新后端已经接受业务写入但需要回退 +- **THEN** 指南要求再次停写并协调新增和删除差异，不能仅切回旧端点导致对象丢失或已删除数据复活 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md b/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md new file mode 100644 index 000000000..bfb3d70d4 --- /dev/null +++ b/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md @@ -0,0 +1,57 @@ +## Purpose + +使部署方可以通过既有 S3 配置选择不同对象存储产品，同时维持技能文件、审核图片、下载链接和同步任务的可观察行为。支持范围由真实契约验收证明，不以供应商的兼容性宣传替代应用验证。 + +## ADDED Requirements + +### Requirement: S3-01 统一协议配置 +系统 SHALL 使用既有 `s3` 配置连接通过验收的 S3 兼容服务，保留 `local` 模式，不要求业务调用者按存储品牌选择接口。 + +#### Scenario: 更换空桶目标 +- **WHEN** 新实例配置已准备的 RustFS 或 Silo 桶及有效业务凭证 +- **THEN** 相同应用版本通过相同接口完成对象操作，无需修改技能元数据或指定品牌 provider + +#### Scenario: 原有部署兼容 +- **WHEN** 部署方沿用有效的 local 或外部 S3 配置 +- **THEN** 原模式继续可用，不因新增选项自动更换后端 + +### Requirement: S3-02 对象操作完整性 +系统 SHALL 保持对象键、字节、长度和内容类型的读写语义；单个和批量删除均不得将后端报告的失败当作全部成功。前缀统计 SHALL 明确数据是否完整。 + +#### Scenario: 特殊对象键往返 +- **WHEN** 写入包含中文、空格和 URL 保留字符的合法对象键并读取元数据和内容 +- **THEN** 键与内容不发生二次编码或截断，字节、长度和内容类型与输入一致 + +#### Scenario: 流式技能上传 +- **WHEN** 上传的合法技能包包含超过 128 KiB 的文件并通过现有流式发布处理 +- **THEN** 写入和下载字节与原文件一致，不因请求体重读而丢失内容或持续超时 + +#### Scenario: 批量删除部分失败 +- **WHEN** 后端在批量删除响应中报告部分对象失败 +- **THEN** 调用方能够识别失败，不得到全部删除成功的结论 + +#### Scenario: 前缀统计截断 +- **WHEN** 对象数量超过一次分页或扫描上限 +- **THEN** 继续分页或明确报告统计不完整，不将部分数量声明为总量 + +### Requirement: S3-03 端点与预签名契约 +系统 SHALL 为实际访问方可达的端点生成预签名 URL，保留下载公开端点及同步上传内部端点的分工；不得通过事后改写已签名 URL 主机解决可达性。 + +#### Scenario: 内外网端点分离 +- **WHEN** 后端使用内部端点，浏览器访问配置的公开端点，Worker 可达内部端点 +- **THEN** 下载签名指向公开端点且读取成功，Worker 上传签名指向内部端点且上传成功 + +#### Scenario: 签名失效 +- **WHEN** 使用过期、篡改签名或缺失签名要求的请求头 +- **THEN** 存储拒绝请求，不返回对象内容或写入成功 + +### Requirement: S3-04 隔离与失败可见 +系统 SHALL 维持现有私有对象与业务授权边界；凭证、网络和存储错误必须可观察，不得静默写入其他后端。凭证不得出现在公开链接、日志或报告中。 + +#### Scenario: 私有对象访问 +- **WHEN** 匿名请求直接读取私有桶对象且无有效签名 +- **THEN** 请求被拒绝，正常业务授权仍可生成限时下载链接 + +#### Scenario: 后端不可用 +- **WHEN** 业务凭证无效、权限不足或目标网络不可达 +- **THEN** 操作失败并提供可定位且脱敏的错误，不转写本地目录、不返回伪造成功 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/tasks.md b/openspec/changes/add-s3-compatible-storage-deployments/tasks.md new file mode 100644 index 000000000..08fb6870d --- /dev/null +++ b/openspec/changes/add-s3-compatible-storage-deployments/tasks.md @@ -0,0 +1,42 @@ +## 1. 制品与回归基线 + +- [x] 1.1 读取本方案与部署/存储测试规范，确认实现 diff 只覆盖既定边界；通过需求到测试映射复核，不引入新 provider、数据库或公开 API。 +- [x] 1.2 核验 RustFS/Silo 发行版、官方镜像来源、digest、CPU 架构、许可证与扫描结果；产出可追溯制品表，阻断问题未处理不得进入交付。 +- [x] 1.3 先补共享契约与确定性失败用例，覆盖 S3-01～04；确认能发现错误字节、删除部分失败和无效签名被接受等缺陷，记录现有偏差。 + +## 2. 协议与诊断 + +- [x] 2.1 按 1.3 的真实缺口局部修复 S3 操作，保留接口与可选能力语义；运行 storage 模块测试及受影响调用方回归，无缺口则记录无需改动。 +- [x] 2.2 提供有整体超时的显式对象诊断入口；成功清理、认证失败、超时、清理失败均有断言，失败非零退出且不触及业务对象/输出凭证。 + +## 3. 部署交付 + +- [x] 3.1 新增 RustFS 叠加文件、独立卷和 env 示例；用固定制品运行配置渲染、健康、重启持久化及初始化幂等检查，应用不持有 root 凭证。 +- [x] 3.2 新增 Silo 叠加文件、独立卷和 env 示例；执行同等检查并回归旧开发服务及 minio 镜像别名，禁止两个产品共用原始卷。 +- [x] 3.3 增加薄包装预检，明确输入文件列表、拒绝双叠加、透传合法组合；用无服务启动的 shell 测试覆盖单选、双选、外部 S3 与非法参数。 +- [x] 3.4 按需扩展现有配置校验并补测试，核对变量绑定/透传/默认值；local、外部凭证链、静态凭证、路径寻址和非法 URL 有确定结果。 +- [x] 3.5 提供 Helm 外部存储与 existingSecret 示例；核对模板键名与优先级，通过 lint/render，确认不安装存储子 Chart、不输出 Secret 明文。 +- [x] 3.6 加入产品独立不可变镜像映射并验证实际目标 digest；保留 minio 别名，测试目标已存在跳过的行为，报告未执行的仓库同步。 + +## 4. 真实验收 + +- [x] 4.1 在隔离目标分别对 RustFS/Silo 运行相同应用共享契约，包括分页、权限、签名和失败路径；保存版本、命令、结果与残留清理证据，不将缺环境或 skip 计为通过。 +- [x] 4.2 按 design.md 矩阵跑发布审核下载、头像对象、Suite 和 GitHub 同步完整路径；验证浏览器/CLI 公开端点与 Worker 内部端点，结果映射回需求编号。 +- [x] 4.3 用覆盖、删除和在途 PUT 样本演练迁移、冻结、最终同步及恢复写入后的回退；对象键、内容与元数据比对通过，不把 ETag 一律当内容哈希。 + +## 5. 用户文档与收尾 + +- [x] 5.1 更新部署手册、Kubernetes 示例和同步运维入口，写明初始化/域名/凭证/互斥/停写/回退命令及副作用；在干净环境按手册复现，未实测项显式标注。 +- [x] 5.2 汇总支持矩阵与脱敏验收报告，逐项关联 S3-01～04、DEP-01～04 和跨能力场景；未通过项目阻断对应支持声明。 +- [x] 5.3 执行 OpenSpec 严格校验、变更范围核查及一次关键阶段设计/实现一致性 Review；记录结果后再进入用户授权范围内的 PR 流程，不自动发布或部署生产。 + +## 执行摘要 + +- 2026-10-21：实现与本地运行验收。RustFS 1.0.3 和 Silo 2026-10-18 使用同一应用构件，实际运行架构为 amd64；镜像清单另核验 arm64 条目，不将其写成 arm64 运行通过。 +- 存储 36 项单元测试、领域 27 项与应用调用方 34 项回归通过。默认跳过的两个显式集成用例已在两产品上分别开启运行，无 skip 冒充通过。 +- 两产品均验证了文件流、签名、分页、删除、发布审核下载、头像、Suite、Go 内部网络上传与暂存清理；故意错误凭证、桶外权限及网络不可达均拒绝且不返回伪造成功。 +- RustFS → Silo 冻结点逐项比对 134 对象，Silo → RustFS 回退冻结点比对 135 对象，包含新写入、覆盖、删除与在途 PUT 排空；原始卷不跨产品复用。 +- 独立 Review 指出的新 Silo 映射目标标签已修复并增加精确配对断言；评审剩余 blocker/high/medium 均为 0。该结论绑定工作区，不替代最终提交与 CI 门禁。 +- 新 Silo 镜像的单条 legacy MinIO 模块告警按源码 commit、二进制版本和真实有界 CSV 拒绝用例裁定为已修补，保留原始扫描结果，不全局忽略告警。 +- 远端镜像同步、真实 K8s、外网 TLS/代理未执行；本地 JAR 构建加官方 Dockerfile runtime 阶段镜像已验证，不宣称完整多阶段 Docker 构建或 PR CI 通过。本地验收记录截止时尚未提交、推送或创建 PR；最终提交、PR 与 CI 状态由后续交付流程记录，不将本地结果视为合并门禁通过。 +- 本地脱敏验收记录单独交付，不将运行日志、凭证、私有路径或工具状态纳入 Git。 diff --git a/scripts/storage-compose.sh b/scripts/storage-compose.sh new file mode 100755 index 000000000..3580308d8 --- /dev/null +++ b/scripts/storage-compose.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Compose entry point that makes the optional local storage selection exclusive. +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +usage() { + echo "Usage: $0 --storage rustfs|silo|external [--env-file FILE] [--project-name NAME] -- COMPOSE_ARGS..." +} +fail() { echo "ERROR: $*" >&2; exit 2; } +storage="" +env_file=".env.release" +project_args=() +while (($#)); do + case "$1" in + --storage) + [[ $# -ge 2 && -z "$storage" ]] || fail "Select exactly one storage backend" + storage="$2"; shift 2 ;; + --env-file) + [[ $# -ge 2 && -n "$2" ]] || fail "Missing env file" + env_file="$2"; shift 2 ;; + --project-name) + [[ $# -ge 2 && -n "$2" ]] || fail "Missing project name" + project_args=(--project-name "$2"); shift 2 ;; + -h|--help) usage; exit 0 ;; + --) shift; break ;; + *) fail "Unknown option: $1" ;; + esac +done +case "$storage" in rustfs|silo|external) ;; *) fail "Select rustfs, silo or external" ;; esac +(($#)) || fail "Missing Compose command" +# Do not allow additional global files to silently merge incompatible storage services. +for arg in "$@"; do + # In logs, -f means follow rather than a global Compose file option. + [[ "$arg" != -f || "${1:-}" != logs ]] || continue + case "$arg" in + -f|--file|-f?*|--file=*|--env-file|--env-file=*|--project-directory|--project-directory=*) + fail "Compose file and env selection must use this script's options" ;; + esac +done +cd "$REPO_ROOT" +[[ -f "$env_file" ]] || fail "Env file not found" +files=(-f compose.release.yml) +if [[ "$storage" != external ]]; then + files+=(-f "compose.storage.$storage.yml") +fi +exec docker compose --env-file "$env_file" "${project_args[@]}" "${files[@]}" "$@" diff --git a/scripts/storage-diagnostics.sh b/scripts/storage-diagnostics.sh new file mode 100755 index 000000000..ec2f39292 --- /dev/null +++ b/scripts/storage-diagnostics.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Source-checkout diagnostic; writes only a random diagnostic prefix in a prepared bucket. +set -euo pipefail +umask 077 +full=false +deadline=180s +if [[ "${1:-}" == --full && $# -eq 1 ]]; then + full=true + deadline=420s +elif (($#)); then + echo "Usage: $0 [--full]" >&2 + exit 2 +fi +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +command -v timeout >/dev/null || { echo "ERROR: GNU timeout is required" >&2; exit 2; } +: "${SKILLHUB_STORAGE_S3_ENDPOINT:?A reachable S3 endpoint is required}" +: "${SKILLHUB_STORAGE_S3_BUCKET:?A prepared private bucket is required}" +log_dir="${SKILLHUB_STORAGE_DIAGNOSTICS_LOG_DIR:-$REPO_ROOT/.dev/storage-diagnostics}" +mkdir -p "$log_dir" +log_file="$(mktemp "$log_dir/run-XXXXXXXX.log")" +cd "$REPO_ROOT/server" +# Raw SDK/network errors may contain signed URLs; retain them only in the private log. +if timeout --signal=TERM --kill-after=15s "$deadline" ./mvnw -pl skillhub-storage -am \ + -Dtest=S3StorageContractTest -Dsurefire.failIfNoSpecifiedTests=false \ + -Dskillhub.storage.contract=true -Dskillhub.storage.contract.full="$full" test >"$log_file" 2>&1; then + echo "Storage diagnostic passed (objects, metadata, signatures, privacy, deletion)." + echo "Private log: $log_file" +else + result=$? + echo "Storage diagnostic failed (exit $result). Check the private log; do not publish it unredacted." >&2 + echo "Private log: $log_file" >&2 + exit "$result" +fi diff --git a/scripts/tests/mirror-storage-images-test.sh b/scripts/tests/mirror-storage-images-test.sh new file mode 100755 index 000000000..41a995830 --- /dev/null +++ b/scripts/tests/mirror-storage-images-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +mkdir "$work/bin" +cat >"$work/bin/docker" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$MIRROR_COMMAND_LOG" +case "$*" in + 'buildx imagetools inspect '*) [[ "${TARGET_EXISTS:-false}" == true ]] ;; + 'buildx imagetools create '*) exit 0 ;; + *) exit 2 ;; +esac +STUB +chmod +x "$work/bin/docker" +export PATH="$work/bin:$PATH" MIRROR_COMMAND_LOG="$work/commands" +export MIRROR_REGISTRY=registry.example.test MIRROR_NAMESPACE=skillhub +printf '%s\n' 'docker.io/rustfs/rustfs@sha256:abc rustfs:immutable-test' >"$work/images" +TARGET_EXISTS=true bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output" +if grep -q 'imagetools create' "$work/commands"; then echo 'FAIL: existing target was refreshed' >&2; exit 1; fi +TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output" +grep -q 'imagetools create --tag registry.example.test/skillhub/rustfs:immutable-test docker.io/rustfs/rustfs@sha256:abc' "$work/commands" +grep -q '^docker.io/pgsty/silo:RELEASE.2026-09-16T00-00-00Z minio:latest$' "$REPO_ROOT/deploy/runtime-mirror-images.txt" +grep -Fxq 'docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7 silo:20261018-2636fc4155d9' "$REPO_ROOT/deploy/runtime-mirror-images.txt" +echo 'Storage mirror skip and legacy alias tests passed' diff --git a/scripts/tests/storage-compose-render-test.sh b/scripts/tests/storage-compose-render-test.sh new file mode 100755 index 000000000..140eb27b8 --- /dev/null +++ b/scripts/tests/storage-compose-render-test.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +python3 - "$REPO_ROOT" <<'CHECK' +import json, os, pathlib, subprocess, sys, tempfile, urllib.parse +root=pathlib.Path(sys.argv[1]) +# Fixed non-production fixture values; no service is started by compose config. +env={**os.environ, + 'SKILLHUB_OBJECT_STORAGE_ROOT_USER':'fixture-storage-admin', + 'SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD':'fixture-storage-admin-secret', + 'SKILLHUB_STORAGE_PROVIDER':'s3', + 'SKILLHUB_STORAGE_S3_ACCESS_KEY':'fixture-business-user', + 'SKILLHUB_STORAGE_S3_SECRET_KEY':'fixture-business-secret', + 'SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT':'https://storage.example.test', + 'SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS':'object-storage:9000', + 'SKILLHUB_DOWNLOAD_ANON_COOKIE_SECRET':'fixture-download-secret-at-least-32-characters'} +for key in ['RUSTFS_IMAGE','SILO_IMAGE']: + env.pop(key,None) +with tempfile.TemporaryDirectory() as work: + config=pathlib.Path(work)/'fixture.env' + config.touch() + for product in ['rustfs','silo','external']: + command=[str(root/'scripts/storage-compose.sh'),'--storage',product,'--env-file',str(config), + '--project-name','storage-render-contract','--','--profile','github-sync','config','--format','json'] + data=json.loads(subprocess.check_output(command,cwd=root,env=env)) + services=data['services'] + server=services['server']['environment'] + assert server['SKILLHUB_STORAGE_S3_ACCESS_KEY']=='fixture-business-user' + assert server['SKILLHUB_STORAGE_S3_SECRET_KEY']=='fixture-business-secret' + assert 'fixture-storage-admin-secret' not in json.dumps(server) + assert not any('ROOT_PASSWORD' in key or 'ROOT_USER' in key for key in server) + if product=='external': + assert 'object-storage' not in services and 'object-storage-permissions' not in services + continue + storage=services['object-storage'] + assert '@sha256:' in storage['image'] and len(storage['image'].rsplit(':',1)[1])==64 + assert server['SKILLHUB_STORAGE_PROVIDER']=='s3' + assert server['SKILLHUB_STORAGE_S3_ENDPOINT']=='http://object-storage:9000' + assert server['SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE']=='true' + assert server['SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET']=='false' + assert storage['volumes'][0]['source']==product+'_data' + other='silo_data' if product=='rustfs' else 'rustfs_data' + assert other not in data['volumes'] + assert services['github-sync']['environment']['SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS']==urllib.parse.urlparse(server['SKILLHUB_STORAGE_S3_ENDPOINT']).netloc + assert services['server']['depends_on']['object-storage']['condition']=='service_healthy' + if product=='rustfs': + initializer=services['object-storage-permissions'] + assert initializer['network_mode']=='none' and initializer['read_only'] + assert initializer['entrypoint']==['chown','10001:10001','/data'] + assert 'environment' not in initializer or not initializer['environment'] +print('Storage Compose rendering, isolated volumes, business credentials and Worker endpoint tests passed') +CHECK diff --git a/scripts/tests/storage-compose-test.sh b/scripts/tests/storage-compose-test.sh new file mode 100755 index 000000000..fd1530d40 --- /dev/null +++ b/scripts/tests/storage-compose-test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +mkdir "$work/bin" +cat >"$work/bin/docker" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$@" >"$DOCKER_ARGS_FILE" +STUB +chmod +x "$work/bin/docker" +export PATH="$work/bin:$PATH" DOCKER_ARGS_FILE="$work/args" +touch "$work/env" +run() { "$REPO_ROOT/scripts/storage-compose.sh" --env-file "$work/env" "$@"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +run --storage rustfs --project-name storage-contract -- config +[[ "$(cat "$work/args")" == *compose.storage.rustfs.yml* ]] || fail "RustFS overlay missing" +[[ "$(cat "$work/args")" != *compose.storage.silo.yml* ]] || fail "Unexpected Silo" +run --storage silo -- config +[[ "$(cat "$work/args")" == *compose.storage.silo.yml* ]] || fail "Silo overlay missing" +run --storage external -- --profile github-sync config +[[ "$(cat "$work/args")" != *compose.storage.* ]] || fail "External storage started a local backend" +run --storage rustfs -- logs -f +[[ "$(cat "$work/args")" == *logs* ]] || fail "Log command not forwarded" +for args in 'rustfs --storage silo' 'unknown'; do + # Intentional word splitting for these fixed malformed input fixtures. + if run --storage $args -- config >/dev/null 2>&1; then fail "Accepted invalid selection"; fi +done +if run --storage rustfs -- -f compose.storage.silo.yml config >/dev/null 2>&1; then fail "Accepted file injection"; fi +if run --storage rustfs -- --file=compose.storage.silo.yml config >/dev/null 2>&1; then fail "Accepted file injection"; fi +if run --storage rustfs >/dev/null 2>&1; then fail "Accepted missing command"; fi +if run --env-file "$work/missing" --storage rustfs -- config >/dev/null 2>&1; then fail "Accepted missing env"; fi +echo "Storage Compose selection tests passed" diff --git a/scripts/tests/storage-diagnostics-test.sh b/scripts/tests/storage-diagnostics-test.sh new file mode 100755 index 000000000..0d8d6d20f --- /dev/null +++ b/scripts/tests/storage-diagnostics-test.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/repo/scripts" "$work/repo/server" "$work/bin" +cp "$REPO_ROOT/scripts/storage-diagnostics.sh" "$work/repo/scripts/" +cat >"$work/repo/server/mvnw" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$DIAGNOSTIC_COMMAND_LOG" +echo 'sensitive-transport-detail-must-not-reach-console' +exit "${DIAGNOSTIC_EXIT_CODE:-0}" +STUB +cat >"$work/bin/timeout" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$DIAGNOSTIC_COMMAND_LOG" +[[ "${DIAGNOSTIC_TIMEOUT:-false}" != true ]] || exit 124 +shift 3 +exec "$@" +STUB +chmod +x "$work/repo/server/mvnw" "$work/bin/timeout" +export PATH="$work/bin:$PATH" DIAGNOSTIC_COMMAND_LOG="$work/commands" +export SKILLHUB_STORAGE_S3_ENDPOINT=https://storage.example.test SKILLHUB_STORAGE_S3_BUCKET=skillhub +fail() { echo "FAIL: $*" >&2; exit 1; } +script="$work/repo/scripts/storage-diagnostics.sh" +bash "$script" >"$work/output" 2>&1 +grep -q 'Storage diagnostic passed' "$work/output" || fail 'Success not reported' +if grep -q sensitive-transport "$work/output"; then fail 'Raw transport details leaked'; fi +for log in "$work/repo/.dev/storage-diagnostics/"*.log; do + [[ "$(stat -c %a "$log")" == 600 ]] || fail 'Raw diagnostic log must be private' +done +bash "$script" --full >"$work/output" 2>&1 +grep -q -- '-Dskillhub.storage.contract.full=true' "$work/commands" || fail 'Full contract suite not enabled' +grep -q '420s' "$work/commands" || fail 'Full suite deadline missing' +if DIAGNOSTIC_EXIT_CODE=7 bash "$script" >"$work/output" 2>&1; then fail 'Test failure reported as success'; else [[ $? == 7 ]] || fail 'Exit code not preserved'; fi +if DIAGNOSTIC_TIMEOUT=true bash "$script" >"$work/output" 2>&1; then fail 'Timeout reported as success'; else [[ $? == 124 ]] || fail 'Timeout code not preserved'; fi +if bash "$script" --unknown >"$work/output" 2>&1; then fail 'Unknown flag accepted'; fi +if SKILLHUB_STORAGE_S3_ENDPOINT= bash "$script" >"$work/output" 2>&1; then fail 'Missing endpoint accepted'; fi +echo 'Storage diagnostic timeout, privacy and exit-code tests passed' diff --git a/scripts/tests/validate-release-config-test.sh b/scripts/tests/validate-release-config-test.sh index 4d1370ccc..0d7426ebe 100755 --- a/scripts/tests/validate-release-config-test.sh +++ b/scripts/tests/validate-release-config-test.sh @@ -363,4 +363,9 @@ while IFS= read -r line || [[ -n "$line" ]]; do done <"$REPO_ROOT/.env.release.draft" >"$draft_env" expect_fail "$draft_env" "POSTGRES_PASSWORD" +invalid_chunked_env="$tmp/invalid-chunked.env" +write_env "$invalid_chunked_env" "release-download-secret-32-bytes-minimum" +printf '%s\n' 'SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=perhaps' >>"$invalid_chunked_env" +expect_fail "$invalid_chunked_env" "SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING" + echo "validate-release-config-test passed" diff --git a/scripts/validate-release-config.sh b/scripts/validate-release-config.sh index d2f770356..f84cbb4e1 100755 --- a/scripts/validate-release-config.sh +++ b/scripts/validate-release-config.sh @@ -315,6 +315,7 @@ validate_boolean SESSION_COOKIE_SECURE validate_boolean BOOTSTRAP_ADMIN_ENABLED validate_boolean SKILLHUB_TRUST_FORWARDED_PROTO validate_boolean SKILLHUB_BUILTIN_SKILLS_ENABLED +validate_boolean SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING validate_boolean SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE validate_boolean SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET validate_boolean SPRING_DATA_REDIS_SSL_ENABLED diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java index c1cc02492..cc45eb055 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -162,10 +162,6 @@ public class S3StorageService implements ObjectStorageService { } } - private void putObjectInternal(String key, InputStream data, long size, String contentType) { - putObjectInternal(key, RequestBody.fromInputStream(data, size), size, contentType); - } - private void putObjectInternal(String key, RequestBody requestBody, long size, String contentType) { s3Client.putObject( PutObjectRequest.builder() @@ -202,18 +198,24 @@ public class S3StorageService implements ObjectStorageService { @Override public void putObject(String key, InputStream data, long size, String contentType) { Path stagedBody = null; try { + // Signing and retries can read a request body more than once. Callers can + // supply non-resettable file streams, so keep a replayable, disk-backed body. + stagedBody = stagePutObjectBody(data); + if (Files.size(stagedBody) != size) { + throw new IllegalArgumentException("Object size does not match the supplied stream"); + } + RequestBody requestBody = RequestBody.fromFile(stagedBody); if (!properties.isAutoCreateBucket() || bucketPrepared) { - putObjectInternal(key, data, size, contentType); + putObjectInternal(key, requestBody, size, contentType); return; } - stagedBody = stagePutObjectBody(data); try { - putObjectInternal(key, RequestBody.fromFile(stagedBody), size, contentType); + putObjectInternal(key, requestBody, size, contentType); bucketPrepared = true; } catch (NoSuchBucketException e) { ensureBucketPrepared(); - putObjectInternal(key, RequestBody.fromFile(stagedBody), size, contentType); + putObjectInternal(key, requestBody, size, contentType); } } catch (IOException e) { throw new StorageAccessException("putObject", key, e); @@ -243,8 +245,23 @@ public class S3StorageService implements ObjectStorageService { @Override public void deleteObjects(List keys) { if (keys.isEmpty()) return; try { - List ids = keys.stream().map(k -> ObjectIdentifier.builder().key(k).build()).toList(); - s3Client.deleteObjects(DeleteObjectsRequest.builder().bucket(properties.getBucket()).delete(Delete.builder().objects(ids).build()).build()); + for (int start = 0; start < keys.size(); start += 1000) { + List ids = keys.subList(start, Math.min(start + 1000, keys.size())).stream() + .map(key -> ObjectIdentifier.builder().key(key).build()).toList(); + DeleteObjectsResponse response = s3Client.deleteObjects(DeleteObjectsRequest.builder() + .bucket(properties.getBucket()) + .delete(Delete.builder().objects(ids).build()) + .build()); + // S3 can return HTTP 200 with per-object failures. Earlier successful deletes + // cannot be rolled back; report failure so callers can retry idempotently. + if (!response.errors().isEmpty()) { + String codes = response.errors().stream() + .map(error -> error.code() == null ? "unknown" : error.code()) + .distinct().limit(5).collect(Collectors.joining(", ")); + throw new IllegalStateException("S3 batch deletion reported " + response.errors().size() + + " failures; codes=" + codes); + } + } } catch (RuntimeException e) { throw new StorageAccessException("deleteObjects", String.join(",", keys), e); } diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageContractTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageContractTest.java new file mode 100644 index 000000000..1372bf3f2 --- /dev/null +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageContractTest.java @@ -0,0 +1,98 @@ +package com.iflytek.skillhub.storage; + +import java.util.ArrayList; +import java.util.UUID; +import java.io.ByteArrayInputStream; +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.core.env.StandardEnvironment; + +import java.util.concurrent.TimeUnit; + +/** Explicit opt-in: an enabled run with missing configuration fails rather than skips. */ +@EnabledIfSystemProperty(named = "skillhub.storage.contract", matches = "true") +class S3StorageContractTest { + @Test + @Timeout(value = 120, unit = TimeUnit.SECONDS, threadMode = Timeout.ThreadMode.SEPARATE_THREAD) + void configuredBackendMustSatisfySharedContract() throws Exception { + var service = configuredService(); + try { + service.init(); + new StorageContractProbe(service).verify(); + } finally { + service.close(); + } + } + + @Test + @EnabledIfSystemProperty(named = "skillhub.storage.contract.full", matches = "true") + @Timeout(value = 240, unit = TimeUnit.SECONDS, threadMode = Timeout.ThreadMode.SEPARATE_THREAD) + void realBackendMustTraverseMultiplePagesAndDeleteMoreThanOneBatch() { + var service = configuredService(); + var prefix = "skillhub-diagnostics/" + UUID.randomUUID() + "/pagination/"; + var keys = new ArrayList(); + System.out.println("Pagination diagnostic object prefix: " + prefix); + Throwable failure = null; + try { + service.init(); + for (int i = 0; i < 1001; i++) { + String key = prefix + i; + keys.add(key); + service.putObject(key, new ByteArrayInputStream(new byte[]{42}), 1, "application/octet-stream"); + } + assertThat(service.listPrefixUsage(prefix, 2000)) + .contains(new StoragePrefixUsage(1001, 1001, true)); + assertThat(service.listPrefixUsage(prefix, 1000)) + .contains(new StoragePrefixUsage(1000, 1000, false)); + // A real service may report transient per-object failures under a full batch. + // Keep these visible, then exercise caller-side idempotent recovery with a bound. + for (int attempt = 1; ; attempt++) { + try { + service.deleteObjects(keys); + break; + } catch (StorageAccessException exception) { + if (attempt == 3) throw exception; + Throwable cause = exception.getCause(); + String detail = cause instanceof IllegalStateException + && cause.getMessage() != null + && cause.getMessage().startsWith("S3 batch deletion reported ") + ? cause.getMessage() : cause == null ? "unknown" : cause.getClass().getSimpleName(); + System.out.println("Batch deletion reported failure; caller recovery attempt " + + attempt + "; " + detail); + } + } + assertThat(service.listPrefixUsage(prefix, 2000)) + .contains(new StoragePrefixUsage(0, 0, true)); + keys.clear(); // Absence was verified; do not issue a redundant cleanup load burst. + } catch (RuntimeException | Error exception) { + failure = exception; + throw exception; + } finally { + try { + service.deleteObjects(keys); + } catch (RuntimeException exception) { + if (failure != null) failure.addSuppressed(exception); + else throw exception; + } finally { + service.close(); + } + } + } + + private S3StorageService configuredService() { + var properties = Binder.get(new StandardEnvironment()) + .bind("skillhub.storage.s3", S3StorageProperties.class).orElseThrow( + () -> new IllegalStateException("S3 diagnostic configuration is missing")); + if (properties.getEndpoint() == null || properties.getEndpoint().isBlank() + || properties.getBucket() == null || properties.getBucket().isBlank()) { + throw new IllegalStateException("S3 endpoint and prepared bucket are required"); + } + properties.setAutoCreateBucket(false); + // Use the same per-call timeout defaults as the application, including batch operations. + return new S3StorageService(properties); + } +} diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageProtocolTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageProtocolTest.java new file mode 100644 index 000000000..85c059ead --- /dev/null +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageProtocolTest.java @@ -0,0 +1,65 @@ +package com.iflytek.skillhub.storage; + +import org.springframework.test.util.ReflectionTestUtils; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class S3StorageProtocolTest { + @Test + void usageFollowsContinuationTokensAndSumsEveryPage() { + var client = mock(S3Client.class); + when(client.listObjectsV2(any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().isTruncated(true).nextContinuationToken("page-2") + .contents(S3Object.builder().key("prefix/a").size(7L).build()).build()) + .thenReturn(ListObjectsV2Response.builder().isTruncated(false) + .contents(S3Object.builder().key("prefix/b").size(11L).build()).build()); + var service = service(client); + + assertThat(service.listPrefixUsage("prefix/", 10)).contains(new StoragePrefixUsage(18, 2, true)); + var requests = ArgumentCaptor.forClass(ListObjectsV2Request.class); + verify(client, times(2)).listObjectsV2(requests.capture()); + assertThat(requests.getAllValues().get(0).continuationToken()).isNull(); + assertThat(requests.getAllValues().get(1).continuationToken()).isEqualTo("page-2"); + assertThat(requests.getAllValues().get(1).prefix()).isEqualTo("prefix/"); + } + + @Test + void truncatedResponseWithoutContinuationDoesNotClaimCompleteUsage() { + var client = mock(S3Client.class); + when(client.listObjectsV2(any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().isTruncated(true) + .contents(S3Object.builder().size(7L).build()).build()); + + assertThat(service(client).listPrefixUsage("prefix/", 10)).contains(new StoragePrefixUsage(7, 1, false)); + } + + @Test + void permissionFailureIsNotMisreportedAsMissingObject() { + var client = mock(S3Client.class); + when(client.headObject(any(HeadObjectRequest.class))).thenThrow(S3Exception.builder().statusCode(403).build()); + + assertThatThrownBy(() -> service(client).exists("private/key")) + .isInstanceOf(StorageAccessException.class).extracting("operation").isEqualTo("exists"); + } + + private S3StorageService service(S3Client client) { + var service = new S3StorageService(new S3StorageProperties()); + ReflectionTestUtils.setField(service, "s3Client", client); + return service; + } +} diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java index f828b071a..32d59cb7c 100644 --- a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java @@ -1,23 +1,28 @@ package com.iflytek.skillhub.storage; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; import software.amazon.awssdk.services.s3.model.CreateBucketResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; -import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; @@ -28,6 +33,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -304,6 +310,79 @@ class S3StorageServiceTest { assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); } + @Test + void putObjectShouldReplayFileStreamsForSigningAndRetry(@org.junit.jupiter.api.io.TempDir java.nio.file.Path directory) + throws Exception { + S3Client client = mock(S3Client.class); + byte[] content = new byte[256 * 1024 + 1]; + new java.util.Random(937).nextBytes(content); + var path = java.nio.file.Files.write(directory.resolve("package.zip"), content); + when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenAnswer(invocation -> { + RequestBody body = invocation.getArgument(1); + for (int read = 0; read < 2; read++) { + try (InputStream input = body.contentStreamProvider().newStream()) { + assertThat(input.readAllBytes()).isEqualTo(content); + } + } + return PutObjectResponse.builder().build(); + }); + var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class)); + service.init(); + + try (InputStream input = java.nio.file.Files.newInputStream(path)) { + assertThat(input.markSupported()).isFalse(); + service.putObject("packages/demo.zip", input, content.length, "application/zip"); + } + } + + @Test + void putObjectShouldRejectSizeMismatchBeforeCallingS3() { + S3Client client = mock(S3Client.class); + var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class)); + service.init(); + + assertThatThrownBy(() -> service.putObject("packages/demo.zip", new ByteArrayInputStream(new byte[]{42}), + 2, "application/zip")) + .isInstanceOf(StorageAccessException.class).extracting("operation").isEqualTo("putObject"); + verifyNoInteractions(client); + } + + @Test + void deleteObjectsShouldRejectPartialFailureEvenWhenRequestSucceeded() { + S3Client client = mock(S3Client.class); + when(client.deleteObjects(any(DeleteObjectsRequest.class))) + .thenReturn(DeleteObjectsResponse.builder() + .errors(S3Error.builder() + .key("private/denied.zip").code("AccessDenied").build()) + .build()); + var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class)); + service.init(); + + assertThatThrownBy(() -> service.deleteObjects(List.of("private/denied.zip"))) + .isInstanceOf(StorageAccessException.class) + .extracting("operation").isEqualTo("deleteObjects"); + } + + @Test + void deleteObjectsShouldPartitionRequestsAtS3Limit() { + S3Client client = mock(S3Client.class); + when(client.deleteObjects(any(DeleteObjectsRequest.class))) + .thenReturn(DeleteObjectsResponse.builder().build()); + var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class)); + service.init(); + var keys = IntStream.range(0, 1001).mapToObj(i -> "packages/" + i).toList(); + + service.deleteObjects(keys); + + var requests = ArgumentCaptor.forClass( + DeleteObjectsRequest.class); + verify(client, times(2)).deleteObjects(requests.capture()); + assertThat(requests.getAllValues().get(0).delete().objects()).hasSize(1000); + assertThat(requests.getAllValues().get(1).delete().objects()).hasSize(1); + assertThat(requests.getAllValues().stream().flatMap(r -> r.delete().objects().stream()) + .map(ObjectIdentifier::key).toList()).isEqualTo(keys); + } + private S3StorageProperties properties(boolean autoCreateBucket) { S3StorageProperties properties = createProperties(true); properties.setBucket("skillhub"); diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbe.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbe.java new file mode 100644 index 000000000..4cf6d7c4c --- /dev/null +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbe.java @@ -0,0 +1,147 @@ +package com.iflytek.skillhub.storage; + +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +/** Real object probe shared by every S3-compatible deployment. Never prints signed URLs. */ +final class StorageContractProbe { + private final ObjectStorageService storage; + private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + + StorageContractProbe(ObjectStorageService storage) { + this.storage = storage; + } + + void verify() throws Exception { + String prefix = "skillhub-diagnostics/" + UUID.randomUUID() + "/"; + System.out.println("Diagnostic object prefix: " + prefix); + List keys = new ArrayList<>(); + Exception failure = null; + try { + byte[] content = new byte[256 * 1024 + 1]; + new Random(937).nextBytes(content); + String key = prefix + "中文 空格+%?#/package.zip"; + keys.add(key); + var sourceFile = java.nio.file.Files.createTempFile("skillhub-contract-", ".zip"); + try { + java.nio.file.Files.write(sourceFile, content); + try (var source = java.nio.file.Files.newInputStream(sourceFile)) { + storage.putObject(key, source, content.length, "application/zip"); + } + } finally { + java.nio.file.Files.deleteIfExists(sourceFile); + } + require(storage.exists(key), "uploaded object missing"); + try (var input = storage.getObject(key)) { + require(Arrays.equals(input.readAllBytes(), content), "object bytes differ"); + } + ObjectMetadata metadata = storage.getMetadata(key); + require(metadata.size() == content.length, "object size differs"); + require("application/zip".equals(metadata.contentType()), "content type differs"); + String signedGet = storage.generatePresignedUrl(key, Duration.ofMinutes(1), "中文 package.zip"); + var download = send(HttpRequest.newBuilder(URI.create(signedGet)).GET()); + require(download.statusCode() == 200 && Arrays.equals(download.body(), content), "signed GET failed"); + require(download.headers().firstValue("Content-Disposition").orElse("").contains("filename*="), + "download filename missing"); + var tampered = send(HttpRequest.newBuilder(tamperSignature(signedGet)).GET()); + require(tampered.statusCode() == 403, "tampered signature was not rejected"); + var anonymous = send(HttpRequest.newBuilder(URI.create(signedGet.split("\\?")[0])).GET()); + require(anonymous.statusCode() == 403, "anonymous private object read was not rejected"); + String expiringGet = storage.generatePresignedUrl(key, Duration.ofSeconds(1), null); + TimeUnit.SECONDS.sleep(3); + require(send(HttpRequest.newBuilder(URI.create(expiringGet)).GET()).statusCode() == 403, + "expired signature was not rejected"); + String putKey = prefix + "staging.zip"; + keys.add(putKey); + PresignedUpload upload = storage.presignPut(putKey, Duration.ofMinutes(1)).orElseThrow(); + String signedHeaderNames = Arrays.stream(URI.create(upload.url()).getRawQuery().split("&")) + .filter(field -> field.startsWith("X-Amz-SignedHeaders=")) + .map(field -> URLDecoder.decode(field.substring(field.indexOf('=') + 1), + StandardCharsets.UTF_8)) + .findFirst().orElseThrow(); + System.out.println("PUT signature-bound header names: " + signedHeaderNames); + var request = HttpRequest.newBuilder(URI.create(upload.url())); + upload.headers().forEach(request::header); + // SDK request headers (e.g. Content-Type) need not all be bound into SigV4. + // Only require rejection for omitted headers listed in X-Amz-SignedHeaders. + if (Arrays.stream(signedHeaderNames.split(";")).anyMatch(name -> !name.equalsIgnoreCase("host"))) { + var missingHeaders = send(HttpRequest.newBuilder(URI.create(upload.url())) + .PUT(HttpRequest.BodyPublishers.ofByteArray(content))); + require(missingHeaders.statusCode() == 400 || missingHeaders.statusCode() == 403, + "missing signature-bound PUT headers were not rejected"); + require(!storage.exists(putKey), "rejected PUT wrote an object"); + System.out.println("Missing-header PUT rejected with HTTP " + missingHeaders.statusCode()); + } + var badPutRequest = HttpRequest.newBuilder(tamperSignature(upload.url())); + upload.headers().forEach(badPutRequest::header); + var badPut = send(badPutRequest.PUT(HttpRequest.BodyPublishers.ofByteArray(content))); + require(badPut.statusCode() == 403, "tampered PUT signature was not rejected"); + require(!storage.exists(putKey), "tampered PUT wrote an object"); + var uploaded = send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(content))); + require(uploaded.statusCode() >= 200 && uploaded.statusCode() < 300, "signed PUT failed"); + try (var input = storage.getObject(putKey)) { + require(Arrays.equals(input.readAllBytes(), content), "signed PUT bytes differ"); + } + require("application/zip".equals(storage.getMetadata(putKey).contentType()), + "signed PUT content type differs"); + var usage = storage.listPrefixUsage(prefix, 10).orElseThrow(); + require(usage.complete() && usage.objectCount() == 2 && usage.bytes() == content.length * 2L, + "prefix usage differs"); + var limited = storage.listPrefixUsage(prefix, 1).orElseThrow(); + require(!limited.complete() && limited.objectCount() == 1, "truncated usage reported complete"); + storage.deleteObject(putKey); + require(!storage.exists(putKey), "single delete failed"); + storage.deleteObjects(keys); + require(!storage.exists(key), "batch delete failed"); + } catch (Exception exception) { + failure = exception; + throw exception; + } finally { + // Delete only keys created by this invocation, even when a write timed out. + Exception cleanupFailure = null; + for (String key : keys) { + try { + storage.deleteObject(key); + } catch (Exception exception) { + if (cleanupFailure == null) cleanupFailure = new IOException("Diagnostic cleanup failed for " + prefix); + cleanupFailure.addSuppressed(exception); + } + } + if (cleanupFailure != null) { + if (failure != null) failure.addSuppressed(cleanupFailure); + else throw cleanupFailure; + } + } + } + + static URI tamperSignature(String signedUrl) { + String marker = "X-Amz-Signature="; + int start = signedUrl.indexOf(marker); + if (start < 0) throw new IllegalArgumentException("Expected a SigV4 query signature"); + start += marker.length(); + // Change the digest while retaining valid signature syntax and length. + char changed = signedUrl.charAt(start) == '0' ? '1' : '0'; + return URI.create(signedUrl.substring(0, start) + changed + signedUrl.substring(start + 1)); + } + + private HttpResponse send(HttpRequest.Builder request) throws Exception { + return http.send(request.timeout(Duration.ofSeconds(10)).build(), HttpResponse.BodyHandlers.ofByteArray()); + } + + private void require(boolean condition, String message) { + if (!condition) throw new IllegalStateException(message); + } +} diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbeTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbeTest.java new file mode 100644 index 000000000..3b8b024a6 --- /dev/null +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/StorageContractProbeTest.java @@ -0,0 +1,80 @@ +package com.iflytek.skillhub.storage; + +import java.io.ByteArrayInputStream; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.io.InputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class StorageContractProbeTest { + @Test + void signatureTamperingChangesOnlyDigestContentNotItsFormat() { + String signature = "a".repeat(64); + String url = "https://storage.example.test/bucket/key?X-Amz-Signature=" + signature + "&X-Amz-Expires=60"; + String tampered = StorageContractProbe.tamperSignature(url).toString(); + + assertThat(tampered.length()).isEqualTo(url.length()); + assertThat(tampered).startsWith("https://storage.example.test/bucket/key?X-Amz-Signature=") + .endsWith("&X-Amz-Expires=60"); + assertThat(IntStream.range(0, url.length()) + .filter(i -> url.charAt(i) != tampered.charAt(i)).count()).isEqualTo(1); + } + + @Test + void writeFailureStillCleansOnlyTheAttemptedObject() { + var storage = mock(ObjectStorageService.class); + var writeFailure = new StorageAccessException("putObject", "object", new IllegalStateException()); + doThrow(writeFailure).when(storage).putObject(anyString(), any(InputStream.class), anyLong(), anyString()); + + assertThatThrownBy(() -> new StorageContractProbe(storage).verify()).isSameAs(writeFailure); + + var key = ArgumentCaptor.forClass(String.class); + verify(storage).deleteObject(key.capture()); + assertThat(key.getValue()).startsWith("skillhub-diagnostics/").endsWith("package.zip"); + } + + @Test + void cleanupFailureIsNotLostWhenPrimaryOperationAlsoFails() { + var storage = mock(ObjectStorageService.class); + var writeFailure = new IllegalStateException("write failed"); + doThrow(writeFailure).when(storage).putObject(anyString(), any(InputStream.class), anyLong(), anyString()); + doThrow(new IllegalStateException("delete failed")).when(storage).deleteObject(anyString()); + + assertThatThrownBy(() -> new StorageContractProbe(storage).verify()).isSameAs(writeFailure); + assertThat(writeFailure.getSuppressed()).hasSize(1); + assertThat(writeFailure.getSuppressed()[0].getMessage()).startsWith("Diagnostic cleanup failed for skillhub-diagnostics/"); + } + + @Test + void probeDetectsCorruptedObjectBytes() { + var storage = mock(ObjectStorageService.class); + when(storage.exists(anyString())).thenReturn(true); + when(storage.getObject(anyString())).thenReturn(new ByteArrayInputStream(new byte[]{0})); + + assertThatThrownBy(() -> new StorageContractProbe(storage).verify()) + .isInstanceOf(IllegalStateException.class).hasMessage("object bytes differ"); + verify(storage).deleteObject(anyString()); + } + + @Test + void probeDetectsFalseWriteSuccess() { + var storage = mock(ObjectStorageService.class); + when(storage.exists(anyString())).thenReturn(false); + + assertThatThrownBy(() -> new StorageContractProbe(storage).verify()) + .isInstanceOf(IllegalStateException.class).hasMessage("uploaded object missing"); + verify(storage).deleteObject(anyString()); + } +} From 6743265a9008a13a74aabb099d281a92789d9157 Mon Sep 17 00:00:00 2001 From: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:02:39 +0800 Subject: [PATCH 2/4] fix(storage): preserve private staging and address deployment review Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> --- docs/object-storage.md | 4 +- .../design.md | 4 +- .../proposal.md | 2 +- .../specs/s3-compatible-storage/spec.md | 4 + .../tasks.md | 9 + scripts/storage-compose.sh | 24 +- scripts/tests/storage-compose-test.sh | 25 ++ .../skillhub/storage/S3StorageService.java | 16 +- .../storage/S3StorageStagingTest.java | 220 ++++++++++++++++++ 9 files changed, 296 insertions(+), 12 deletions(-) create mode 100644 server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageStagingTest.java diff --git a/docs/object-storage.md b/docs/object-storage.md index 516a1a7ed..9f7c9e7f7 100644 --- a/docs/object-storage.md +++ b/docs/object-storage.md @@ -106,7 +106,7 @@ scripts/storage-diagnostics.sh 交付验收时另运行 `scripts/storage-diagnostics.sh --full`，额外创建 1001 个小对象，检查真实翻页、统计上限与跨批次删除。满批删除如遇后端短暂逐项失败，适配层必须明确报错；验收会记录失败并最多尝试 3 次调用方恢复，最终独立确认对象已经不存在，不把 HTTP 200 当成全部删除成功；该模式最多运行 420 秒。普通诊断最多 180 秒，外层强制终止后可能需要按日志核查残留。 -S3 PUT 会在应用私有临时文件中暂存请求体，以支持文件流签名和重试；部署时应计入并发上传的临时磁盘预算，不能只预算对象存储的数据卷。临时文件在操作成功或失败后清理，不将整个技能包缓存到 JVM 堆中。 +S3 PUT 会在应用私有临时文件中暂存请求体，以支持文件流签名和重试；暂存写入保持原文件的私有权限（POSIX 环境为 0600），读取或上传失败时清理文件，不因清理失败掩盖原始错误；部署时应计入并发上传的临时磁盘预算，不能只预算对象存储的数据卷。临时文件在操作成功或失败后清理，不将整个技能包缓存到 JVM 堆中。 诊断不是应用 liveness，不要把这个有写入动作的命令接到公开健康检查。 @@ -217,7 +217,7 @@ mcli cat new/skillhub/OBJECT_KEY | sha256sum | 发布审核下载、头像待审/批准/移除、Suite | 通过 | 通过 | | Go Worker 内部端点 PUT、导入及暂存清理 | 通过 | 通过 | -RustFS 的满批临时错误不是“HTTP 200 即全部成功”；适配层会明确抛错，诊断的有界恢复只证明操作可幂等恢复，不能证明后端从不产生逐项失败。业务调用者应沿用现有补偿，不把诊断脚本的三次恢复误认为生产适配器自动重试。 +RustFS 的满批临时错误不是“HTTP 200 即全部成功”；适配层会明确抛错，诊断的有界恢复只证明操作可幂等恢复，不能证明后端从不产生逐项失败。只有已接入持久化补偿的删除链路才会由现有任务重试；部分草稿删除和 Suite 暂存清理仍为 best-effort，失败仅记录日志，可能留下对象，并不保证自动恢复。部署方应监测清理失败与残留，再按业务记录核查和精确清理，不能直接清空桶。不把诊断脚本的三次恢复误认为生产适配器自动重试。 制品核验采用 Trivy 0.76.0：RustFS 镜像系统包和官方发行 SBOM（1157 个组件）均未检出 High/Critical；Silo 镜像尚有 `CVE-2026-39414` 的 legacy MinIO 模块匹配。对这一条告警，已将镜像二进制的 commit `0cd388c666f8211f552604ae73448a066e9c25ce` 与发行标签源码绑定，并核对 CSV 有界读取修复；实际普通 2 MiB 与 gzip 解压后 2 MiB 的无换行记录均被拒绝且服务存活，本地分析裁定该特定漏洞路径已修补（依据：[上游公告](https://github.com/minio/minio/security/advisories/GHSA-h749-fxx7-pwpg)、[镜像对应提交的 CSV 读取实现](https://github.com/pgsty/silo/blob/0cd388c666f8211f552604ae73448a066e9c25ce/internal/s3select/csv/reader.go)）。没有全局忽略 CVE，也不据此声称整体镜像无风险。部署时应使用自己的当前漏洞库再次扫描。 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/design.md b/openspec/changes/add-s3-compatible-storage-deployments/design.md index 0e46abd74..c38419b82 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/design.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/design.md @@ -103,7 +103,7 @@ sequenceDiagram scripts/storage-compose.sh --storage rustfs --env-file .env.release --project-name skillhub -- up -d ``` -Silo 将最后一个文件改为 `compose.storage.silo.yml`；外部 S3 只使用基础 Compose。status、升级和 down 必须沿用相同 project、env 和文件组合，不删除数据卷作为更新步骤。 +Silo 将参数 `--storage rustfs` 改为 `--storage silo`；外部 S3 使用 `--storage external`，仅加载基础 Compose。status、升级和 down 必须沿用相同 project、env 和文件组合，不删除数据卷作为更新步骤。 启动顺序：存储健康 → 操作者或显式一次性初始化准备桶/业务账号 → 业务凭证诊断通过 → 启动应用 → 业务冒烟。基础 Compose 的本地存储卷和权限初始化暂不清理，避免扩大改动。 @@ -115,7 +115,7 @@ Helm 将存储作为独立服务管理。Chart 的安装、升级和卸载不承 实际发布通过 `Files.newInputStream` 提供不可 reset 的流。AWS SDK 2.20.26 的签名/重试会重读请求体，直接 `RequestBody.fromInputStream` 在此路径下会消耗流并使后续发送不完整；真实应用和文件流契约已复现超时，内存流测试不能覆盖此缺陷。 -复用适配层已有暂存 helper，将每次 PUT 的请求体暂存到私有临时文件，用 `RequestBody.fromFile` 提供可重放的源；不在内存缓存整个包。上传前校验实际字节数与声明大小，成功失败均释放暂存文件。接口与调用方保持不变，代价为每次 S3 PUT 增加一次临时磁盘写入，部署需计入并发上传的临时空间预算。 +复用适配层已有暂存 helper，将每次 PUT 的请求体暂存到私有临时文件，用 `RequestBody.fromFile` 提供可重放的源；不在内存缓存整个包。上传前校验实际字节数与声明大小，成功失败均释放暂存文件。暂存写入必须保留创建时的私有权限，不能通过替换文件重建为默认权限；POSIX 环境仅授予运行身份读写权限（0600）。读取发生 IOException 或运行时异常时，helper 必须清理尚未返回给调用方的文件；清理失败不得掩盖原始异常。接口与调用方保持不变，代价为每次 S3 PUT 增加一次临时磁盘写入，部署需计入并发上传的临时空间预算。 ### D6. 镜像、诊断与兼容性证据 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/proposal.md b/openspec/changes/add-s3-compatible-storage-deployments/proposal.md index c3855433d..99a89f54d 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/proposal.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/proposal.md @@ -24,4 +24,4 @@ 主要影响 `server/skillhub-storage`、Compose、配置校验、镜像清单、Helm 示例和部署手册。验证发布下载、头像审核对象、Suite 包和 GitHub 同步调用链；业务模块不引入供应商 SDK。 -不改数据库、公开 API、技能生命周期或权限策略，不增加后台存储配置页面。本轮仅交付设计，代码、镜像、部署和兼容性证明待实施。 +不改数据库、公开 API、技能生命周期或权限策略，不增加后台存储配置页面。实现与本地验收状态见 tasks.md；最终提交、CI 和发布门禁由对应 PR 记录。 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md b/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md index bfb3d70d4..42c658e43 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/specs/s3-compatible-storage/spec.md @@ -26,6 +26,10 @@ - **WHEN** 上传的合法技能包包含超过 128 KiB 的文件并通过现有流式发布处理 - **THEN** 写入和下载字节与原文件一致，不因请求体重读而丢失内容或持续超时 +#### Scenario: 私有上传暂存和失败清理 +- **WHEN** S3 上传为签名和重试暂存请求体，或在暂存读取期间发生 I/O 或运行时异常 +- **THEN** 暂存期间不放宽文件创建时的私有访问权限，在支持 POSIX 权限的平台保持仅运行身份读写；读取失败后清理尚未返回的暂存文件，并保留原始失败供调用方识别 + #### Scenario: 批量删除部分失败 - **WHEN** 后端在批量删除响应中报告部分对象失败 - **THEN** 调用方能够识别失败，不得到全部删除成功的结论 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/tasks.md b/openspec/changes/add-s3-compatible-storage-deployments/tasks.md index 08fb6870d..d69d2ed6f 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/tasks.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/tasks.md @@ -40,3 +40,12 @@ - 新 Silo 镜像的单条 legacy MinIO 模块告警按源码 commit、二进制版本和真实有界 CSV 拒绝用例裁定为已修补，保留原始扫描结果，不全局忽略告警。 - 远端镜像同步、真实 K8s、外网 TLS/代理未执行；本地 JAR 构建加官方 Dockerfile runtime 阶段镜像已验证，不宣称完整多阶段 Docker 构建或 PR CI 通过。本地验收记录截止时尚未提交、推送或创建 PR；最终提交、PR 与 CI 状态由后续交付流程记录，不将本地结果视为合并门禁通过。 - 本地脱敏验收记录单独交付，不将运行日志、凭证、私有路径或工具状态纳入 Git。 + +## 6. PR #996 深度 Review 跟进 + +- [x] 6.1 修复 R1/R2：暂存期间保持私有权限，读取 I/O／运行时异常可靠清理并保留主异常；新增测试先在旧实现失败，再验证修复通过。 +- [x] 6.2 修复 R3：允许 profile 前置参数下的 logs -f，仍拒绝全局额外文件注入，并覆盖代表性合法／非法参数。 +- [x] 6.3 修复 R4/R5：更新 Proposal 状态边界及 Silo 包装器命令，明确不具备持久化补偿的 best-effort 清理限制。 +- [x] 6.4 运行修复后的相关回归、实际存储及受影响上传验证，独立复核修复并整理交付摘要；推送后在 PR 评论汇总结果和未验证边界，不自动合并。 + +本轮修复验证：新增 9 项暂存权限与清理测试，旧实现先复现 4 项失败；修复后 106 项回归通过（普通运行另有 2 个 opt-in 集成跳过，两产品已分别显式运行各 2 项通过）。实际 Java 21 非 root 运行环境验证了暂存期间 0600 与运行时异常后无残留；重建应用后认证 18/18、303010 字节技能包发布审核下载和头像流程通过。profile 日志跟随与额外文件拒绝回归通过。迁移／高可用等未受本轮修改影响的范围没有重新扩大验证声明。 diff --git a/scripts/storage-compose.sh b/scripts/storage-compose.sh index 3580308d8..dc92ef0c5 100755 --- a/scripts/storage-compose.sh +++ b/scripts/storage-compose.sh @@ -28,13 +28,31 @@ done case "$storage" in rustfs|silo|external) ;; *) fail "Select rustfs, silo or external" ;; esac (($#)) || fail "Missing Compose command" # Do not allow additional global files to silently merge incompatible storage services. +compose_command="" +global_value_pending=false for arg in "$@"; do - # In logs, -f means follow rather than a global Compose file option. - [[ "$arg" != -f || "${1:-}" != logs ]] || continue + # Only standalone -f after the actual logs subcommand means follow. Global + # option values (for example --profile logs) must not be mistaken for it. case "$arg" in - -f|--file|-f?*|--file=*|--env-file|--env-file=*|--project-directory|--project-directory=*) + -f) + [[ "$compose_command" == logs && "$global_value_pending" == false ]] || + fail "Compose file and env selection must use this script's options" + continue ;; + --file|-f?*|--file=*|--env-file|--env-file=*|--project-directory|--project-directory=*) fail "Compose file and env selection must use this script's options" ;; esac + [[ -z "$compose_command" ]] || continue + if [[ "$global_value_pending" == true ]]; then + global_value_pending=false + continue + fi + # These Compose global options consume the next argument; their =value forms + # are already single tokens. Leave command-specific options to Compose. + case "$arg" in + --ansi|--parallel|--profile|--progress|-p|--project-name) global_value_pending=true ;; + -*) ;; + *) compose_command="$arg" ;; + esac done cd "$REPO_ROOT" [[ -f "$env_file" ]] || fail "Env file not found" diff --git a/scripts/tests/storage-compose-test.sh b/scripts/tests/storage-compose-test.sh index fd1530d40..2d4044143 100755 --- a/scripts/tests/storage-compose-test.sh +++ b/scripts/tests/storage-compose-test.sh @@ -22,12 +22,37 @@ run --storage external -- --profile github-sync config [[ "$(cat "$work/args")" != *compose.storage.* ]] || fail "External storage started a local backend" run --storage rustfs -- logs -f [[ "$(cat "$work/args")" == *logs* ]] || fail "Log command not forwarded" +run --storage rustfs -- --profile github-sync logs -f github-sync +[[ "$(cat "$work/args")" == *$'--profile\ngithub-sync\nlogs\n-f\ngithub-sync' ]] || fail "Profile log follow command not forwarded" +run --storage rustfs -- --profile github-sync logs --follow github-sync +[[ "$(cat "$work/args")" == *$'logs\n--follow\ngithub-sync' ]] || fail "Long log follow command not forwarded" +run --storage rustfs -- --ansi never --profile=github-sync --parallel 2 --progress=plain logs -f github-sync +[[ "$(cat "$work/args")" == *$'logs\n-f\ngithub-sync' ]] || fail "Global options obscured log command" for args in 'rustfs --storage silo' 'unknown'; do # Intentional word splitting for these fixed malformed input fixtures. if run --storage $args -- config >/dev/null 2>&1; then fail "Accepted invalid selection"; fi done if run --storage rustfs -- -f compose.storage.silo.yml config >/dev/null 2>&1; then fail "Accepted file injection"; fi if run --storage rustfs -- --file=compose.storage.silo.yml config >/dev/null 2>&1; then fail "Accepted file injection"; fi +reject_compose_args() { + rm -f "$work/args" + if run --storage rustfs -- "$@" >"$work/rejected" 2>&1; then + fail "Accepted forbidden Compose selection: $*" + fi + [[ ! -e "$work/args" ]] || fail "Forwarded forbidden selection to Docker" +} +reject_compose_args --profile github-sync -f compose.storage.silo.yml config +reject_compose_args --profile github-sync -fcompose.storage.silo.yml config +reject_compose_args --profile github-sync --file=compose.storage.silo.yml config +reject_compose_args --profile github-sync --file compose.storage.silo.yml config +reject_compose_args --profile logs -f compose.storage.silo.yml config +reject_compose_args --profile=logs -f compose.storage.silo.yml config +reject_compose_args --profile github-sync logs -fcompose.storage.silo.yml +reject_compose_args --profile github-sync logs --file=compose.storage.silo.yml +reject_compose_args --profile github-sync logs --env-file "$work/env" +reject_compose_args --profile github-sync logs --env-file="$work/env" +reject_compose_args --profile github-sync logs --project-directory "$work" +reject_compose_args --profile github-sync logs --project-directory="$work" if run --storage rustfs >/dev/null 2>&1; then fail "Accepted missing command"; fi if run --env-file "$work/missing" --storage rustfs -- config >/dev/null 2>&1; then fail "Accepted missing env"; fi echo "Storage Compose selection tests passed" diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java index cc45eb055..77a2444f6 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -28,7 +28,6 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.List; import java.util.Map; @@ -176,10 +175,19 @@ public class S3StorageService implements ObjectStorageService { private Path stagePutObjectBody(InputStream data) throws IOException { Path stagedBody = Files.createTempFile("skillhub-s3-upload-", ".tmp"); try { - Files.copy(data, stagedBody, StandardCopyOption.REPLACE_EXISTING); + // Write into the existing private file; replacing it can widen its permissions. + try (var output = Files.newOutputStream(stagedBody)) { + data.transferTo(output); + } return stagedBody; - } catch (IOException e) { - Files.deleteIfExists(stagedBody); + } catch (IOException | RuntimeException | Error e) { + try { + Files.deleteIfExists(stagedBody); + } catch (IOException | RuntimeException | Error cleanupFailure) { + if (cleanupFailure != e) { + e.addSuppressed(cleanupFailure); + } + } throw e; } } diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageStagingTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageStagingTest.java new file mode 100644 index 000000000..00ebe0af8 --- /dev/null +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageStagingTest.java @@ -0,0 +1,220 @@ +package com.iflytek.skillhub.storage; + +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class S3StorageStagingTest { + @Test + void stagedBodyRemainsOwnerOnlyDuringCopyAndUpload() throws Exception { + Path probe = Files.createTempFile("skillhub-s3-permission-probe-", ".tmp"); + try { + assumeTrue(Files.getFileStore(probe).supportsFileAttributeView("posix"), + "Owner-only POSIX permissions require a POSIX file system"); + } finally { + Files.deleteIfExists(probe); + } + var staged = new AtomicReference(); + S3Client client = mock(S3Client.class); + S3StorageService service = service(client); + try (var files = trackStagedBody(staged)) { + when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenAnswer(invocation -> { + assertOwnerOnly(staged.get()); + return PutObjectResponse.builder().build(); + }); + InputStream input = new InputStream() { + private boolean consumed; + + @Override + public int read() throws IOException { + assertOwnerOnly(staged.get()); + if (consumed) return -1; + consumed = true; + return 42; + } + }; + service.putObject("private/package.zip", input, 1, "application/zip"); + assertThat(staged.get()).doesNotExist(); + } finally { + if (staged.get() != null) Files.deleteIfExists(staged.get()); + } + } + + @Test + void stagingIOExceptionRemovesPartialFile() throws Exception { + assertStagingFailureCleaned(new IOException("read failed")); + } + + @Test + void stagingRuntimeExceptionRemovesPartialFile() throws Exception { + assertStagingFailureCleaned(new IllegalStateException("read failed")); + } + + @Test + void stagingErrorRemovesPartialFile() throws Exception { + assertStagingFailureCleaned(new AssertionError("read failed")); + } + + @Test + void stagingCleanupFailureDoesNotReplaceReadFailure() throws Exception { + assertCleanupFailureSuppressed(new IOException("read failed"), new IOException("delete failed")); + } + + @Test + void stagingCleanupFailureDoesNotReplaceRuntimeFailure() throws Exception { + assertCleanupFailureSuppressed(new IllegalStateException("read failed"), new IOException("delete failed")); + } + + @Test + void stagingRuntimeCleanupFailureDoesNotReplaceReadFailure() throws Exception { + assertCleanupFailureSuppressed(new IOException("read failed"), new IllegalStateException("delete failed")); + } + + private void assertCleanupFailureSuppressed(Exception primary, Exception cleanup) throws Exception { + var staged = new AtomicReference(); + S3Client client = mock(S3Client.class); + S3StorageService service = service(client); + try (var files = trackStagedBody(staged)) { + InputStream input = new InputStream() { + @Override + public int read() throws IOException { + files.when(() -> Files.deleteIfExists(staged.get())).thenThrow(cleanup); + if (primary instanceof IOException io) throw io; + throw (RuntimeException) primary; + } + }; + assertThatThrownBy(() -> service.putObject("private/package.zip", input, 1, "application/zip")) + .isInstanceOf(StorageAccessException.class).hasCause(primary); + assertThat(primary.getSuppressed()).containsExactly(cleanup); + verifyNoInteractions(client); + } finally { + if (staged.get() != null) Files.deleteIfExists(staged.get()); + } + } + + @Test + void s3FailureRemovesStagedFile() throws Exception { + var staged = new AtomicReference(); + S3Client client = mock(S3Client.class); + S3StorageService service = service(client); + RuntimeException failure = new IllegalStateException("S3 unavailable"); + when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenThrow(failure); + try (var files = trackStagedBody(staged)) { + assertThatThrownBy(() -> service.putObject("private/package.zip", + new ByteArrayInputStream(new byte[]{42}), 1, "application/zip")) + .isInstanceOf(StorageAccessException.class).hasCause(failure); + assertThat(staged.get()).doesNotExist(); + } + } + + @Test + void sizeMismatchRemovesStagedFileBeforeS3Call() throws Exception { + var staged = new AtomicReference(); + S3Client client = mock(S3Client.class); + S3StorageService service = service(client); + try (var files = trackStagedBody(staged)) { + assertThatThrownBy(() -> service.putObject("private/package.zip", + new ByteArrayInputStream(new byte[]{42}), 2, "application/zip")) + .isInstanceOf(StorageAccessException.class).hasCauseInstanceOf(IllegalArgumentException.class); + assertThat(staged.get()).doesNotExist(); + verifyNoInteractions(client); + } + } + + private void assertStagingFailureCleaned(Throwable failure) throws Exception { + var staged = new AtomicReference(); + S3Client client = mock(S3Client.class); + S3StorageService service = service(client); + InputStream input = new InputStream() { + private boolean partialWritten; + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (!partialWritten) { + partialWritten = true; + buffer[offset] = 42; + return 1; + } + if (failure instanceof IOException io) throw io; + if (failure instanceof RuntimeException runtime) throw runtime; + throw (Error) failure; + } + + @Override + public int read() { + throw new AssertionError("bulk read expected"); + } + }; + try (var files = trackStagedBody(staged)) { + if (failure instanceof Error) { + assertThatThrownBy(() -> service.putObject("private/package.zip", input, 2, "application/zip")) + .isSameAs(failure); + } else { + assertThatThrownBy(() -> service.putObject("private/package.zip", input, 2, "application/zip")) + .isInstanceOf(StorageAccessException.class).hasCause(failure); + } + assertThat(staged.get()).doesNotExist(); + verifyNoInteractions(client); + } finally { + if (staged.get() != null) Files.deleteIfExists(staged.get()); + } + } + + private MockedStatic trackStagedBody(AtomicReference staged) throws IOException { + MockedStatic files = mockStatic(Files.class, CALLS_REAL_METHODS); + files.when(() -> Files.createTempFile("skillhub-s3-upload-", ".tmp")).thenAnswer(invocation -> { + Path path = (Path) invocation.callRealMethod(); + staged.set(path); + return path; + }); + return files; + } + + private void assertOwnerOnly(Path path) throws IOException { + assertThat(Files.getPosixFilePermissions(path)).containsExactlyInAnyOrder( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); + } + + private S3StorageService service(S3Client client) { + var properties = new S3StorageProperties(); + properties.setBucket("private-test-bucket"); + properties.setAutoCreateBucket(false); + var presigner = mock(S3Presigner.class); + var service = new S3StorageService(properties) { + @Override + protected S3Client buildS3Client(ApacheHttpClient.Builder builder) { + return client; + } + + @Override + S3Presigner buildPresigner() { + return presigner; + } + }; + service.init(); + return service; + } +} From 89be3016b97153706e0e14b2f0df97459c0e9766 Mon Sep 17 00:00:00 2001 From: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:25:04 +0800 Subject: [PATCH 3/4] docs(storage): bound deployment and migration claims Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> --- docs/object-storage.md | 16 ++++++++++------ .../design.md | 12 +++++++----- .../proposal.md | 4 ++-- .../specs/object-storage-deployment/spec.md | 11 ++++++----- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/docs/object-storage.md b/docs/object-storage.md index 9f7c9e7f7..1ff940943 100644 --- a/docs/object-storage.md +++ b/docs/object-storage.md @@ -1,6 +1,6 @@ -# 对象存储部署与迁移 +# 对象存储部署与迁移参考 -SkillHub 通过同一个 S3 接口连接 RustFS、PGSTY Silo 或外部 S3 服务，不需要按存储品牌修改业务代码。`local` 模式保留。本文新增的单机叠加配置不是高可用存储集群；正式支持范围以本文末尾的验证矩阵为准。 +SkillHub 通过同一个 S3 接口连接 RustFS、PGSTY Silo 或外部 S3 服务，不需要按存储品牌修改业务代码。`local` 模式保留。本文新增的单机叠加配置不是高可用存储集群；本次验证范围是文末所列指定版本的新部署接入与业务操作，不包含生产数据迁移支持。 ## 1. 选择部署方式 @@ -13,6 +13,8 @@ SkillHub 通过同一个 S3 接口连接 RustFS、PGSTY Silo 或外部 S3 服务 一次只选择一个本地存储。使用 `scripts/storage-compose.sh`，它会拒绝重复选择和额外 `-f` 文件；不要自行同时合并两个存储文件。所有命令在仓库根目录执行，要求 Bash、Docker Compose v2；诊断另外要求 JDK 21+、Maven Wrapper 和 GNU timeout。 +RustFS 与 Silo 是并列可选项，RustFS 不是全局默认值，也不代表最优推荐；下文先用它举例不代表优先选用。`.env.release.example` 仍显式选择 `local`，开发 Compose 的 `minio` 服务仍使用原有 Silo 镜像；仅使用基础 `compose.release.yml` 时，provider 的缺省值为 `s3`，但不会自动部署任一存储产品。已有实例不会自动切换。本次没有长期稳定性、大规模负载或高可用对比证据，部署方应按自身运维条件选择并验收。 + ## 2. 准备受保护配置 从 `.env.release.example` 准备 `.env.release`，再把 `deploy/storage/rustfs.env.example` 或 `deploy/storage/silo.env.example` 中的存储配置合入该文件，并替换所有占位值。不要直接用示例部署，也不要把实际配置加入 Git。 @@ -161,7 +163,9 @@ scripts/storage-compose.sh --storage rustfs --env-file .env.release \ `deploy/runtime-mirror-images.txt` 保留原有 `minio:latest` 别名。新产品采用独立版本标签；已有镜像目标会被同步脚本跳过，修改映射清单不等于更新仓库内容。实际同步属于单独的发布操作，验收应比较目标 manifest digest。 -## 7. 从已有后端迁移 +## 7. 已有数据迁移的风险与操作参考 + +本节不宣称生产迁移支持，也不提供自动迁移功能。现有证据仅包括独立测试桶中 RustFS → Silo → RustFS 的小样本演练；未验证旧 MinIO 或 local 到新后端的真实迁移，也未完成大规模生产数据迁移、恢复及停机窗口验收。下列步骤是安全参考，不能作为可直接执行的生产迁移方案；具体实例需要另行清点数据、制定方案并完成代表性演练。 修改 endpoint/provider 不会迁移文件。RustFS、Silo 和 local 不得默认共用原始数据目录。 @@ -173,9 +177,9 @@ scripts/storage-compose.sh --storage rustfs --env-file .env.release \ 6. 切换配置、重建受影响服务，检查下载原始字节、头像、Suite 和同步导入/清理，再恢复写入。 7. 恢复写入前可以回退配置到未变的旧源；恢复写入后，必须重新停写并协调新后端的新增、覆盖、删除差异后再回切，防止丢失新文件或恢复已删除内容。 -### 已验证的逻辑复制方式 +### 小样本逻辑复制演练参考 -本次用上述版本的 `mcli` 在独立测试桶演练 RustFS → Silo → RustFS，包含在线初始复制、冻结后的新增／覆盖／删除、在途 PUT 过期，以及切换后产生新数据再回退。`mirror` 会保存本次验证对象的 Content-Type 和自定义元数据；版本历史、对象锁、加密和存储策略仍需部署方分别评估。 +本次用上述版本的 `mcli` 在独立测试桶进行 RustFS → Silo → RustFS 小样本演练（正向清单 134 个对象，回退清单 135 个对象），包含在线初始复制、冻结后的新增／覆盖／删除、在途 PUT 过期，以及切换后产生新数据再回退。`mirror` 会保存本次验证对象的 Content-Type 和自定义元数据；版本历史、对象锁、加密和存储策略仍需部署方分别评估。 下面的 `old`、`new` 是操作者预先配置的管理客户端别名，不是内置端点。先核对方向、桶名和私有权限，再执行；`--remove` 会删除目标中源端不存在的对象，只能在已批准的目标桶范围操作。不能把它指向尚在写入的旧源或其他业务桶。 @@ -208,7 +212,7 @@ mcli cat new/skillhub/OBJECT_KEY | sha256sum 两者的官方 manifest 均包含 linux/amd64 和 linux/arm64；本机运行结果只证明实际执行的架构。新增 Silo 叠加文件选用上述版本；已有开发服务及 `minio:latest` 别名保持原版本，不自动升级已有实例。 -本次在 linux/amd64 上使用同一应用构件完成两种产品的真实契约验收；不代表任意 S3 产品、任意版本、arm64 运行或高可用部署已获验证。 +本次在 linux/amd64 上使用同一应用构件完成两种产品的新部署真实契约验收；不代表任意 S3 产品、任意版本、arm64 运行、长期稳定性、大规模负载、高可用部署或生产迁移已获验证。下表只列接入和业务操作结果，不是存储产品优劣排名或生产迁移支持矩阵。 | 范围 | RustFS 1.0.3 | Silo 2026-10-18 | |---|---|---| diff --git a/openspec/changes/add-s3-compatible-storage-deployments/design.md b/openspec/changes/add-s3-compatible-storage-deployments/design.md index c38419b82..3a39f904a 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/design.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/design.md @@ -18,9 +18,9 @@ S3 当前客户端初始化不等于存储可用性验证。既有单元测试 ## Goals / Non-Goals -**Goals:** 以统一 S3 协议保持业务代码稳定，交付可选存储运行方式、可重复验收及迁移指南。 +**Goals:** 以统一 S3 协议保持业务代码稳定，交付可选存储运行方式、可重复的新部署接入验收及迁移风险参考。 -**Non-Goals:** 不引入 `rustfs` / `silo` provider、业务模块专用 SDK、插件加载框架、多后端路由、双写、热切换、数据库迁移或后台配置页面。不将本次单机示例包装成高可用集群方案，不自动升级已有 Silo 部署。 +**Non-Goals:** 不引入 `rustfs` / `silo` provider、业务模块专用 SDK、插件加载框架、多后端路由、双写、热切换、数据库迁移或后台配置页面。不将本次单机示例包装成高可用集群方案，不自动升级已有 Silo 部署。不提供自动迁移或宣称生产迁移支持，旧 MinIO/local 真实迁移及大规模生产迁移尚未验收。 ## Decisions @@ -43,7 +43,7 @@ flowchart TB ### D2. 技术配置：复用已有键，部署时确定后端 -下表全部是已有配置键。此处是新本地存储示例的目标取值，不覆盖全局默认值。 +下表全部是已有配置键。此处是新本地存储示例的目标取值，不覆盖全局默认值。RustFS 与 Silo 并列可选；RustFS 不是全局默认或最优推荐。`.env.release.example` 保持 local，开发 Compose 保持原有 Silo，基础发布 Compose 的 provider 缺省为 s3 且不捆绑存储服务。本次没有长期稳定性、大规模负载或 HA 对比证据。 | 环境变量 | 示例或要求 | |---|---| @@ -135,7 +135,9 @@ Helm 将存储作为独立服务管理。Chart 的安装、升级和卸载不承 - [单机存储失败导致不可用] → 明确持久卷、备份恢复和容量监控，不将本方案宣称为 HA。 - [更多配置增加维护成本] → 不改核心业务接口，使用共享测试与单一协议操作手册，产品特有内容限于部署附录。 -## Migration Plan +## Migration Reference and Boundaries + +以下保留迁移安全要求，作为操作参考，不构成已支持的生产迁移方案。已完成的证据仅为独立测试桶中 RustFS → Silo → RustFS 的小样本演练（正向 134 个对象、回退 135 个对象）；没有旧 MinIO/local 真实迁移或大规模生产迁移验收。生产实例需要另行设计和演练，不能用新部署接入通过或小样本结果替代。 1. 清点现有 backend、桶、对象键、元数据和加密方式；备份对象及部署配置，演练恢复。原始数据卷不能直接挂给 RustFS；Silo 即使声称格式兼容，也不默认做原地切换。 2. 准备独立目标桶和业务账号，在专用前缀跑契约探针；逻辑复制对象，保留 key、Content-Type 及必要元数据。跨加密后端通过授权逻辑读取再写入，不能拷贝内部加密文件。 @@ -170,7 +172,7 @@ Helm 将存储作为独立服务管理。Chart 的安装、升级和卸载不承 以下项目不改变架构与行为契约，但进入交付前必须补齐证据： - 已按官方 manifest 固定 RustFS 1.0.3、Silo RELEASE.2026-10-18T00-00-00Z（含 amd64/arm64），清单见 `docs/object-storage.md`。实际 amd64 镜像与初始化权限、扫描结果已核验，Silo 单条 legacy 模块告警的源码和行为裁定见用户手册；不把 manifest 核验扩大为 arm64 运行支持。 -- 迁移工具在所选产品组合下的元数据与删除差异处理能力；先小样本演练，再补实际操作指南。 +- 迁移参考已记录小样本演练；实际旧 MinIO/local 来源、数据规模、元数据、版本历史、锁及加密场景没有生产迁移证据，不列入本次接入支持范围。 ## 参考与证据边界 diff --git a/openspec/changes/add-s3-compatible-storage-deployments/proposal.md b/openspec/changes/add-s3-compatible-storage-deployments/proposal.md index 99a89f54d..c97a07a16 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/proposal.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/proposal.md @@ -7,14 +7,14 @@ - 保持 `local` / `s3` 配置契约，通过同一 S3 接口接入 RustFS、PGSTY Silo 或外部 S3 服务。 - 为正式 Compose 栈增加可选 RustFS / Silo 叠加文件，使用独立数据卷，不静默替换现有部署。 - 明确内外网端点、预签名、业务凭证、桶初始化、GitHub 同步 Worker 网络和 Helm 外部存储配置。 -- 增加共享真实契约验收、迁移停写和回退指南，以及镜像供应链说明。 +- 增加共享真实契约验收、迁移风险及停写和回退参考，以及镜像供应链说明；新部署接入验证不构成生产迁移支持。 ## Capabilities ### New Capabilities - `s3-compatible-storage`: 不同 S3 实现下的对象读写、签名、权限和错误行为契约。 -- `object-storage-deployment`: 可选部署、配置诊断、兼容性声明与安全迁移交付契约。 +- `object-storage-deployment`: 可选部署、配置诊断、兼容性声明与迁移参考的安全边界。 ### Modified Capabilities diff --git a/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md b/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md index dd0e88f2a..7fbcd848f 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/specs/object-storage-deployment/spec.md @@ -1,6 +1,6 @@ ## Purpose -为部署者提供可选的 S3 兼容存储交付方式、配置检查及可追溯的验收依据，并明确已有对象迁移的安全要求。新增选项不应使原有部署被自动替换，也不能把配置更新误认为数据已完成迁移。 +为部署者提供可选的 S3 兼容存储交付方式、配置检查及可追溯的验收依据，并明确已有对象迁移参考的安全要求。新部署接入验证不代表支持生产迁移。新增选项不应使原有部署被自动替换，也不能把配置更新误认为数据已完成迁移。 ## ADDED Requirements @@ -29,16 +29,17 @@ ### Requirement: DEP-03 可验证的交付与兼容声明 交付 SHALL 记录应用修订、被测产品版本、不可变镜像摘要、架构和验收结果。配置检查与真实对象诊断必须区分，服务存活不能替代真实操作。旧镜像别名 SHALL 保持兼容，映射文件修改不能视为目标镜像已刷新。 -#### Scenario: 发布支持矩阵 -- **WHEN** 宣布某个产品版本受支持 +#### Scenario: 发布接入验证矩阵 +- **WHEN** 宣布某个产品版本通过新部署接入验证 - **THEN** 共享契约和关键业务路径有真实结果，未执行项及限制被列出，阻断用例失败时不能声明通过 +- **AND** 不将接入结果扩大为产品最优推荐、长期稳定性、大规模、高可用或生产迁移支持 #### Scenario: 运行诊断 - **WHEN** 操作者执行存储诊断 - **THEN** 在专用前缀进行有时限的真实对象操作并清理，失败非零退出，不泄露密钥或触碰业务对象 -### Requirement: DEP-04 逻辑迁移与回退 -迁移指南 SHALL 要求保留对象键和必要元数据、校验内容、暂停相关写入并完成最终同步后切换。不同产品不得默认复用原始数据卷；切换配置不代表迁移完成。 +### Requirement: DEP-04 逻辑迁移与回退参考 +迁移参考 SHALL 明确本次不宣称生产迁移支持；仅有独立测试桶小样本演练，未验证旧 MinIO/local 真实迁移或大规模生产迁移。迁移参考 SHALL 要求保留对象键和必要元数据、校验内容、暂停相关写入并完成最终同步后切换。不同产品不得默认复用原始数据卷；切换配置不代表迁移完成。 #### Scenario: 已有数据切换 - **WHEN** 实例从现有存储迁往另一产品 From 87edd52bc20361272cbf3dc75ec46e877db1c303 Mon Sep 17 00:00:00 2001 From: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:49:36 +0800 Subject: [PATCH 4/4] docs(storage): lead self-hosted S3 examples with Silo Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com> --- docs/object-storage.md | 20 +++++++++---------- .../design.md | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/object-storage.md b/docs/object-storage.md index 1ff940943..f45db3d38 100644 --- a/docs/object-storage.md +++ b/docs/object-storage.md @@ -6,18 +6,18 @@ SkillHub 通过同一个 S3 接口连接 RustFS、PGSTY Silo 或外部 S3 服务 | 方式 | 使用配置 | 数据归属 | |---|---|---| -| RustFS | `compose.release.yml` + `compose.storage.rustfs.yml` | 独立 `rustfs_data` 卷 | | Silo | `compose.release.yml` + `compose.storage.silo.yml` | 独立 `silo_data` 卷 | +| RustFS | `compose.release.yml` + `compose.storage.rustfs.yml` | 独立 `rustfs_data` 卷 | | 外部 S3 | 仅 `compose.release.yml` | 外部服务的桶 | | local | 仅 `compose.release.yml`，显式 `SKILLHUB_STORAGE_PROVIDER=local` | 现有应用存储卷 | 一次只选择一个本地存储。使用 `scripts/storage-compose.sh`，它会拒绝重复选择和额外 `-f` 文件；不要自行同时合并两个存储文件。所有命令在仓库根目录执行，要求 Bash、Docker Compose v2；诊断另外要求 JDK 21+、Maven Wrapper 和 GNU timeout。 -RustFS 与 Silo 是并列可选项，RustFS 不是全局默认值，也不代表最优推荐；下文先用它举例不代表优先选用。`.env.release.example` 仍显式选择 `local`，开发 Compose 的 `minio` 服务仍使用原有 Silo 镜像；仅使用基础 `compose.release.yml` 时，provider 的缺省值为 `s3`，但不会自动部署任一存储产品。已有实例不会自动切换。本次没有长期稳定性、大规模负载或高可用对比证据，部署方应按自身运维条件选择并验收。 +本文优先以 Silo 展示自建 S3 的操作步骤，RustFS 保留可选。这一顺序基于本项目当前接入验收及既有部署方式，不是通用可靠性排名，也不代表已验证 Silo 的长期生产可靠性。`.env.release.example` 仍显式选择 `local`，开发 Compose 的 `minio` 服务仍使用原有 Silo 镜像；仅使用基础 `compose.release.yml` 时，provider 的缺省值为 `s3`，但不会自动部署任一存储产品。已有实例不会自动切换。本次没有长期稳定性、大规模负载或高可用对比证据，部署方应按自身运维条件选择并验收。 ## 2. 准备受保护配置 -从 `.env.release.example` 准备 `.env.release`，再把 `deploy/storage/rustfs.env.example` 或 `deploy/storage/silo.env.example` 中的存储配置合入该文件，并替换所有占位值。不要直接用示例部署，也不要把实际配置加入 Git。 +从 `.env.release.example` 准备 `.env.release`，再把 `deploy/storage/silo.env.example`（RustFS 使用 `deploy/storage/rustfs.env.example`）中的存储配置合入该文件，并替换所有占位值。不要直接用示例部署，也不要把实际配置加入 Git。 ```sh # 已有 .env.release 时不要重复复制覆盖 @@ -39,12 +39,12 @@ chmod 600 .env.release ## 3. 分步初始化，避免应用先于桶启动 -下面用 RustFS，Silo 把每条命令的 `--storage rustfs` 改为 `--storage silo`。固定 `--project-name`，后续更新、状态和停机也使用相同参数。 +下面用 Silo；选择 RustFS 时，改用其 env 示例，并把下文每条命令的 `--storage silo` 改为 `--storage rustfs`。固定 `--project-name`，后续更新、状态和停机也使用相同参数。 ```sh -scripts/storage-compose.sh --storage rustfs --env-file .env.release \ +scripts/storage-compose.sh --storage silo --env-file .env.release \ --project-name skillhub -- up -d object-storage -scripts/storage-compose.sh --storage rustfs --env-file .env.release \ +scripts/storage-compose.sh --storage silo --env-file .env.release \ --project-name skillhub -- ps object-storage ``` @@ -114,7 +114,7 @@ S3 PUT 会在应用私有临时文件中暂存请求体，以支持文件流签 ```sh make validate-release-config -scripts/storage-compose.sh --storage rustfs --env-file .env.release \ +scripts/storage-compose.sh --storage silo --env-file .env.release \ --project-name skillhub -- up -d ``` @@ -132,7 +132,7 @@ SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824 允许列表按完整主机及端口匹配，不能只填 `object-storage`。容量按部署预算调整为正数。Worker 必须可达内部 endpoint，不能把宿主机 `localhost` 作为容器上传地址。 ```sh -scripts/storage-compose.sh --storage rustfs --env-file .env.release \ +scripts/storage-compose.sh --storage silo --env-file .env.release \ --project-name skillhub -- --profile github-sync up -d ``` @@ -154,10 +154,10 @@ scripts/storage-compose.sh --storage rustfs --env-file .env.release \ 先备份并确认目标版本的迁移要求，再更新固定镜像。通过 `RUSTFS_IMAGE` / `SILO_IMAGE` 覆盖时仍应指定经验证的不可变摘要；覆盖制品需要重新验收。 ```sh -scripts/storage-compose.sh --storage rustfs --env-file .env.release \ +scripts/storage-compose.sh --storage silo --env-file .env.release \ --project-name skillhub -- ps # 停止并移除本项目容器，保留命名数据卷；不要增加 -v -scripts/storage-compose.sh --storage rustfs --env-file .env.release \ +scripts/storage-compose.sh --storage silo --env-file .env.release \ --project-name skillhub -- down ``` diff --git a/openspec/changes/add-s3-compatible-storage-deployments/design.md b/openspec/changes/add-s3-compatible-storage-deployments/design.md index 3a39f904a..08e7ea356 100644 --- a/openspec/changes/add-s3-compatible-storage-deployments/design.md +++ b/openspec/changes/add-s3-compatible-storage-deployments/design.md @@ -43,7 +43,7 @@ flowchart TB ### D2. 技术配置：复用已有键，部署时确定后端 -下表全部是已有配置键。此处是新本地存储示例的目标取值，不覆盖全局默认值。RustFS 与 Silo 并列可选；RustFS 不是全局默认或最优推荐。`.env.release.example` 保持 local，开发 Compose 保持原有 Silo，基础发布 Compose 的 provider 缺省为 s3 且不捆绑存储服务。本次没有长期稳定性、大规模负载或 HA 对比证据。 +下表全部是已有配置键。此处是新本地存储示例的目标取值，不覆盖全局默认值。自建 S3 操作示例优先展示 Silo，RustFS 保留可选；这是基于本项目当前接入验收及既有部署方式的示例顺序，不是通用可靠性排名，不代表已验证 Silo 的长期生产可靠性。`.env.release.example` 保持 local，开发 Compose 保持原有 Silo，基础发布 Compose 的 provider 缺省为 s3 且不捆绑存储服务。本次没有长期稳定性、大规模负载或 HA 对比证据。 | 环境变量 | 示例或要求 | |---|---| @@ -100,10 +100,10 @@ sequenceDiagram # 先准备私有桶和业务身份，再使用一个选定的叠加文件 # 仓库根目录；操作会启动并修改相应项目的容器/卷 # .env.release 由部署者准备，存储初始化步骤另见手册 -scripts/storage-compose.sh --storage rustfs --env-file .env.release --project-name skillhub -- up -d +scripts/storage-compose.sh --storage silo --env-file .env.release --project-name skillhub -- up -d ``` -Silo 将参数 `--storage rustfs` 改为 `--storage silo`；外部 S3 使用 `--storage external`，仅加载基础 Compose。status、升级和 down 必须沿用相同 project、env 和文件组合，不删除数据卷作为更新步骤。 +RustFS 使用对应 env 示例并将参数 `--storage silo` 改为 `--storage rustfs`；外部 S3 使用 `--storage external`，仅加载基础 Compose。status、升级和 down 必须沿用相同 project、env 和文件组合，不删除数据卷作为更新步骤。 启动顺序：存储健康 → 操作者或显式一次性初始化准备桶/业务账号 → 业务凭证诊断通过 → 启动应用 → 业务冒烟。基础 Compose 的本地存储卷和权限初始化暂不清理，避免扩大改动。