mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-15 09:11:18 +00:00
Merge pull request #996 from iflytek/docs/issue-937-s3-storage-design
feat(storage): add S3-compatible deployment options
This commit is contained in:
XiaoSeS
GitHub
commit
a7a272a37b
35 changed files with 1808 additions and 17 deletions
1
.github/workflows/pr-helm-chart.yml
vendored
1
.github/workflows/pr-helm-chart.yml
vendored
|
|
@ -4,6 +4,7 @@ on:
|
|||
pull_request:
|
||||
paths:
|
||||
- charts/skillhub/**
|
||||
- deploy/storage/helm-values.example.yaml
|
||||
- .github/workflows/pr-helm-chart.yml
|
||||
- .github/workflows/publish-chart.yml
|
||||
types:
|
||||
|
|
|
|||
7
.github/workflows/pr-scripts.yml
vendored
7
.github/workflows/pr-scripts.yml
vendored
|
|
@ -9,6 +9,9 @@ on:
|
|||
- '.env.release.example'
|
||||
- '.env.release.draft'
|
||||
- 'compose.release.yml'
|
||||
- 'compose.storage.*.yml'
|
||||
- 'deploy/storage/**'
|
||||
- 'deploy/runtime-mirror-images.txt'
|
||||
- 'deploy/k8s/base/configmap.yaml'
|
||||
- 'deploy/k8s/base/scanner-deployment.yaml'
|
||||
- 'charts/skillhub/values.yaml'
|
||||
|
|
@ -51,6 +54,10 @@ jobs:
|
|||
- run: bash scripts/tests/publish-cli-test.sh
|
||||
- run: bash scripts/tests/runtime-secret-test.sh
|
||||
- run: bash scripts/tests/validate-release-config-test.sh
|
||||
- run: bash scripts/tests/storage-compose-test.sh
|
||||
- run: bash scripts/tests/storage-compose-render-test.sh
|
||||
- run: bash scripts/tests/storage-diagnostics-test.sh
|
||||
- run: bash scripts/tests/mirror-storage-images-test.sh
|
||||
- run: bash scripts/tests/nginx-forwarded-proto-test.sh
|
||||
- run: bash scripts/tests/smoke-test-admin-mode-test.sh
|
||||
- run: bash scripts/tests/web-base-path-routing-test.sh
|
||||
|
|
|
|||
|
|
@ -468,4 +468,19 @@ assert_rejected public-base-url-query --set-string publicBaseUrl=https://skills.
|
|||
assert_rejected public-base-url-fragment --set-string publicBaseUrl=https://skills.example.com#frag
|
||||
assert_rejected public-base-url-no-host --set-string publicBaseUrl=https://
|
||||
|
||||
# Example values must reuse the external application Secret and existing S3 config keys.
|
||||
helm template storage-contract "$CHART_DIR" -f "$TEST_VALUES" \
|
||||
-f "$CHART_DIR/../../deploy/storage/helm-values.example.yaml" \
|
||||
>"$TMP_DIR/storage-example.yaml"
|
||||
grep -Fq 'skillhub-storage-provider: "s3"' "$TMP_DIR/storage-example.yaml" \
|
||||
|| fail "Storage example must select the existing S3 provider"
|
||||
grep -Fq 's3-force-path-style: "true"' "$TMP_DIR/storage-example.yaml" \
|
||||
|| fail "Storage example must use path style"
|
||||
grep -Fq 'name: skillhub-app-secrets' "$TMP_DIR/storage-example.yaml" \
|
||||
|| fail "Storage example must reference the external application Secret"
|
||||
grep -Fq 'key: skillhub-storage-s3-access-key' "$TMP_DIR/storage-example.yaml" \
|
||||
|| fail "Storage access key Secret mapping changed"
|
||||
grep -Fq 'key: skillhub-storage-s3-secret-key' "$TMP_DIR/storage-example.yaml" \
|
||||
|| fail "Storage secret key Secret mapping changed"
|
||||
|
||||
echo "Helm configuration contract tests passed"
|
||||
|
|
|
|||
49
compose.storage.rustfs.yml
Normal file
49
compose.storage.rustfs.yml
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
# Optional single-node storage. Select exactly one overlay with scripts/storage-compose.sh.
|
||||
services:
|
||||
object-storage-permissions:
|
||||
image: ${RUSTFS_IMAGE:-docker.io/rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b}
|
||||
user: "0:0"
|
||||
entrypoint: ["chown", "10001:10001", "/data"]
|
||||
restart: "no"
|
||||
network_mode: none
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
cap_add: [CHOWN]
|
||||
volumes:
|
||||
- rustfs_data:/data
|
||||
|
||||
object-storage:
|
||||
image: ${RUSTFS_IMAGE:-docker.io/rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
object-storage-permissions:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
RUSTFS_ACCESS_KEY: ${SKILLHUB_OBJECT_STORAGE_ROOT_USER:?Set storage administrator user}
|
||||
RUSTFS_SECRET_KEY: ${SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD:?Set storage administrator password}
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_CONSOLE_ADDRESS: ":9001"
|
||||
ports:
|
||||
- "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_API_PORT:-9000}:9000"
|
||||
- "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_CONSOLE_PORT:-9001}:9001"
|
||||
volumes:
|
||||
- rustfs_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "--max-time", "3", "http://127.0.0.1:9000/health"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
|
||||
server:
|
||||
environment:
|
||||
SKILLHUB_STORAGE_PROVIDER: s3
|
||||
SKILLHUB_STORAGE_S3_ENDPOINT: http://object-storage:9000
|
||||
SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE: "true"
|
||||
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: "false"
|
||||
depends_on:
|
||||
object-storage:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
rustfs_data:
|
||||
33
compose.storage.silo.yml
Normal file
33
compose.storage.silo.yml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Optional single-node storage. Select exactly one overlay with scripts/storage-compose.sh.
|
||||
services:
|
||||
object-storage:
|
||||
image: ${SILO_IMAGE:-docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7}
|
||||
restart: unless-stopped
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${SKILLHUB_OBJECT_STORAGE_ROOT_USER:?Set storage administrator user}
|
||||
MINIO_ROOT_PASSWORD: ${SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD:?Set storage administrator password}
|
||||
ports:
|
||||
- "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_API_PORT:-9000}:9000"
|
||||
- "${OBJECT_STORAGE_BIND_ADDRESS:-127.0.0.1}:${OBJECT_STORAGE_CONSOLE_PORT:-9001}:9001"
|
||||
volumes:
|
||||
- silo_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "--max-time", "3", "http://127.0.0.1:9000/minio/health/live"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
|
||||
server:
|
||||
environment:
|
||||
SKILLHUB_STORAGE_PROVIDER: s3
|
||||
SKILLHUB_STORAGE_S3_ENDPOINT: http://object-storage:9000
|
||||
SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE: "true"
|
||||
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET: "false"
|
||||
depends_on:
|
||||
object-storage:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
silo_data:
|
||||
|
|
@ -5,3 +5,6 @@ docker.io/pgsty/silo:RELEASE.2026-09-16T00-00-00Z minio:latest
|
|||
prom/prometheus:latest prometheus:latest
|
||||
grafana/grafana:latest grafana:latest
|
||||
nginx:alpine nginx:alpine
|
||||
# Optional single-node storage overlays; do not reuse minio:latest for RustFS.
|
||||
docker.io/rustfs/rustfs:1.0.3@sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b rustfs:1.0.3-9ca9559533b5
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7 silo:20261018-2636fc4155d9
|
||||
|
|
|
|||
15
deploy/storage/application-policy.json
Normal file
15
deploy/storage/application-policy.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::skillhub"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::skillhub/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
17
deploy/storage/helm-values.example.yaml
Normal file
17
deploy/storage/helm-values.example.yaml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Merge into an existing valid SkillHub values file; this is not a full installation.
|
||||
# Storage is installed independently. The pre-created Secret must also contain
|
||||
# all existing application-required keys, not only these two S3 keys:
|
||||
# skillhub-storage-s3-access-key / skillhub-storage-s3-secret-key.
|
||||
existingSecret: skillhub-app-secrets
|
||||
s3:
|
||||
enabled: true
|
||||
endpoint: http://object-storage.storage.svc.cluster.local:9000
|
||||
publicEndpoint: https://storage.example.com
|
||||
bucket: skillhub
|
||||
region: us-east-1
|
||||
forcePathStyle: true
|
||||
disableChunkedEncoding: true
|
||||
autoCreateBucket: false
|
||||
presignExpiry: PT10M
|
||||
accessKey: ""
|
||||
secretKey: ""
|
||||
24
deploy/storage/rustfs.env.example
Normal file
24
deploy/storage/rustfs.env.example
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Merge these values into a protected .env.release based on .env.release.example.
|
||||
# Do not source an untrusted env file. Replace placeholders before provisioning.
|
||||
# Select with: scripts/storage-compose.sh --storage rustfs -- COMPOSE_COMMAND
|
||||
SKILLHUB_STORAGE_PROVIDER=s3
|
||||
SKILLHUB_STORAGE_S3_ENDPOINT=http://object-storage:9000
|
||||
# Must be reachable by download clients; loopback is only for local validation.
|
||||
SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000
|
||||
SKILLHUB_STORAGE_S3_BUCKET=skillhub
|
||||
SKILLHUB_STORAGE_S3_REGION=us-east-1
|
||||
SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=true
|
||||
SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true
|
||||
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false
|
||||
SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-with-business-access-key
|
||||
SKILLHUB_STORAGE_S3_SECRET_KEY=replace-with-business-secret
|
||||
# Storage administrator only; never pass these credentials to the SkillHub server.
|
||||
SKILLHUB_OBJECT_STORAGE_ROOT_USER=replace-with-storage-admin
|
||||
SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD=replace-with-storage-admin-secret
|
||||
OBJECT_STORAGE_BIND_ADDRESS=127.0.0.1
|
||||
OBJECT_STORAGE_API_PORT=9000
|
||||
OBJECT_STORAGE_CONSOLE_PORT=9001
|
||||
# Enable only with the existing GitHub-sync service and credential configuration.
|
||||
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=object-storage:9000
|
||||
# Set a positive byte budget appropriate for the deployment before enabling sync.
|
||||
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824
|
||||
24
deploy/storage/silo.env.example
Normal file
24
deploy/storage/silo.env.example
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Merge these values into a protected .env.release based on .env.release.example.
|
||||
# Do not source an untrusted env file. Replace placeholders before provisioning.
|
||||
# Select with: scripts/storage-compose.sh --storage silo -- COMPOSE_COMMAND
|
||||
SKILLHUB_STORAGE_PROVIDER=s3
|
||||
SKILLHUB_STORAGE_S3_ENDPOINT=http://object-storage:9000
|
||||
# Must be reachable by download clients; loopback is only for local validation.
|
||||
SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000
|
||||
SKILLHUB_STORAGE_S3_BUCKET=skillhub
|
||||
SKILLHUB_STORAGE_S3_REGION=us-east-1
|
||||
SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=true
|
||||
SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true
|
||||
SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET=false
|
||||
SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-with-business-access-key
|
||||
SKILLHUB_STORAGE_S3_SECRET_KEY=replace-with-business-secret
|
||||
# Storage administrator only; never pass these credentials to the SkillHub server.
|
||||
SKILLHUB_OBJECT_STORAGE_ROOT_USER=replace-with-storage-admin
|
||||
SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD=replace-with-storage-admin-secret
|
||||
OBJECT_STORAGE_BIND_ADDRESS=127.0.0.1
|
||||
OBJECT_STORAGE_API_PORT=9000
|
||||
OBJECT_STORAGE_CONSOLE_PORT=9001
|
||||
# Enable only with the existing GitHub-sync service and credential configuration.
|
||||
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=object-storage:9000
|
||||
# Set a positive byte budget appropriate for the deployment before enabling sync.
|
||||
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
# skillhub 部署架构与运维
|
||||
|
||||
对象存储可选部署、凭证和迁移说明见[对象存储手册](object-storage.md)。
|
||||
|
||||
## 1 运行模型
|
||||
|
||||
当前仓库只保留两种运行方式：
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
# GitHub 公开 Skill 同步：管理员运行手册
|
||||
|
||||
对象存储可选部署、凭证和迁移说明见[对象存储手册](object-storage.md)。
|
||||
|
||||
本功能只采集 GitHub 公开仓库。SkillHub Java 保存配置、来源、任务、审核和发布状态；独立 Go 服务轮询 Java 的内部接口执行发现与采集。Go 停止后，Java 中已保存的任务和检查点仍在；重新启动 Go 后由租约机制继续处理。功能仍在实施验收阶段，生产启用前须完成 [OpenSpec 验收任务](../openspec/changes/add-github-skill-sync/tasks.md) 中未勾选的部署检查。
|
||||
|
||||
## 启用前配置
|
||||
|
|
|
|||
230
docs/object-storage.md
Normal file
230
docs/object-storage.md
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
# 对象存储部署与迁移参考
|
||||
|
||||
SkillHub 通过同一个 S3 接口连接 RustFS、PGSTY Silo 或外部 S3 服务，不需要按存储品牌修改业务代码。`local` 模式保留。本文新增的单机叠加配置不是高可用存储集群；本次验证范围是文末所列指定版本的新部署接入与业务操作，不包含生产数据迁移支持。
|
||||
|
||||
## 1. 选择部署方式
|
||||
|
||||
| 方式 | 使用配置 | 数据归属 |
|
||||
|---|---|---|
|
||||
| Silo | `compose.release.yml` + `compose.storage.silo.yml` | 独立 `silo_data` 卷 |
|
||||
| RustFS | `compose.release.yml` + `compose.storage.rustfs.yml` | 独立 `rustfs_data` 卷 |
|
||||
| 外部 S3 | 仅 `compose.release.yml` | 外部服务的桶 |
|
||||
| local | 仅 `compose.release.yml`，显式 `SKILLHUB_STORAGE_PROVIDER=local` | 现有应用存储卷 |
|
||||
|
||||
一次只选择一个本地存储。使用 `scripts/storage-compose.sh`，它会拒绝重复选择和额外 `-f` 文件；不要自行同时合并两个存储文件。所有命令在仓库根目录执行，要求 Bash、Docker Compose v2；诊断另外要求 JDK 21+、Maven Wrapper 和 GNU timeout。
|
||||
|
||||
本文优先以 Silo 展示自建 S3 的操作步骤，RustFS 保留可选。这一顺序基于本项目当前接入验收及既有部署方式，不是通用可靠性排名，也不代表已验证 Silo 的长期生产可靠性。`.env.release.example` 仍显式选择 `local`，开发 Compose 的 `minio` 服务仍使用原有 Silo 镜像；仅使用基础 `compose.release.yml` 时，provider 的缺省值为 `s3`，但不会自动部署任一存储产品。已有实例不会自动切换。本次没有长期稳定性、大规模负载或高可用对比证据，部署方应按自身运维条件选择并验收。
|
||||
|
||||
## 2. 准备受保护配置
|
||||
|
||||
从 `.env.release.example` 准备 `.env.release`，再把 `deploy/storage/silo.env.example`（RustFS 使用 `deploy/storage/rustfs.env.example`）中的存储配置合入该文件，并替换所有占位值。不要直接用示例部署，也不要把实际配置加入 Git。
|
||||
|
||||
```sh
|
||||
# 已有 .env.release 时不要重复复制覆盖
|
||||
cp .env.release.example .env.release
|
||||
chmod 600 .env.release
|
||||
# 编辑完整应用配置，再合入选定产品的 env 示例
|
||||
```
|
||||
|
||||
配置分为两组：
|
||||
|
||||
- `SKILLHUB_OBJECT_STORAGE_ROOT_USER` / `SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD` 仅供对象存储管理员初始化；叠加文件不会将它们传给 SkillHub server。
|
||||
- `SKILLHUB_STORAGE_S3_ACCESS_KEY` / `SKILLHUB_STORAGE_S3_SECRET_KEY` 是单独的应用账号，限定在应用桶范围。不要复用存储 root 账号。
|
||||
|
||||
`SKILLHUB_STORAGE_S3_ENDPOINT=http://object-storage:9000` 用于后端对象操作及 Worker 的预签名 PUT；`SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT` 用于用户下载签名。后者必须是下载方真实可达的地址；示例中的 `127.0.0.1` 仅适用于当前机器本地验证。生产环境应准备专用存储域名、TLS 与保持签名主机/路径的反向代理，不能签名后替换 URL。
|
||||
|
||||
新叠加配置明确启用 S3、路径寻址、关闭自动建桶；其值覆盖基础 Compose 的对应配置。其他应用字段仍来自 `.env.release`。更改配置后重建对应服务才能生效，不由后台系统配置页面管理。
|
||||
|
||||
两个存储 API/Console 默认仅绑定 `127.0.0.1:9000/9001`。端口冲突时调整 `OBJECT_STORAGE_API_PORT` / `OBJECT_STORAGE_CONSOLE_PORT`，同时调整下载公开端点；容器内部端口和 endpoint 仍是 9000。不要为了远程浏览 Console 将管理接口无保护地暴露到公网。
|
||||
|
||||
## 3. 分步初始化，避免应用先于桶启动
|
||||
|
||||
下面用 Silo；选择 RustFS 时，改用其 env 示例，并把下文每条命令的 `--storage silo` 改为 `--storage rustfs`。固定 `--project-name`，后续更新、状态和停机也使用相同参数。
|
||||
|
||||
```sh
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release \
|
||||
--project-name skillhub -- up -d object-storage
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release \
|
||||
--project-name skillhub -- ps object-storage
|
||||
```
|
||||
|
||||
上述命令会创建选定产品的容器和数据卷，但不会启动整个应用栈。RustFS 会先用同一固定镜像的一次性权限容器，将数据卷根目录交给其运行身份 `10001:10001`；它无网络、不持有密钥，完成后退出，不对已有对象递归修改权限。
|
||||
|
||||
存储健康后，由存储管理员完成以下初始化。使用对应版本的官方 Console 或管理客户端，不使用 SkillHub 的业务进程代为创建账号：
|
||||
|
||||
1. 在本地 Console `http://127.0.0.1:9001` 登录存储管理员账号。
|
||||
2. 创建私有桶 `skillhub`；已存在时核对，不清空、不重建桶。
|
||||
3. 创建独立业务身份并授予 `deploy/storage/application-policy.json` 中的桶内权限。若修改桶名，同步修改 policy 的两个 Resource。
|
||||
4. 使用业务身份的凭证填写应用的 S3 access/secret。确认匿名对象读取被拒绝，业务身份不能操作其他桶或管理账号。
|
||||
5. 重复初始化只做存在性和权限核对，不自动轮换密钥、删除用户或变更已有桶的数据。
|
||||
|
||||
不同产品的管理 API 与 Console 功能不完全一致；不要把 MinIO 管理命令未经验证地套到 RustFS。政策中包含读写删除对象、列举桶和读取桶区域；不授予创建桶、管理用户或公开桶权限。
|
||||
|
||||
### 可选：用已验证管理客户端初始化
|
||||
|
||||
本次使用 [PGSTY mcli RELEASE.2026-09-16T00-00-00Z](https://github.com/pgsty/mc/releases/tag/RELEASE.2026-09-16T00-00-00Z) 验证了 RustFS 1.0.3 和 Silo RELEASE.2026-10-18T00-00-00Z 的账号与策略管理。客户端安装和校验按其官方发布说明进行；`mcli` 是部署管理员工具，不是 SkillHub 的运行依赖。
|
||||
|
||||
先将 `.env.release` 中的 root、业务账号和桶参数安全注入当前管理会话。以下命令仅用于**首次创建账号**；已有账号先查询并核对，不能重复 `user add` 来重置凭证。
|
||||
|
||||
```sh
|
||||
umask 077
|
||||
export MC_CONFIG_DIR="$PWD/.dev/object-storage-admin"
|
||||
mkdir -p "$MC_CONFIG_DIR"
|
||||
# 这里是管理员从宿主机访问的地址，端口需与本部署一致
|
||||
mcli alias set storage http://127.0.0.1:9000 \
|
||||
"$SKILLHUB_OBJECT_STORAGE_ROOT_USER" "$SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD"
|
||||
mcli mb --ignore-existing "storage/$SKILLHUB_STORAGE_S3_BUCKET"
|
||||
# 首次创建之前先查询；存在时跳过 user add，核对已有身份和权限
|
||||
mcli admin user info storage "$SKILLHUB_STORAGE_S3_ACCESS_KEY"
|
||||
# 仅确认不存在且信息查询不是网络/权限错误时执行
|
||||
mcli admin user add storage "$SKILLHUB_STORAGE_S3_ACCESS_KEY" "$SKILLHUB_STORAGE_S3_SECRET_KEY"
|
||||
# 先核对 policy 的桶名和权限；同名已有策略需人工核对，不能盲目覆盖
|
||||
mcli admin policy create storage skillhub-app deploy/storage/application-policy.json
|
||||
mcli admin policy attach storage skillhub-app --user "$SKILLHUB_STORAGE_S3_ACCESS_KEY"
|
||||
```
|
||||
|
||||
重复部署时，使用 `mb --ignore-existing`、`admin user info` 和 `admin policy info` 核对状态，不删除桶、不重建账号或轮换凭证。管理客户端的本地配置含管理员凭证，应保持私有；它不挂载到应用容器。
|
||||
|
||||
## 4. 诊断和启动应用
|
||||
|
||||
先将上一步的业务 S3 参数注入当前 shell。诊断从宿主机执行时，endpoint 必须改为宿主机可达的 API 地址，而非仅容器网络可见的 `object-storage`。不要把含密钥的整行命令粘贴到共享日志中。
|
||||
|
||||
以下仅为本机验证示例；生产诊断的公开端点应使用真实下载域名，不能把 loopback 测试当成外网连通性证明。用子 shell 限定临时地址，避免它们覆盖后续 Compose 读取的发布配置。
|
||||
|
||||
```sh
|
||||
# 其他 S3 参数和业务凭证应通过当前会话的安全配置方式注入
|
||||
(
|
||||
export SKILLHUB_STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
|
||||
export SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000
|
||||
export SKILLHUB_STORAGE_S3_BUCKET=skillhub
|
||||
export SKILLHUB_STORAGE_S3_REGION=us-east-1
|
||||
export SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE=true
|
||||
export SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=true
|
||||
scripts/storage-diagnostics.sh
|
||||
)
|
||||
```
|
||||
|
||||
该命令使用实际 `ObjectStorageService`，写入随机 `skillhub-diagnostics/<随机值>/` 对象，检查字节、元数据、GET/PUT 签名、匿名拒绝、签名过期/篡改和删除。完成后仅删除本次创建的精确对象；失败返回非零，原始日志默认保存在 Git 忽略的 `.dev/storage-diagnostics/`，权限受 umask 077 保护。日志可能包含签名 URL，分享前必须脱敏。超时或进程被强制终止时可能有残留，使用私有日志中的前缀核查并精确清理；不要清空整个桶。
|
||||
|
||||
交付验收时另运行 `scripts/storage-diagnostics.sh --full`，额外创建 1001 个小对象，检查真实翻页、统计上限与跨批次删除。满批删除如遇后端短暂逐项失败，适配层必须明确报错；验收会记录失败并最多尝试 3 次调用方恢复，最终独立确认对象已经不存在，不把 HTTP 200 当成全部删除成功；该模式最多运行 420 秒。普通诊断最多 180 秒，外层强制终止后可能需要按日志核查残留。
|
||||
|
||||
S3 PUT 会在应用私有临时文件中暂存请求体，以支持文件流签名和重试；暂存写入保持原文件的私有权限（POSIX 环境为 0600），读取或上传失败时清理文件，不因清理失败掩盖原始错误；部署时应计入并发上传的临时磁盘预算，不能只预算对象存储的数据卷。临时文件在操作成功或失败后清理，不将整个技能包缓存到 JVM 堆中。
|
||||
|
||||
诊断不是应用 liveness，不要把这个有写入动作的命令接到公开健康检查。
|
||||
|
||||
```sh
|
||||
make validate-release-config
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release \
|
||||
--project-name skillhub -- up -d
|
||||
```
|
||||
|
||||
`make validate-release-config` 校验应用发布配置，不证明存储可用；同样，存储健康不证明发布、审核和下载业务已通过。启动后还需按本次变更的业务验收矩阵验证。
|
||||
|
||||
### GitHub 同步
|
||||
|
||||
沿用[同步运行手册](github-sync-operations.md)的服务密钥和其他配置，另外设置：
|
||||
|
||||
```dotenv
|
||||
SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS=object-storage:9000
|
||||
SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES=1073741824
|
||||
```
|
||||
|
||||
允许列表按完整主机及端口匹配，不能只填 `object-storage`。容量按部署预算调整为正数。Worker 必须可达内部 endpoint，不能把宿主机 `localhost` 作为容器上传地址。
|
||||
|
||||
```sh
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release \
|
||||
--project-name skillhub -- --profile github-sync up -d
|
||||
```
|
||||
|
||||
## 5. Helm / Kubernetes
|
||||
|
||||
存储独立部署和维护，SkillHub Chart 不安装 RustFS/Silo 子 Chart，不在应用卸载时删除对象存储。
|
||||
|
||||
`deploy/storage/helm-values.example.yaml` 只包含新增接入所需字段，需要合并到已有有效 values 中。使用已有 `existingSecret` 时，Chart 不创建应用 Secret；必须保留其他应用必需键，并加入：
|
||||
|
||||
- `skillhub-storage-s3-access-key`
|
||||
- `skillhub-storage-s3-secret-key`
|
||||
|
||||
它们只存业务凭证，不存对象存储 root 密钥。启用 `s3.enabled` 后，`s3.*` 决定地址和协议参数，凭证从已有 Secret 读取；`s3.accessKey` / `s3.secretKey` 留空，不把明文写入 values。变更外部 Secret 后按部署策略滚动应用。
|
||||
|
||||
参见 [Kubernetes 指南](skillhub/guide/kubernetes.md)。模板渲染通过与真实 K8s 连通性、对象操作、升级验收分别记录。
|
||||
|
||||
## 6. 更新与停机
|
||||
|
||||
先备份并确认目标版本的迁移要求，再更新固定镜像。通过 `RUSTFS_IMAGE` / `SILO_IMAGE` 覆盖时仍应指定经验证的不可变摘要；覆盖制品需要重新验收。
|
||||
|
||||
```sh
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release \
|
||||
--project-name skillhub -- ps
|
||||
# 停止并移除本项目容器，保留命名数据卷；不要增加 -v
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release \
|
||||
--project-name skillhub -- down
|
||||
```
|
||||
|
||||
`deploy/runtime-mirror-images.txt` 保留原有 `minio:latest` 别名。新产品采用独立版本标签；已有镜像目标会被同步脚本跳过，修改映射清单不等于更新仓库内容。实际同步属于单独的发布操作，验收应比较目标 manifest digest。
|
||||
|
||||
## 7. 已有数据迁移的风险与操作参考
|
||||
|
||||
本节不宣称生产迁移支持，也不提供自动迁移功能。现有证据仅包括独立测试桶中 RustFS → Silo → RustFS 的小样本演练；未验证旧 MinIO 或 local 到新后端的真实迁移，也未完成大规模生产数据迁移、恢复及停机窗口验收。下列步骤是安全参考，不能作为可直接执行的生产迁移方案；具体实例需要另行清点数据、制定方案并完成代表性演练。
|
||||
|
||||
修改 endpoint/provider 不会迁移文件。RustFS、Silo 和 local 不得默认共用原始数据目录。
|
||||
|
||||
1. 记录现有配置、对象清单、元数据与加密方式，备份并验证恢复。目标使用独立桶/卷和业务账号。
|
||||
2. 逻辑复制对象，保留 key、Content-Type 及业务需要的元数据；记录迁移工具版本与选项。加密对象经授权解密读取后写入新服务，不能直接复制内部存储文件。
|
||||
3. 用对象数量、大小和独立内容哈希校验；不能把 multipart 或加密对象的 ETag 一律当 MD5。
|
||||
4. 进入维护窗口，暂停发布、头像变更/审核、Suite 写入、同步、后台删除/过期清理；排空任务。已签发 PUT 也必须等到过期或撤销旧端写权限，避免最终同步后仍写旧桶。
|
||||
5. 在冻结点完成最终同步，同时处理新增、覆盖和删除差异。保留旧源只读；已签发 GET 可在有效期内继续指向旧源，应保留读取或声明短暂中断窗口。
|
||||
6. 切换配置、重建受影响服务，检查下载原始字节、头像、Suite 和同步导入/清理，再恢复写入。
|
||||
7. 恢复写入前可以回退配置到未变的旧源；恢复写入后，必须重新停写并协调新后端的新增、覆盖、删除差异后再回切，防止丢失新文件或恢复已删除内容。
|
||||
|
||||
### 小样本逻辑复制演练参考
|
||||
|
||||
本次用上述版本的 `mcli` 在独立测试桶进行 RustFS → Silo → RustFS 小样本演练（正向清单 134 个对象，回退清单 135 个对象），包含在线初始复制、冻结后的新增／覆盖／删除、在途 PUT 过期，以及切换后产生新数据再回退。`mirror` 会保存本次验证对象的 Content-Type 和自定义元数据；版本历史、对象锁、加密和存储策略仍需部署方分别评估。
|
||||
|
||||
下面的 `old`、`new` 是操作者预先配置的管理客户端别名，不是内置端点。先核对方向、桶名和私有权限，再执行；`--remove` 会删除目标中源端不存在的对象，只能在已批准的目标桶范围操作。不能把它指向尚在写入的旧源或其他业务桶。
|
||||
|
||||
```sh
|
||||
# 初始复制；此阶段源仍可写，不作为最终一致状态
|
||||
mcli mirror --overwrite --max-workers 2 old/skillhub new/skillhub
|
||||
# 按前述步骤暂停应用、Worker、定时清理等全部写入，并排空已签发 PUT
|
||||
mcli mirror --dry-run --overwrite --remove --max-workers 2 old/skillhub new/skillhub
|
||||
# 审核删除差异后，在冻结窗口执行最终同步
|
||||
mcli mirror --overwrite --remove --max-workers 2 old/skillhub new/skillhub
|
||||
# 对应对象逐项核对原始字节和元数据；不要用 ETag 代替哈希
|
||||
mcli stat --json old/skillhub/OBJECT_KEY
|
||||
mcli stat --json new/skillhub/OBJECT_KEY
|
||||
mcli cat old/skillhub/OBJECT_KEY | sha256sum
|
||||
mcli cat new/skillhub/OBJECT_KEY | sha256sum
|
||||
```
|
||||
|
||||
示例的 `OBJECT_KEY` 必须替换为实际对象键；正式迁移应对完整清单逐项自动比对，不只抽一个文件。未使用 `--preserve`，避免同时复制桶策略和锁配置；目标桶及业务权限单独初始化并保持私有。对额外业务元数据先做代表性验证。
|
||||
|
||||
恢复写入后的回退采用同一停写和签名排空规则，重新核对方向后先 `new/skillhub → old/skillhub` 逻辑同步及比对，再回切应用，不能直接启动旧源。旧签名 URL 不保证跨产品切换后继续有效，应保留旧读端或明确中断窗口；新签名需在切换后重新申请。
|
||||
|
||||
旧源按部署方备份保留政策保存，确认恢复能力后再单独授权清理。失败时保留清单与源数据，不通过删除业务数据“修复”验收。
|
||||
|
||||
## 8. 制品和兼容性边界
|
||||
|
||||
| 产品 | 固定版本 | 多架构 manifest digest | 许可证 |
|
||||
|---|---|---|---|
|
||||
| RustFS | 1.0.3 | `sha256:9ca9559533b537d10482fe923288b676ad08f92a7069d9a17327fbae3cab688b` | Apache-2.0 |
|
||||
| PGSTY Silo | RELEASE.2026-10-18T00-00-00Z | `sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7` | AGPL-3.0-or-later |
|
||||
|
||||
两者的官方 manifest 均包含 linux/amd64 和 linux/arm64；本机运行结果只证明实际执行的架构。新增 Silo 叠加文件选用上述版本；已有开发服务及 `minio:latest` 别名保持原版本，不自动升级已有实例。
|
||||
|
||||
本次在 linux/amd64 上使用同一应用构件完成两种产品的新部署真实契约验收；不代表任意 S3 产品、任意版本、arm64 运行、长期稳定性、大规模负载、高可用部署或生产迁移已获验证。下表只列接入和业务操作结果，不是存储产品优劣排名或生产迁移支持矩阵。
|
||||
|
||||
| 范围 | RustFS 1.0.3 | Silo 2026-10-18 |
|
||||
|---|---|---|
|
||||
| 文件流上传、字节和元数据、超过 1000 项分页和删除 | 通过；满批曾出现 `ServiceUnavailable` 逐项失败，必须由调用方处理 | 通过 |
|
||||
| GET/PUT 签名、所需头、过期/篡改、匿名私有访问 | 通过 | 通过 |
|
||||
| 发布审核下载、头像待审/批准/移除、Suite | 通过 | 通过 |
|
||||
| Go Worker 内部端点 PUT、导入及暂存清理 | 通过 | 通过 |
|
||||
|
||||
RustFS 的满批临时错误不是“HTTP 200 即全部成功”；适配层会明确抛错，诊断的有界恢复只证明操作可幂等恢复，不能证明后端从不产生逐项失败。只有已接入持久化补偿的删除链路才会由现有任务重试；部分草稿删除和 Suite 暂存清理仍为 best-effort，失败仅记录日志，可能留下对象，并不保证自动恢复。部署方应监测清理失败与残留，再按业务记录核查和精确清理，不能直接清空桶。不把诊断脚本的三次恢复误认为生产适配器自动重试。
|
||||
|
||||
制品核验采用 Trivy 0.76.0：RustFS 镜像系统包和官方发行 SBOM（1157 个组件）均未检出 High/Critical；Silo 镜像尚有 `CVE-2026-39414` 的 legacy MinIO 模块匹配。对这一条告警，已将镜像二进制的 commit `0cd388c666f8211f552604ae73448a066e9c25ce` 与发行标签源码绑定，并核对 CSV 有界读取修复；实际普通 2 MiB 与 gzip 解压后 2 MiB 的无换行记录均被拒绝且服务存活，本地分析裁定该特定漏洞路径已修补（依据：[上游公告](https://github.com/minio/minio/security/advisories/GHSA-h749-fxx7-pwpg)、[镜像对应提交的 CSV 读取实现](https://github.com/pgsty/silo/blob/0cd388c666f8211f552604ae73448a066e9c25ce/internal/s3select/csv/reader.go)）。没有全局忽略 CVE，也不据此声称整体镜像无风险。部署时应使用自己的当前漏洞库再次扫描。
|
||||
|
||||
未运行真实 Kubernetes 集群、外网 TLS/代理或远端镜像仓库同步；Helm 证据限于 lint 和模板契约。旧 `minio:latest` 仍映射既有版本，其风险与升级另行管理，不能把新增 Silo 的扫描结论套给旧别名。
|
||||
|
||||
参考：[RustFS 官方部署资料](https://github.com/rustfs/docs.rustfs.com/blob/main/content/en/installation/container/docker.md)、[RustFS 发布](https://github.com/rustfs/rustfs/releases/tag/1.0.3)、[Silo 源仓库](https://github.com/pgsty/silo)。厂商兼容性声明不代替 SkillHub 自身验收。
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
# Kubernetes 部署指南
|
||||
|
||||
对象存储可选部署、凭证和迁移说明见[对象存储手册（GitHub）](https://github.com/iflytek/skillhub/blob/main/docs/object-storage.md)。
|
||||
|
||||
本文档说明如何在 Kubernetes 集群中部署 SkillHub。
|
||||
|
||||
## 前置条件
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
schema: spec-driven
|
||||
created: 2026-10-21
|
||||
185
openspec/changes/add-s3-compatible-storage-deployments/design.md
Normal file
185
openspec/changes/add-s3-compatible-storage-deployments/design.md
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
## Context
|
||||
|
||||
动机和范围见 [proposal.md](./proposal.md)。设计基线为 `6c739b838ed543dac9967e60b248fbacad4ad9f0`，源代码核对日期为 2026-10-21。本文件定义目标方案；实际实现与验收状态以 tasks.md 和对应交付记录为准。
|
||||
|
||||
### 已有能力与实际缺口
|
||||
|
||||
| 位置 | 已有事实 | 本次处理 |
|
||||
|---|---|---|
|
||||
| `server/skillhub-storage/.../ObjectStorageService.java` | 对象读写、删除、元数据、下载签名；上传签名与前缀统计为可选能力 | 保持接口边界，新增共享契约测试 |
|
||||
| 同模块 `S3StorageService.java` | AWS SDK，支持 endpoint、publicEndpoint、路径寻址、预签名 | 验证协议差异，只有测试证明必要时才局部修复 |
|
||||
| `compose.release.yml` | 传入 S3 参数，无本地对象存储服务 | 新增可选叠加文件，不改现有部署默认值 |
|
||||
| `docker-compose.yml` | 开发环境 `minio` 服务实际使用 PGSTY Silo | 保留服务名和旧镜像别名兼容 |
|
||||
| `.env.release.example` | 显式 local；Compose 缺省 provider 为 s3 | 新示例显式设置 provider，避免依赖不同层默认值 |
|
||||
| `charts/skillhub/values.yaml` | `s3.enabled`、外部端点、`existingSecret` | 提供接入示例，不增加必装存储子 Chart |
|
||||
| `GithubSyncUploadAppService` | local 流式上传、s3 预签名 PUT | 保留模式和上传凭证边界，验证 Worker 可达性 |
|
||||
|
||||
S3 当前客户端初始化不等于存储可用性验证。既有单元测试不能证明 RustFS 或 Silo 的真实兼容性。批量删除响应中的逐项错误需要补充失败响应测试，以判断是否需要修复成功判定；不能仅凭 HTTP 200 宣告全部成功。
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals:** 以统一 S3 协议保持业务代码稳定，交付可选存储运行方式、可重复的新部署接入验收及迁移风险参考。
|
||||
|
||||
**Non-Goals:** 不引入 `rustfs` / `silo` provider、业务模块专用 SDK、插件加载框架、多后端路由、双写、热切换、数据库迁移或后台配置页面。不将本次单机示例包装成高可用集群方案，不自动升级已有 Silo 部署。不提供自动迁移或宣称生产迁移支持，旧 MinIO/local 真实迁移及大规模生产迁移尚未验收。
|
||||
|
||||
## Decisions
|
||||
|
||||
### D1. 架构：沿用协议接口，供应商差异留在部署层
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
App[技能发布、下载、头像、Suite、同步编排] --> SPI[ObjectStorageService]
|
||||
SPI --> Local[LocalFileStorageService]
|
||||
SPI --> S3[S3StorageService / AWS SDK]
|
||||
S3 --> R[RustFS]
|
||||
S3 --> S[PGSTY Silo]
|
||||
S3 --> E[外部 S3 兼容服务]
|
||||
Env[部署环境与 Secret] --> S3
|
||||
```
|
||||
|
||||
`skillhub-storage` 持有接口、实现、属性与协议测试；应用服务只编排业务调用。产品初始化命令和镜像参数留在交付文件及操作手册中。现有 domain 到 storage 的已知依赖偏差不在本次扩大或重构。
|
||||
|
||||
替代方案：每个品牌实现一个 adapter 会重复签名、重试与错误处理，无法提升本次扩展性；通用插件框架增加没有实际调用需求的机制。只有将来出现实际非 S3 协议需求时再评估新实现。
|
||||
|
||||
### D2. 技术配置：复用已有键，部署时确定后端
|
||||
|
||||
下表全部是已有配置键。此处是新本地存储示例的目标取值，不覆盖全局默认值。自建 S3 操作示例优先展示 Silo，RustFS 保留可选；这是基于本项目当前接入验收及既有部署方式的示例顺序，不是通用可靠性排名，不代表已验证 Silo 的长期生产可靠性。`.env.release.example` 保持 local，开发 Compose 保持原有 Silo，基础发布 Compose 的 provider 缺省为 s3 且不捆绑存储服务。本次没有长期稳定性、大规模负载或 HA 对比证据。
|
||||
|
||||
| 环境变量 | 示例或要求 |
|
||||
|---|---|
|
||||
| `SKILLHUB_STORAGE_PROVIDER` | `s3` |
|
||||
| `SKILLHUB_STORAGE_S3_ENDPOINT` | `http://object-storage:9000`，后端和 Worker 可达 |
|
||||
| `SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT` | 例如 `https://storage.example.com`，下载方可达；留空则使用内部端点 |
|
||||
| `SKILLHUB_STORAGE_S3_BUCKET` | `skillhub`，先准备 |
|
||||
| `SKILLHUB_STORAGE_S3_REGION` | 示例 `us-east-1`，与服务配置匹配 |
|
||||
| `SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE` | 本地产品示例明确 `true` |
|
||||
| `SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING` | 按契约测试结果显式设置，不以品牌条件散落在业务代码中 |
|
||||
| `SKILLHUB_STORAGE_S3_ACCESS_KEY` / `SKILLHUB_STORAGE_S3_SECRET_KEY` | 业务账号，由受保护 env / Secret 注入 |
|
||||
| `SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET` | 新示例 `false` |
|
||||
| `SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY` | 沿用现有 `PT10M` 默认值，验收另用短期签名 |
|
||||
|
||||
AWS 默认凭证链的既有接入能力保留，不强制所有外部 S3 使用静态密钥。静态凭证要求成对配置。密钥不进入 Git、镜像、系统配置表或诊断输出。
|
||||
|
||||
Helm 使用已有 `s3.*` 与 `existingSecret`，实施时核对模板实际 Secret 键名和优先级后提供可执行示例，不在此编造新的 values 字段。
|
||||
|
||||
### D3. 网络与签名：按实际访问方分流
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant User as 浏览器或CLI
|
||||
participant API as SkillHub
|
||||
participant Public as 存储公开端点
|
||||
participant Internal as 存储内部端点
|
||||
participant Worker as GitHub同步Worker
|
||||
User->>API: 请求下载（现有业务授权）
|
||||
API-->>User: 公开端点的限时GET签名
|
||||
User->>Public: 原样使用签名URL
|
||||
API->>Internal: 读写对象及元数据
|
||||
Worker->>API: 申请同步上传凭证
|
||||
API-->>Worker: 内部端点的PUT签名及所需请求头
|
||||
Worker->>Internal: 上传staging对象
|
||||
```
|
||||
|
||||
当前 GET presigner 使用 publicEndpoint，未设置时回退 endpoint；PUT presigner 使用 endpoint。公开域名、证书和反向代理由部署者准备，不能在生成签名后替换 host、scheme 或路径。公网下载可达但 Worker 不可达不算验收通过。
|
||||
|
||||
启用同步时配置已有 `SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS`，以及正数的 `SKILLHUB_GITHUB_SYNC_STAGING_CAPACITY_LIMIT_BYTES`；沿用现有上传 ticket、lease 和容量检查，不绕过安全主机校验。必要的专用存储域名代理转发应保持签名语义，不把未验证的子路径重写列为支持配置。
|
||||
|
||||
### D4. Compose 与 Helm 的交付方式
|
||||
|
||||
交付文件：
|
||||
|
||||
- `compose.storage.rustfs.yml`：RustFS 服务、健康检查、独立 `rustfs_data` 卷。
|
||||
- `compose.storage.silo.yml`：Silo 服务、健康检查、独立 `silo_data` 卷。
|
||||
- 产品配套 env 示例与初始化操作说明：由实施任务确定文档路径并链接到现有部署手册。
|
||||
|
||||
两个叠加文件各自提供 `object-storage` 服务，**只能选择一个**。由于 Compose 合并同名服务可能产生混合配置，不能依赖 Compose 自己报错：新增薄包装部署预检入口，检查输入文件列表，在执行 Compose 前拒绝同时选择二者，并将余下参数透传。手册采用该入口；直接调用 Compose 的高级用法需明确互斥条件。
|
||||
|
||||
部署操作与初始化步骤见 `docs/object-storage.md`；通过包装脚本保证单选：
|
||||
|
||||
```sh
|
||||
# 先准备私有桶和业务身份，再使用一个选定的叠加文件
|
||||
# 仓库根目录；操作会启动并修改相应项目的容器/卷
|
||||
# .env.release 由部署者准备，存储初始化步骤另见手册
|
||||
scripts/storage-compose.sh --storage silo --env-file .env.release --project-name skillhub -- up -d
|
||||
```
|
||||
|
||||
RustFS 使用对应 env 示例并将参数 `--storage silo` 改为 `--storage rustfs`；外部 S3 使用 `--storage external`，仅加载基础 Compose。status、升级和 down 必须沿用相同 project、env 和文件组合，不删除数据卷作为更新步骤。
|
||||
|
||||
启动顺序：存储健康 → 操作者或显式一次性初始化准备桶/业务账号 → 业务凭证诊断通过 → 启动应用 → 业务冒烟。基础 Compose 的本地存储卷和权限初始化暂不清理，避免扩大改动。
|
||||
|
||||
初始化优先提供人工可执行的明确步骤，不默认启动持有 root 密钥的常驻容器；如封装一次性初始化，必须幂等、失败阻断且完成后退出。具体产品命令、环境变量、健康接口和 UID 以所选发行版官方资料与运行结果确认，不能照搬另一产品参数。
|
||||
|
||||
Helm 将存储作为独立服务管理。Chart 的安装、升级和卸载不承担对象存储生命周期，避免卸载应用误删数据。现有测试脚本中使用 Silo 不代表 Chart 默认捆绑它。
|
||||
|
||||
### D5. 请求体重放与临时文件
|
||||
|
||||
实际发布通过 `Files.newInputStream` 提供不可 reset 的流。AWS SDK 2.20.26 的签名/重试会重读请求体，直接 `RequestBody.fromInputStream` 在此路径下会消耗流并使后续发送不完整；真实应用和文件流契约已复现超时，内存流测试不能覆盖此缺陷。
|
||||
|
||||
复用适配层已有暂存 helper，将每次 PUT 的请求体暂存到私有临时文件，用 `RequestBody.fromFile` 提供可重放的源；不在内存缓存整个包。上传前校验实际字节数与声明大小，成功失败均释放暂存文件。暂存写入必须保留创建时的私有权限，不能通过替换文件重建为默认权限；POSIX 环境仅授予运行身份读写权限（0600）。读取发生 IOException 或运行时异常时，helper 必须清理尚未返回给调用方的文件；清理失败不得掩盖原始异常。接口与调用方保持不变，代价为每次 S3 PUT 增加一次临时磁盘写入，部署需计入并发上传的临时空间预算。
|
||||
|
||||
### D6. 镜像、诊断与兼容性证据
|
||||
|
||||
镜像必须固定经验证版本与 digest，记录支持的 CPU 架构、许可证、来源和制品扫描结果。镜像同步使用产品独立且不可变的目标标签；保留已有 `minio:latest` 兼容映射，不用 RustFS 接管这个别名。`scripts/mirror-runtime-images.sh` 遇到已有目标会跳过，若需要刷新仓库必须显式验证目标 digest，不能靠改清单完成。
|
||||
|
||||
不新增公开可写健康接口。新增显式调用的存储诊断脚本/测试入口，经配置绑定真实 `ObjectStorageService`，在随机专用前缀执行 put/get/head/list/delete 与签名验证，设置整体超时，使用 finally 清理；清理失败也返回非零并报告脱敏的残留键。只删除本次生成的确切对象，不按宽泛前缀清空桶。普通 liveness 不执行远程写入。
|
||||
|
||||
共享契约测试用同一应用构件，按后端注入配置。真实测试失败不能因凭证或 Docker 缺失被计为通过；默认单元测试可不启动存储，但被选为交付门禁的集成任务缺环境必须失败或明确阻断。负向权限和批量部分失败先用确定性模拟响应验证错误分支，再在实际产品可控权限条件下验收。满批真实删除遇短暂逐项失败时，适配层必须抛错；契约测试记录这条失败证据，最多进行 3 次调用方恢复，最后独立列举确认对象不存在。生产适配器不新增一套隐式重试策略，沿用现有业务补偿。
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
- [供应商声称 S3 兼容但有操作差异] → 只发布被测版本与契约范围，不能宣称任意版本全兼容。
|
||||
- [签名可生成但目标不可达] → 从真实浏览器/CLI 和 Worker 网络分别调用，不只断言 URL 字符串。
|
||||
- [最小权限不足或过宽] → 按实际对象、列举和批量删除操作配置桶权限；初始化权限不交给业务进程。
|
||||
- [故障时静默回退导致数据分散] → 不增加 fallback，沿用明确存储异常与脱敏日志。
|
||||
- [对象公开直读绕过业务权限] → 桶保持私有；检查匿名直读拒绝与签名有效期；临时签名持有者有效期内可读是现有行为。
|
||||
- [单机存储失败导致不可用] → 明确持久卷、备份恢复和容量监控，不将本方案宣称为 HA。
|
||||
- [更多配置增加维护成本] → 不改核心业务接口，使用共享测试与单一协议操作手册，产品特有内容限于部署附录。
|
||||
|
||||
## Migration Reference and Boundaries
|
||||
|
||||
以下保留迁移安全要求，作为操作参考，不构成已支持的生产迁移方案。已完成的证据仅为独立测试桶中 RustFS → Silo → RustFS 的小样本演练（正向 134 个对象、回退 135 个对象）；没有旧 MinIO/local 真实迁移或大规模生产迁移验收。生产实例需要另行设计和演练，不能用新部署接入通过或小样本结果替代。
|
||||
|
||||
1. 清点现有 backend、桶、对象键、元数据和加密方式；备份对象及部署配置，演练恢复。原始数据卷不能直接挂给 RustFS；Silo 即使声称格式兼容，也不默认做原地切换。
|
||||
2. 准备独立目标桶和业务账号，在专用前缀跑契约探针；逻辑复制对象，保留 key、Content-Type 及必要元数据。跨加密后端通过授权逻辑读取再写入，不能拷贝内部加密文件。
|
||||
3. 记录对象清单、大小及内容校验结果，不能把 ETag 一律当 MD5；迁移工具以实际能力选择，记录版本与命令，不预设某个工具已支持全部元数据。
|
||||
4. 进入维护窗口，暂停发布、头像申请/处理、Suite 写入、GitHub 同步、过期清理等所有存储写入和删除；排空在途任务。对已签发 PUT 链接等待过期或以旧存储权限撤销写入，防止最终同步后仍有旧端点写入。
|
||||
5. 在明确冻结点执行最终同步，协调新增、覆盖和删除差异；旧源只读保留。GET 旧签名仍可能在有效期内指向旧端点，应保留可读访问或明确短暂中断窗口。
|
||||
6. 切换部署配置并重建受影响服务，验收元数据、实际下载、头像审核、Suite、同步上传/入库/清理。结果符合场景矩阵才恢复写入。
|
||||
7. 恢复写入前可切回未变更旧源；恢复写入后回退必须再次停写，将新后端新增、覆盖及删除差异协调回旧源并校验，不能仅改 endpoint。保留期由部署方按备份政策确定，确认恢复能力后才单独授权清理旧源。
|
||||
|
||||
无需 SQL 迁移或 CHECK 约束。文件系统 local → S3 也需逻辑复制，不能只修改 provider。
|
||||
|
||||
## 验收矩阵与质量门禁
|
||||
|
||||
下面是验收要求；实际执行结果在 tasks.md 和交付验收记录中逐项关联，未执行项不能借文档校验声明通过。
|
||||
|
||||
| 需求 | 验收场景和独立断言 | 阶段 / 阻断条件 |
|
||||
|---|---|---|
|
||||
| S3-01 | 同一应用对 RustFS/Silo 运行共享套件；local、外部 S3 配置兼容 | 集成；任一新增产品失败不得声明支持 |
|
||||
| S3-02 | 随机二进制与中文/空格键精确读回；元数据比对；缺失对象；单删/批删/部分失败；分页及上限完整标志 | 单元＋真实集成；字节或错误判定不一致阻断 |
|
||||
| S3-03 | 实际请求 GET/PUT；必需头；公开/内部网络；过期/篡改拒绝 | 集成；只生成 URL 不算通过 |
|
||||
| S3-04 | 错密钥、拒绝权限、断网超时；匿名私有对象拒绝；无本地 fallback；日志无凭证 | 负向集成；越权、静默成功或泄密阻断 |
|
||||
| DEP-01 | 两个文件分别渲染与启动；双选预检拒绝；独立卷；外部 S3 无本地服务 | 配置＋真实部署；配置混合或卷复用阻断 |
|
||||
| DEP-02 | 初始化幂等、root 不进入应用；Helm Secret 模板；Worker allowed-host 与容量预算 | 配置＋集成；权限越界或同步失败阻断 |
|
||||
| DEP-03 | 固定 digest/架构与镜像扫描证据；诊断成功清理、失败退出；旧镜像别名回归 | 交付；缺不可变制品或存在未处理阻断漏洞不发布 |
|
||||
| DEP-04 | 带覆盖/删除/在途 PUT 的小样本迁移与回退，校验逻辑对象差异 | 迁移演练；丢对象/复活已删除内容阻断 |
|
||||
| 跨能力 | 发布 ZIP→审核→下载字节一致；头像私有待审→批准→移除；Suite 包；GitHub staging PUT→导入→清理 | E2E；只用 SDK 测试不能替代业务路径 |
|
||||
|
||||
需求门禁：本变更行为和范围明确；设计门禁：接口、网络、凭证、停写及回退边界一致；实现门禁：模块测试/配置回归通过；交付门禁：两种产品真实契约、关键业务链和迁移演练有证据。某项失败先修复并重跑受影响集合，不通过不得升级为“可发布”。保留支持矩阵，包含失败、未执行和限制，不只填写通过数量。
|
||||
|
||||
## Open Questions
|
||||
|
||||
以下项目不改变架构与行为契约，但进入交付前必须补齐证据：
|
||||
|
||||
- 已按官方 manifest 固定 RustFS 1.0.3、Silo RELEASE.2026-10-18T00-00-00Z（含 amd64/arm64），清单见 `docs/object-storage.md`。实际 amd64 镜像与初始化权限、扫描结果已核验，Silo 单条 legacy 模块告警的源码和行为裁定见用户手册；不把 manifest 核验扩大为 arm64 运行支持。
|
||||
- 迁移参考已记录小样本演练；实际旧 MinIO/local 来源、数据规模、元数据、版本历史、锁及加密场景没有生产迁移证据，不列入本次接入支持范围。
|
||||
|
||||
## 参考与证据边界
|
||||
|
||||
- [Issue #937](https://github.com/iflytek/skillhub/issues/937)：问题来源；讨论不等于产品已验收。
|
||||
- [MinIO 源仓库](https://github.com/minio/minio)：维护状态参考，具体迁移仍需部署方决定。
|
||||
- [RustFS 官方 S3 兼容说明](https://docs.rustfs.com/en/reference/s3-compatibility)：厂商协议覆盖说明，不是 SkillHub 实测结果。
|
||||
- [RustFS 官方发布](https://github.com/rustfs/rustfs/releases)：实施时选择并核验制品，不能使用可变 latest 作为验收基线。
|
||||
- [PGSTY Silo 源仓库](https://github.com/pgsty/silo)：独立社区分支来源；格式兼容声明不替代备份恢复测试。
|
||||
|
||||
实现采用本次固定镜像及同一工作区应用构件验证，结果见任务状态与交付记录；不继承历史预览、其他应用或厂商测试的通过结论。
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
## Why
|
||||
|
||||
[#937](https://github.com/iflytek/skillhub/issues/937) 希望提供 MinIO 之外的对象存储选择。SkillHub 已有 S3 协议适配层，缺的是可复用的交付配置、真实兼容性证据和迁移操作边界。
|
||||
|
||||
## What Changes
|
||||
|
||||
- 保持 `local` / `s3` 配置契约，通过同一 S3 接口接入 RustFS、PGSTY Silo 或外部 S3 服务。
|
||||
- 为正式 Compose 栈增加可选 RustFS / Silo 叠加文件，使用独立数据卷，不静默替换现有部署。
|
||||
- 明确内外网端点、预签名、业务凭证、桶初始化、GitHub 同步 Worker 网络和 Helm 外部存储配置。
|
||||
- 增加共享真实契约验收、迁移风险及停写和回退参考，以及镜像供应链说明；新部署接入验证不构成生产迁移支持。
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
|
||||
- `s3-compatible-storage`: 不同 S3 实现下的对象读写、签名、权限和错误行为契约。
|
||||
- `object-storage-deployment`: 可选部署、配置诊断、兼容性声明与迁移参考的安全边界。
|
||||
|
||||
### Modified Capabilities
|
||||
|
||||
无。当前 `openspec/specs/` 尚无相应能力正文；新增规格记录本次目标，不代表重写全部已有实现。
|
||||
|
||||
## Impact
|
||||
|
||||
主要影响 `server/skillhub-storage`、Compose、配置校验、镜像清单、Helm 示例和部署手册。验证发布下载、头像审核对象、Suite 包和 GitHub 同步调用链；业务模块不引入供应商 SDK。
|
||||
|
||||
不改数据库、公开 API、技能生命周期或权限策略，不增加后台存储配置页面。实现与本地验收状态见 tasks.md；最终提交、CI 和发布门禁由对应 PR 记录。
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
## Purpose
|
||||
|
||||
为部署者提供可选的 S3 兼容存储交付方式、配置检查及可追溯的验收依据，并明确已有对象迁移参考的安全要求。新部署接入验证不代表支持生产迁移。新增选项不应使原有部署被自动替换，也不能把配置更新误认为数据已完成迁移。
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: DEP-01 存储选项独立部署
|
||||
交付物 SHALL 提供 RustFS 和 Silo 可选 Compose 叠加配置，每次仅启用一个本地存储选项，数据卷独立；外部 S3 不要求启动本地存储。Helm SHALL 保留外部存储接入，不强制安装存储子 Chart。
|
||||
|
||||
#### Scenario: 选择本地存储
|
||||
- **WHEN** 部署者按手册选择一种存储叠加配置
|
||||
- **THEN** 应用通过稳定的内部服务地址连接所选服务，另一产品不会启动或共用数据目录
|
||||
|
||||
#### Scenario: 选择冲突
|
||||
- **WHEN** 部署者试图同时选择两种本地存储选项
|
||||
- **THEN** 交付预检明确拒绝此组合，不能静默选择其中之一
|
||||
|
||||
### Requirement: DEP-02 部署配置与凭证边界
|
||||
端点、桶、凭证和产品 SHALL 由部署配置决定，变更通过重建相关服务生效，不进入系统设置表或后台热切换。示例 SHALL 区分初始化管理员凭证与应用业务凭证，默认不依赖应用自动建桶。
|
||||
|
||||
#### Scenario: 新实例初始化
|
||||
- **WHEN** 部署者按示例准备桶和具备所需对象权限的业务账号
|
||||
- **THEN** 应用无需存储 root 凭证即可运行，文档指出前缀统计所需的列举权限
|
||||
|
||||
#### Scenario: Worker 接入
|
||||
- **WHEN** 启用 S3 模式 GitHub 同步
|
||||
- **THEN** 手册要求 Worker 可达上传端点、允许目标主机并配置有效 staging 容量预算，完整导入与清理能够验收
|
||||
|
||||
### Requirement: DEP-03 可验证的交付与兼容声明
|
||||
交付 SHALL 记录应用修订、被测产品版本、不可变镜像摘要、架构和验收结果。配置检查与真实对象诊断必须区分，服务存活不能替代真实操作。旧镜像别名 SHALL 保持兼容，映射文件修改不能视为目标镜像已刷新。
|
||||
|
||||
#### Scenario: 发布接入验证矩阵
|
||||
- **WHEN** 宣布某个产品版本通过新部署接入验证
|
||||
- **THEN** 共享契约和关键业务路径有真实结果，未执行项及限制被列出，阻断用例失败时不能声明通过
|
||||
- **AND** 不将接入结果扩大为产品最优推荐、长期稳定性、大规模、高可用或生产迁移支持
|
||||
|
||||
#### Scenario: 运行诊断
|
||||
- **WHEN** 操作者执行存储诊断
|
||||
- **THEN** 在专用前缀进行有时限的真实对象操作并清理，失败非零退出，不泄露密钥或触碰业务对象
|
||||
|
||||
### Requirement: DEP-04 逻辑迁移与回退参考
|
||||
迁移参考 SHALL 明确本次不宣称生产迁移支持；仅有独立测试桶小样本演练，未验证旧 MinIO/local 真实迁移或大规模生产迁移。迁移参考 SHALL 要求保留对象键和必要元数据、校验内容、暂停相关写入并完成最终同步后切换。不同产品不得默认复用原始数据卷；切换配置不代表迁移完成。
|
||||
|
||||
#### Scenario: 已有数据切换
|
||||
- **WHEN** 实例从现有存储迁往另一产品
|
||||
- **THEN** 先备份复制，再停写并处理在途任务，校验最终差异，验证发布下载和同步后才恢复写入
|
||||
|
||||
#### Scenario: 接受新写入后回退
|
||||
- **WHEN** 新后端已经接受业务写入但需要回退
|
||||
- **THEN** 指南要求再次停写并协调新增和删除差异，不能仅切回旧端点导致对象丢失或已删除数据复活
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
## Purpose
|
||||
|
||||
使部署方可以通过既有 S3 配置选择不同对象存储产品，同时维持技能文件、审核图片、下载链接和同步任务的可观察行为。支持范围由真实契约验收证明，不以供应商的兼容性宣传替代应用验证。
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: S3-01 统一协议配置
|
||||
系统 SHALL 使用既有 `s3` 配置连接通过验收的 S3 兼容服务，保留 `local` 模式，不要求业务调用者按存储品牌选择接口。
|
||||
|
||||
#### Scenario: 更换空桶目标
|
||||
- **WHEN** 新实例配置已准备的 RustFS 或 Silo 桶及有效业务凭证
|
||||
- **THEN** 相同应用版本通过相同接口完成对象操作，无需修改技能元数据或指定品牌 provider
|
||||
|
||||
#### Scenario: 原有部署兼容
|
||||
- **WHEN** 部署方沿用有效的 local 或外部 S3 配置
|
||||
- **THEN** 原模式继续可用，不因新增选项自动更换后端
|
||||
|
||||
### Requirement: S3-02 对象操作完整性
|
||||
系统 SHALL 保持对象键、字节、长度和内容类型的读写语义；单个和批量删除均不得将后端报告的失败当作全部成功。前缀统计 SHALL 明确数据是否完整。
|
||||
|
||||
#### Scenario: 特殊对象键往返
|
||||
- **WHEN** 写入包含中文、空格和 URL 保留字符的合法对象键并读取元数据和内容
|
||||
- **THEN** 键与内容不发生二次编码或截断，字节、长度和内容类型与输入一致
|
||||
|
||||
#### Scenario: 流式技能上传
|
||||
- **WHEN** 上传的合法技能包包含超过 128 KiB 的文件并通过现有流式发布处理
|
||||
- **THEN** 写入和下载字节与原文件一致，不因请求体重读而丢失内容或持续超时
|
||||
|
||||
#### Scenario: 私有上传暂存和失败清理
|
||||
- **WHEN** S3 上传为签名和重试暂存请求体，或在暂存读取期间发生 I/O 或运行时异常
|
||||
- **THEN** 暂存期间不放宽文件创建时的私有访问权限，在支持 POSIX 权限的平台保持仅运行身份读写；读取失败后清理尚未返回的暂存文件，并保留原始失败供调用方识别
|
||||
|
||||
#### Scenario: 批量删除部分失败
|
||||
- **WHEN** 后端在批量删除响应中报告部分对象失败
|
||||
- **THEN** 调用方能够识别失败，不得到全部删除成功的结论
|
||||
|
||||
#### Scenario: 前缀统计截断
|
||||
- **WHEN** 对象数量超过一次分页或扫描上限
|
||||
- **THEN** 继续分页或明确报告统计不完整，不将部分数量声明为总量
|
||||
|
||||
### Requirement: S3-03 端点与预签名契约
|
||||
系统 SHALL 为实际访问方可达的端点生成预签名 URL，保留下载公开端点及同步上传内部端点的分工；不得通过事后改写已签名 URL 主机解决可达性。
|
||||
|
||||
#### Scenario: 内外网端点分离
|
||||
- **WHEN** 后端使用内部端点，浏览器访问配置的公开端点，Worker 可达内部端点
|
||||
- **THEN** 下载签名指向公开端点且读取成功，Worker 上传签名指向内部端点且上传成功
|
||||
|
||||
#### Scenario: 签名失效
|
||||
- **WHEN** 使用过期、篡改签名或缺失签名要求的请求头
|
||||
- **THEN** 存储拒绝请求，不返回对象内容或写入成功
|
||||
|
||||
### Requirement: S3-04 隔离与失败可见
|
||||
系统 SHALL 维持现有私有对象与业务授权边界；凭证、网络和存储错误必须可观察，不得静默写入其他后端。凭证不得出现在公开链接、日志或报告中。
|
||||
|
||||
#### Scenario: 私有对象访问
|
||||
- **WHEN** 匿名请求直接读取私有桶对象且无有效签名
|
||||
- **THEN** 请求被拒绝，正常业务授权仍可生成限时下载链接
|
||||
|
||||
#### Scenario: 后端不可用
|
||||
- **WHEN** 业务凭证无效、权限不足或目标网络不可达
|
||||
- **THEN** 操作失败并提供可定位且脱敏的错误，不转写本地目录、不返回伪造成功
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
## 1. 制品与回归基线
|
||||
|
||||
- [x] 1.1 读取本方案与部署/存储测试规范，确认实现 diff 只覆盖既定边界；通过需求到测试映射复核，不引入新 provider、数据库或公开 API。
|
||||
- [x] 1.2 核验 RustFS/Silo 发行版、官方镜像来源、digest、CPU 架构、许可证与扫描结果；产出可追溯制品表，阻断问题未处理不得进入交付。
|
||||
- [x] 1.3 先补共享契约与确定性失败用例，覆盖 S3-01～04；确认能发现错误字节、删除部分失败和无效签名被接受等缺陷，记录现有偏差。
|
||||
|
||||
## 2. 协议与诊断
|
||||
|
||||
- [x] 2.1 按 1.3 的真实缺口局部修复 S3 操作，保留接口与可选能力语义；运行 storage 模块测试及受影响调用方回归，无缺口则记录无需改动。
|
||||
- [x] 2.2 提供有整体超时的显式对象诊断入口；成功清理、认证失败、超时、清理失败均有断言，失败非零退出且不触及业务对象/输出凭证。
|
||||
|
||||
## 3. 部署交付
|
||||
|
||||
- [x] 3.1 新增 RustFS 叠加文件、独立卷和 env 示例；用固定制品运行配置渲染、健康、重启持久化及初始化幂等检查，应用不持有 root 凭证。
|
||||
- [x] 3.2 新增 Silo 叠加文件、独立卷和 env 示例；执行同等检查并回归旧开发服务及 minio 镜像别名，禁止两个产品共用原始卷。
|
||||
- [x] 3.3 增加薄包装预检，明确输入文件列表、拒绝双叠加、透传合法组合；用无服务启动的 shell 测试覆盖单选、双选、外部 S3 与非法参数。
|
||||
- [x] 3.4 按需扩展现有配置校验并补测试，核对变量绑定/透传/默认值；local、外部凭证链、静态凭证、路径寻址和非法 URL 有确定结果。
|
||||
- [x] 3.5 提供 Helm 外部存储与 existingSecret 示例；核对模板键名与优先级，通过 lint/render，确认不安装存储子 Chart、不输出 Secret 明文。
|
||||
- [x] 3.6 加入产品独立不可变镜像映射并验证实际目标 digest；保留 minio 别名，测试目标已存在跳过的行为，报告未执行的仓库同步。
|
||||
|
||||
## 4. 真实验收
|
||||
|
||||
- [x] 4.1 在隔离目标分别对 RustFS/Silo 运行相同应用共享契约，包括分页、权限、签名和失败路径；保存版本、命令、结果与残留清理证据，不将缺环境或 skip 计为通过。
|
||||
- [x] 4.2 按 design.md 矩阵跑发布审核下载、头像对象、Suite 和 GitHub 同步完整路径；验证浏览器/CLI 公开端点与 Worker 内部端点，结果映射回需求编号。
|
||||
- [x] 4.3 用覆盖、删除和在途 PUT 样本演练迁移、冻结、最终同步及恢复写入后的回退；对象键、内容与元数据比对通过，不把 ETag 一律当内容哈希。
|
||||
|
||||
## 5. 用户文档与收尾
|
||||
|
||||
- [x] 5.1 更新部署手册、Kubernetes 示例和同步运维入口，写明初始化/域名/凭证/互斥/停写/回退命令及副作用；在干净环境按手册复现，未实测项显式标注。
|
||||
- [x] 5.2 汇总支持矩阵与脱敏验收报告，逐项关联 S3-01～04、DEP-01～04 和跨能力场景；未通过项目阻断对应支持声明。
|
||||
- [x] 5.3 执行 OpenSpec 严格校验、变更范围核查及一次关键阶段设计/实现一致性 Review；记录结果后再进入用户授权范围内的 PR 流程，不自动发布或部署生产。
|
||||
|
||||
## 执行摘要
|
||||
|
||||
- 2026-10-21：实现与本地运行验收。RustFS 1.0.3 和 Silo 2026-10-18 使用同一应用构件，实际运行架构为 amd64；镜像清单另核验 arm64 条目，不将其写成 arm64 运行通过。
|
||||
- 存储 36 项单元测试、领域 27 项与应用调用方 34 项回归通过。默认跳过的两个显式集成用例已在两产品上分别开启运行，无 skip 冒充通过。
|
||||
- 两产品均验证了文件流、签名、分页、删除、发布审核下载、头像、Suite、Go 内部网络上传与暂存清理；故意错误凭证、桶外权限及网络不可达均拒绝且不返回伪造成功。
|
||||
- RustFS → Silo 冻结点逐项比对 134 对象，Silo → RustFS 回退冻结点比对 135 对象，包含新写入、覆盖、删除与在途 PUT 排空；原始卷不跨产品复用。
|
||||
- 独立 Review 指出的新 Silo 映射目标标签已修复并增加精确配对断言；评审剩余 blocker/high/medium 均为 0。该结论绑定工作区，不替代最终提交与 CI 门禁。
|
||||
- 新 Silo 镜像的单条 legacy MinIO 模块告警按源码 commit、二进制版本和真实有界 CSV 拒绝用例裁定为已修补，保留原始扫描结果，不全局忽略告警。
|
||||
- 远端镜像同步、真实 K8s、外网 TLS/代理未执行；本地 JAR 构建加官方 Dockerfile runtime 阶段镜像已验证，不宣称完整多阶段 Docker 构建或 PR CI 通过。本地验收记录截止时尚未提交、推送或创建 PR；最终提交、PR 与 CI 状态由后续交付流程记录，不将本地结果视为合并门禁通过。
|
||||
- 本地脱敏验收记录单独交付，不将运行日志、凭证、私有路径或工具状态纳入 Git。
|
||||
|
||||
## 6. PR #996 深度 Review 跟进
|
||||
|
||||
- [x] 6.1 修复 R1/R2：暂存期间保持私有权限，读取 I/O／运行时异常可靠清理并保留主异常；新增测试先在旧实现失败，再验证修复通过。
|
||||
- [x] 6.2 修复 R3：允许 profile 前置参数下的 logs -f，仍拒绝全局额外文件注入，并覆盖代表性合法／非法参数。
|
||||
- [x] 6.3 修复 R4/R5：更新 Proposal 状态边界及 Silo 包装器命令，明确不具备持久化补偿的 best-effort 清理限制。
|
||||
- [x] 6.4 运行修复后的相关回归、实际存储及受影响上传验证，独立复核修复并整理交付摘要；推送后在 PR 评论汇总结果和未验证边界，不自动合并。
|
||||
|
||||
本轮修复验证：新增 9 项暂存权限与清理测试，旧实现先复现 4 项失败；修复后 106 项回归通过（普通运行另有 2 个 opt-in 集成跳过，两产品已分别显式运行各 2 项通过）。实际 Java 21 非 root 运行环境验证了暂存期间 0600 与运行时异常后无残留；重建应用后认证 18/18、303010 字节技能包发布审核下载和头像流程通过。profile 日志跟随与额外文件拒绝回归通过。迁移／高可用等未受本轮修改影响的范围没有重新扩大验证声明。
|
||||
63
scripts/storage-compose.sh
Executable file
63
scripts/storage-compose.sh
Executable file
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env bash
|
||||
# Compose entry point that makes the optional local storage selection exclusive.
|
||||
set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
usage() {
|
||||
echo "Usage: $0 --storage rustfs|silo|external [--env-file FILE] [--project-name NAME] -- COMPOSE_ARGS..."
|
||||
}
|
||||
fail() { echo "ERROR: $*" >&2; exit 2; }
|
||||
storage=""
|
||||
env_file=".env.release"
|
||||
project_args=()
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--storage)
|
||||
[[ $# -ge 2 && -z "$storage" ]] || fail "Select exactly one storage backend"
|
||||
storage="$2"; shift 2 ;;
|
||||
--env-file)
|
||||
[[ $# -ge 2 && -n "$2" ]] || fail "Missing env file"
|
||||
env_file="$2"; shift 2 ;;
|
||||
--project-name)
|
||||
[[ $# -ge 2 && -n "$2" ]] || fail "Missing project name"
|
||||
project_args=(--project-name "$2"); shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--) shift; break ;;
|
||||
*) fail "Unknown option: $1" ;;
|
||||
esac
|
||||
done
|
||||
case "$storage" in rustfs|silo|external) ;; *) fail "Select rustfs, silo or external" ;; esac
|
||||
(($#)) || fail "Missing Compose command"
|
||||
# Do not allow additional global files to silently merge incompatible storage services.
|
||||
compose_command=""
|
||||
global_value_pending=false
|
||||
for arg in "$@"; do
|
||||
# Only standalone -f after the actual logs subcommand means follow. Global
|
||||
# option values (for example --profile logs) must not be mistaken for it.
|
||||
case "$arg" in
|
||||
-f)
|
||||
[[ "$compose_command" == logs && "$global_value_pending" == false ]] ||
|
||||
fail "Compose file and env selection must use this script's options"
|
||||
continue ;;
|
||||
--file|-f?*|--file=*|--env-file|--env-file=*|--project-directory|--project-directory=*)
|
||||
fail "Compose file and env selection must use this script's options" ;;
|
||||
esac
|
||||
[[ -z "$compose_command" ]] || continue
|
||||
if [[ "$global_value_pending" == true ]]; then
|
||||
global_value_pending=false
|
||||
continue
|
||||
fi
|
||||
# These Compose global options consume the next argument; their =value forms
|
||||
# are already single tokens. Leave command-specific options to Compose.
|
||||
case "$arg" in
|
||||
--ansi|--parallel|--profile|--progress|-p|--project-name) global_value_pending=true ;;
|
||||
-*) ;;
|
||||
*) compose_command="$arg" ;;
|
||||
esac
|
||||
done
|
||||
cd "$REPO_ROOT"
|
||||
[[ -f "$env_file" ]] || fail "Env file not found"
|
||||
files=(-f compose.release.yml)
|
||||
if [[ "$storage" != external ]]; then
|
||||
files+=(-f "compose.storage.$storage.yml")
|
||||
fi
|
||||
exec docker compose --env-file "$env_file" "${project_args[@]}" "${files[@]}" "$@"
|
||||
33
scripts/storage-diagnostics.sh
Executable file
33
scripts/storage-diagnostics.sh
Executable file
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/env bash
|
||||
# Source-checkout diagnostic; writes only a random diagnostic prefix in a prepared bucket.
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
full=false
|
||||
deadline=180s
|
||||
if [[ "${1:-}" == --full && $# -eq 1 ]]; then
|
||||
full=true
|
||||
deadline=420s
|
||||
elif (($#)); then
|
||||
echo "Usage: $0 [--full]" >&2
|
||||
exit 2
|
||||
fi
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
command -v timeout >/dev/null || { echo "ERROR: GNU timeout is required" >&2; exit 2; }
|
||||
: "${SKILLHUB_STORAGE_S3_ENDPOINT:?A reachable S3 endpoint is required}"
|
||||
: "${SKILLHUB_STORAGE_S3_BUCKET:?A prepared private bucket is required}"
|
||||
log_dir="${SKILLHUB_STORAGE_DIAGNOSTICS_LOG_DIR:-$REPO_ROOT/.dev/storage-diagnostics}"
|
||||
mkdir -p "$log_dir"
|
||||
log_file="$(mktemp "$log_dir/run-XXXXXXXX.log")"
|
||||
cd "$REPO_ROOT/server"
|
||||
# Raw SDK/network errors may contain signed URLs; retain them only in the private log.
|
||||
if timeout --signal=TERM --kill-after=15s "$deadline" ./mvnw -pl skillhub-storage -am \
|
||||
-Dtest=S3StorageContractTest -Dsurefire.failIfNoSpecifiedTests=false \
|
||||
-Dskillhub.storage.contract=true -Dskillhub.storage.contract.full="$full" test >"$log_file" 2>&1; then
|
||||
echo "Storage diagnostic passed (objects, metadata, signatures, privacy, deletion)."
|
||||
echo "Private log: $log_file"
|
||||
else
|
||||
result=$?
|
||||
echo "Storage diagnostic failed (exit $result). Check the private log; do not publish it unredacted." >&2
|
||||
echo "Private log: $log_file" >&2
|
||||
exit "$result"
|
||||
fi
|
||||
26
scripts/tests/mirror-storage-images-test.sh
Executable file
26
scripts/tests/mirror-storage-images-test.sh
Executable file
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
mkdir "$work/bin"
|
||||
cat >"$work/bin/docker" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$MIRROR_COMMAND_LOG"
|
||||
case "$*" in
|
||||
'buildx imagetools inspect '*) [[ "${TARGET_EXISTS:-false}" == true ]] ;;
|
||||
'buildx imagetools create '*) exit 0 ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
STUB
|
||||
chmod +x "$work/bin/docker"
|
||||
export PATH="$work/bin:$PATH" MIRROR_COMMAND_LOG="$work/commands"
|
||||
export MIRROR_REGISTRY=registry.example.test MIRROR_NAMESPACE=skillhub
|
||||
printf '%s\n' 'docker.io/rustfs/rustfs@sha256:abc rustfs:immutable-test' >"$work/images"
|
||||
TARGET_EXISTS=true bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output"
|
||||
if grep -q 'imagetools create' "$work/commands"; then echo 'FAIL: existing target was refreshed' >&2; exit 1; fi
|
||||
TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output"
|
||||
grep -q 'imagetools create --tag registry.example.test/skillhub/rustfs:immutable-test docker.io/rustfs/rustfs@sha256:abc' "$work/commands"
|
||||
grep -q '^docker.io/pgsty/silo:RELEASE.2026-09-16T00-00-00Z minio:latest$' "$REPO_ROOT/deploy/runtime-mirror-images.txt"
|
||||
grep -Fxq 'docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7 silo:20261018-2636fc4155d9' "$REPO_ROOT/deploy/runtime-mirror-images.txt"
|
||||
echo 'Storage mirror skip and legacy alias tests passed'
|
||||
52
scripts/tests/storage-compose-render-test.sh
Executable file
52
scripts/tests/storage-compose-render-test.sh
Executable file
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
python3 - "$REPO_ROOT" <<'CHECK'
|
||||
import json, os, pathlib, subprocess, sys, tempfile, urllib.parse
|
||||
root=pathlib.Path(sys.argv[1])
|
||||
# Fixed non-production fixture values; no service is started by compose config.
|
||||
env={**os.environ,
|
||||
'SKILLHUB_OBJECT_STORAGE_ROOT_USER':'fixture-storage-admin',
|
||||
'SKILLHUB_OBJECT_STORAGE_ROOT_PASSWORD':'fixture-storage-admin-secret',
|
||||
'SKILLHUB_STORAGE_PROVIDER':'s3',
|
||||
'SKILLHUB_STORAGE_S3_ACCESS_KEY':'fixture-business-user',
|
||||
'SKILLHUB_STORAGE_S3_SECRET_KEY':'fixture-business-secret',
|
||||
'SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT':'https://storage.example.test',
|
||||
'SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS':'object-storage:9000',
|
||||
'SKILLHUB_DOWNLOAD_ANON_COOKIE_SECRET':'fixture-download-secret-at-least-32-characters'}
|
||||
for key in ['RUSTFS_IMAGE','SILO_IMAGE']:
|
||||
env.pop(key,None)
|
||||
with tempfile.TemporaryDirectory() as work:
|
||||
config=pathlib.Path(work)/'fixture.env'
|
||||
config.touch()
|
||||
for product in ['rustfs','silo','external']:
|
||||
command=[str(root/'scripts/storage-compose.sh'),'--storage',product,'--env-file',str(config),
|
||||
'--project-name','storage-render-contract','--','--profile','github-sync','config','--format','json']
|
||||
data=json.loads(subprocess.check_output(command,cwd=root,env=env))
|
||||
services=data['services']
|
||||
server=services['server']['environment']
|
||||
assert server['SKILLHUB_STORAGE_S3_ACCESS_KEY']=='fixture-business-user'
|
||||
assert server['SKILLHUB_STORAGE_S3_SECRET_KEY']=='fixture-business-secret'
|
||||
assert 'fixture-storage-admin-secret' not in json.dumps(server)
|
||||
assert not any('ROOT_PASSWORD' in key or 'ROOT_USER' in key for key in server)
|
||||
if product=='external':
|
||||
assert 'object-storage' not in services and 'object-storage-permissions' not in services
|
||||
continue
|
||||
storage=services['object-storage']
|
||||
assert '@sha256:' in storage['image'] and len(storage['image'].rsplit(':',1)[1])==64
|
||||
assert server['SKILLHUB_STORAGE_PROVIDER']=='s3'
|
||||
assert server['SKILLHUB_STORAGE_S3_ENDPOINT']=='http://object-storage:9000'
|
||||
assert server['SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE']=='true'
|
||||
assert server['SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET']=='false'
|
||||
assert storage['volumes'][0]['source']==product+'_data'
|
||||
other='silo_data' if product=='rustfs' else 'rustfs_data'
|
||||
assert other not in data['volumes']
|
||||
assert services['github-sync']['environment']['SKILLHUB_GITHUB_SYNC_ALLOWED_S3_HOSTS']==urllib.parse.urlparse(server['SKILLHUB_STORAGE_S3_ENDPOINT']).netloc
|
||||
assert services['server']['depends_on']['object-storage']['condition']=='service_healthy'
|
||||
if product=='rustfs':
|
||||
initializer=services['object-storage-permissions']
|
||||
assert initializer['network_mode']=='none' and initializer['read_only']
|
||||
assert initializer['entrypoint']==['chown','10001:10001','/data']
|
||||
assert 'environment' not in initializer or not initializer['environment']
|
||||
print('Storage Compose rendering, isolated volumes, business credentials and Worker endpoint tests passed')
|
||||
CHECK
|
||||
58
scripts/tests/storage-compose-test.sh
Executable file
58
scripts/tests/storage-compose-test.sh
Executable file
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
mkdir "$work/bin"
|
||||
cat >"$work/bin/docker" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$@" >"$DOCKER_ARGS_FILE"
|
||||
STUB
|
||||
chmod +x "$work/bin/docker"
|
||||
export PATH="$work/bin:$PATH" DOCKER_ARGS_FILE="$work/args"
|
||||
touch "$work/env"
|
||||
run() { "$REPO_ROOT/scripts/storage-compose.sh" --env-file "$work/env" "$@"; }
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
run --storage rustfs --project-name storage-contract -- config
|
||||
[[ "$(cat "$work/args")" == *compose.storage.rustfs.yml* ]] || fail "RustFS overlay missing"
|
||||
[[ "$(cat "$work/args")" != *compose.storage.silo.yml* ]] || fail "Unexpected Silo"
|
||||
run --storage silo -- config
|
||||
[[ "$(cat "$work/args")" == *compose.storage.silo.yml* ]] || fail "Silo overlay missing"
|
||||
run --storage external -- --profile github-sync config
|
||||
[[ "$(cat "$work/args")" != *compose.storage.* ]] || fail "External storage started a local backend"
|
||||
run --storage rustfs -- logs -f
|
||||
[[ "$(cat "$work/args")" == *logs* ]] || fail "Log command not forwarded"
|
||||
run --storage rustfs -- --profile github-sync logs -f github-sync
|
||||
[[ "$(cat "$work/args")" == *$'--profile\ngithub-sync\nlogs\n-f\ngithub-sync' ]] || fail "Profile log follow command not forwarded"
|
||||
run --storage rustfs -- --profile github-sync logs --follow github-sync
|
||||
[[ "$(cat "$work/args")" == *$'logs\n--follow\ngithub-sync' ]] || fail "Long log follow command not forwarded"
|
||||
run --storage rustfs -- --ansi never --profile=github-sync --parallel 2 --progress=plain logs -f github-sync
|
||||
[[ "$(cat "$work/args")" == *$'logs\n-f\ngithub-sync' ]] || fail "Global options obscured log command"
|
||||
for args in 'rustfs --storage silo' 'unknown'; do
|
||||
# Intentional word splitting for these fixed malformed input fixtures.
|
||||
if run --storage $args -- config >/dev/null 2>&1; then fail "Accepted invalid selection"; fi
|
||||
done
|
||||
if run --storage rustfs -- -f compose.storage.silo.yml config >/dev/null 2>&1; then fail "Accepted file injection"; fi
|
||||
if run --storage rustfs -- --file=compose.storage.silo.yml config >/dev/null 2>&1; then fail "Accepted file injection"; fi
|
||||
reject_compose_args() {
|
||||
rm -f "$work/args"
|
||||
if run --storage rustfs -- "$@" >"$work/rejected" 2>&1; then
|
||||
fail "Accepted forbidden Compose selection: $*"
|
||||
fi
|
||||
[[ ! -e "$work/args" ]] || fail "Forwarded forbidden selection to Docker"
|
||||
}
|
||||
reject_compose_args --profile github-sync -f compose.storage.silo.yml config
|
||||
reject_compose_args --profile github-sync -fcompose.storage.silo.yml config
|
||||
reject_compose_args --profile github-sync --file=compose.storage.silo.yml config
|
||||
reject_compose_args --profile github-sync --file compose.storage.silo.yml config
|
||||
reject_compose_args --profile logs -f compose.storage.silo.yml config
|
||||
reject_compose_args --profile=logs -f compose.storage.silo.yml config
|
||||
reject_compose_args --profile github-sync logs -fcompose.storage.silo.yml
|
||||
reject_compose_args --profile github-sync logs --file=compose.storage.silo.yml
|
||||
reject_compose_args --profile github-sync logs --env-file "$work/env"
|
||||
reject_compose_args --profile github-sync logs --env-file="$work/env"
|
||||
reject_compose_args --profile github-sync logs --project-directory "$work"
|
||||
reject_compose_args --profile github-sync logs --project-directory="$work"
|
||||
if run --storage rustfs >/dev/null 2>&1; then fail "Accepted missing command"; fi
|
||||
if run --env-file "$work/missing" --storage rustfs -- config >/dev/null 2>&1; then fail "Accepted missing env"; fi
|
||||
echo "Storage Compose selection tests passed"
|
||||
39
scripts/tests/storage-diagnostics-test.sh
Executable file
39
scripts/tests/storage-diagnostics-test.sh
Executable file
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
mkdir -p "$work/repo/scripts" "$work/repo/server" "$work/bin"
|
||||
cp "$REPO_ROOT/scripts/storage-diagnostics.sh" "$work/repo/scripts/"
|
||||
cat >"$work/repo/server/mvnw" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$DIAGNOSTIC_COMMAND_LOG"
|
||||
echo 'sensitive-transport-detail-must-not-reach-console'
|
||||
exit "${DIAGNOSTIC_EXIT_CODE:-0}"
|
||||
STUB
|
||||
cat >"$work/bin/timeout" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$DIAGNOSTIC_COMMAND_LOG"
|
||||
[[ "${DIAGNOSTIC_TIMEOUT:-false}" != true ]] || exit 124
|
||||
shift 3
|
||||
exec "$@"
|
||||
STUB
|
||||
chmod +x "$work/repo/server/mvnw" "$work/bin/timeout"
|
||||
export PATH="$work/bin:$PATH" DIAGNOSTIC_COMMAND_LOG="$work/commands"
|
||||
export SKILLHUB_STORAGE_S3_ENDPOINT=https://storage.example.test SKILLHUB_STORAGE_S3_BUCKET=skillhub
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
script="$work/repo/scripts/storage-diagnostics.sh"
|
||||
bash "$script" >"$work/output" 2>&1
|
||||
grep -q 'Storage diagnostic passed' "$work/output" || fail 'Success not reported'
|
||||
if grep -q sensitive-transport "$work/output"; then fail 'Raw transport details leaked'; fi
|
||||
for log in "$work/repo/.dev/storage-diagnostics/"*.log; do
|
||||
[[ "$(stat -c %a "$log")" == 600 ]] || fail 'Raw diagnostic log must be private'
|
||||
done
|
||||
bash "$script" --full >"$work/output" 2>&1
|
||||
grep -q -- '-Dskillhub.storage.contract.full=true' "$work/commands" || fail 'Full contract suite not enabled'
|
||||
grep -q '420s' "$work/commands" || fail 'Full suite deadline missing'
|
||||
if DIAGNOSTIC_EXIT_CODE=7 bash "$script" >"$work/output" 2>&1; then fail 'Test failure reported as success'; else [[ $? == 7 ]] || fail 'Exit code not preserved'; fi
|
||||
if DIAGNOSTIC_TIMEOUT=true bash "$script" >"$work/output" 2>&1; then fail 'Timeout reported as success'; else [[ $? == 124 ]] || fail 'Timeout code not preserved'; fi
|
||||
if bash "$script" --unknown >"$work/output" 2>&1; then fail 'Unknown flag accepted'; fi
|
||||
if SKILLHUB_STORAGE_S3_ENDPOINT= bash "$script" >"$work/output" 2>&1; then fail 'Missing endpoint accepted'; fi
|
||||
echo 'Storage diagnostic timeout, privacy and exit-code tests passed'
|
||||
|
|
@ -363,4 +363,9 @@ while IFS= read -r line || [[ -n "$line" ]]; do
|
|||
done <"$REPO_ROOT/.env.release.draft" >"$draft_env"
|
||||
expect_fail "$draft_env" "POSTGRES_PASSWORD"
|
||||
|
||||
invalid_chunked_env="$tmp/invalid-chunked.env"
|
||||
write_env "$invalid_chunked_env" "release-download-secret-32-bytes-minimum"
|
||||
printf '%s\n' 'SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING=perhaps' >>"$invalid_chunked_env"
|
||||
expect_fail "$invalid_chunked_env" "SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING"
|
||||
|
||||
echo "validate-release-config-test passed"
|
||||
|
|
|
|||
|
|
@ -315,6 +315,7 @@ validate_boolean SESSION_COOKIE_SECURE
|
|||
validate_boolean BOOTSTRAP_ADMIN_ENABLED
|
||||
validate_boolean SKILLHUB_TRUST_FORWARDED_PROTO
|
||||
validate_boolean SKILLHUB_BUILTIN_SKILLS_ENABLED
|
||||
validate_boolean SKILLHUB_STORAGE_S3_DISABLE_CHUNKED_ENCODING
|
||||
validate_boolean SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE
|
||||
validate_boolean SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET
|
||||
validate_boolean SPRING_DATA_REDIS_SSL_ENABLED
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import java.net.URI;
|
|||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -162,10 +161,6 @@ public class S3StorageService implements ObjectStorageService {
|
|||
}
|
||||
}
|
||||
|
||||
private void putObjectInternal(String key, InputStream data, long size, String contentType) {
|
||||
putObjectInternal(key, RequestBody.fromInputStream(data, size), size, contentType);
|
||||
}
|
||||
|
||||
private void putObjectInternal(String key, RequestBody requestBody, long size, String contentType) {
|
||||
s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
|
|
@ -180,10 +175,19 @@ public class S3StorageService implements ObjectStorageService {
|
|||
private Path stagePutObjectBody(InputStream data) throws IOException {
|
||||
Path stagedBody = Files.createTempFile("skillhub-s3-upload-", ".tmp");
|
||||
try {
|
||||
Files.copy(data, stagedBody, StandardCopyOption.REPLACE_EXISTING);
|
||||
// Write into the existing private file; replacing it can widen its permissions.
|
||||
try (var output = Files.newOutputStream(stagedBody)) {
|
||||
data.transferTo(output);
|
||||
}
|
||||
return stagedBody;
|
||||
} catch (IOException e) {
|
||||
Files.deleteIfExists(stagedBody);
|
||||
} catch (IOException | RuntimeException | Error e) {
|
||||
try {
|
||||
Files.deleteIfExists(stagedBody);
|
||||
} catch (IOException | RuntimeException | Error cleanupFailure) {
|
||||
if (cleanupFailure != e) {
|
||||
e.addSuppressed(cleanupFailure);
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
|
@ -202,18 +206,24 @@ public class S3StorageService implements ObjectStorageService {
|
|||
@Override public void putObject(String key, InputStream data, long size, String contentType) {
|
||||
Path stagedBody = null;
|
||||
try {
|
||||
// Signing and retries can read a request body more than once. Callers can
|
||||
// supply non-resettable file streams, so keep a replayable, disk-backed body.
|
||||
stagedBody = stagePutObjectBody(data);
|
||||
if (Files.size(stagedBody) != size) {
|
||||
throw new IllegalArgumentException("Object size does not match the supplied stream");
|
||||
}
|
||||
RequestBody requestBody = RequestBody.fromFile(stagedBody);
|
||||
if (!properties.isAutoCreateBucket() || bucketPrepared) {
|
||||
putObjectInternal(key, data, size, contentType);
|
||||
putObjectInternal(key, requestBody, size, contentType);
|
||||
return;
|
||||
}
|
||||
|
||||
stagedBody = stagePutObjectBody(data);
|
||||
try {
|
||||
putObjectInternal(key, RequestBody.fromFile(stagedBody), size, contentType);
|
||||
putObjectInternal(key, requestBody, size, contentType);
|
||||
bucketPrepared = true;
|
||||
} catch (NoSuchBucketException e) {
|
||||
ensureBucketPrepared();
|
||||
putObjectInternal(key, RequestBody.fromFile(stagedBody), size, contentType);
|
||||
putObjectInternal(key, requestBody, size, contentType);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new StorageAccessException("putObject", key, e);
|
||||
|
|
@ -243,8 +253,23 @@ public class S3StorageService implements ObjectStorageService {
|
|||
@Override public void deleteObjects(List<String> keys) {
|
||||
if (keys.isEmpty()) return;
|
||||
try {
|
||||
List<ObjectIdentifier> ids = keys.stream().map(k -> ObjectIdentifier.builder().key(k).build()).toList();
|
||||
s3Client.deleteObjects(DeleteObjectsRequest.builder().bucket(properties.getBucket()).delete(Delete.builder().objects(ids).build()).build());
|
||||
for (int start = 0; start < keys.size(); start += 1000) {
|
||||
List<ObjectIdentifier> ids = keys.subList(start, Math.min(start + 1000, keys.size())).stream()
|
||||
.map(key -> ObjectIdentifier.builder().key(key).build()).toList();
|
||||
DeleteObjectsResponse response = s3Client.deleteObjects(DeleteObjectsRequest.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.delete(Delete.builder().objects(ids).build())
|
||||
.build());
|
||||
// S3 can return HTTP 200 with per-object failures. Earlier successful deletes
|
||||
// cannot be rolled back; report failure so callers can retry idempotently.
|
||||
if (!response.errors().isEmpty()) {
|
||||
String codes = response.errors().stream()
|
||||
.map(error -> error.code() == null ? "unknown" : error.code())
|
||||
.distinct().limit(5).collect(Collectors.joining(", "));
|
||||
throw new IllegalStateException("S3 batch deletion reported " + response.errors().size()
|
||||
+ " failures; codes=" + codes);
|
||||
}
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
throw new StorageAccessException("deleteObjects", String.join(",", keys), e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.UUID;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Timeout;
|
||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
||||
import org.springframework.boot.context.properties.bind.Binder;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/** Explicit opt-in: an enabled run with missing configuration fails rather than skips. */
|
||||
@EnabledIfSystemProperty(named = "skillhub.storage.contract", matches = "true")
|
||||
class S3StorageContractTest {
|
||||
@Test
|
||||
@Timeout(value = 120, unit = TimeUnit.SECONDS, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
|
||||
void configuredBackendMustSatisfySharedContract() throws Exception {
|
||||
var service = configuredService();
|
||||
try {
|
||||
service.init();
|
||||
new StorageContractProbe(service).verify();
|
||||
} finally {
|
||||
service.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfSystemProperty(named = "skillhub.storage.contract.full", matches = "true")
|
||||
@Timeout(value = 240, unit = TimeUnit.SECONDS, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
|
||||
void realBackendMustTraverseMultiplePagesAndDeleteMoreThanOneBatch() {
|
||||
var service = configuredService();
|
||||
var prefix = "skillhub-diagnostics/" + UUID.randomUUID() + "/pagination/";
|
||||
var keys = new ArrayList<String>();
|
||||
System.out.println("Pagination diagnostic object prefix: " + prefix);
|
||||
Throwable failure = null;
|
||||
try {
|
||||
service.init();
|
||||
for (int i = 0; i < 1001; i++) {
|
||||
String key = prefix + i;
|
||||
keys.add(key);
|
||||
service.putObject(key, new ByteArrayInputStream(new byte[]{42}), 1, "application/octet-stream");
|
||||
}
|
||||
assertThat(service.listPrefixUsage(prefix, 2000))
|
||||
.contains(new StoragePrefixUsage(1001, 1001, true));
|
||||
assertThat(service.listPrefixUsage(prefix, 1000))
|
||||
.contains(new StoragePrefixUsage(1000, 1000, false));
|
||||
// A real service may report transient per-object failures under a full batch.
|
||||
// Keep these visible, then exercise caller-side idempotent recovery with a bound.
|
||||
for (int attempt = 1; ; attempt++) {
|
||||
try {
|
||||
service.deleteObjects(keys);
|
||||
break;
|
||||
} catch (StorageAccessException exception) {
|
||||
if (attempt == 3) throw exception;
|
||||
Throwable cause = exception.getCause();
|
||||
String detail = cause instanceof IllegalStateException
|
||||
&& cause.getMessage() != null
|
||||
&& cause.getMessage().startsWith("S3 batch deletion reported ")
|
||||
? cause.getMessage() : cause == null ? "unknown" : cause.getClass().getSimpleName();
|
||||
System.out.println("Batch deletion reported failure; caller recovery attempt "
|
||||
+ attempt + "; " + detail);
|
||||
}
|
||||
}
|
||||
assertThat(service.listPrefixUsage(prefix, 2000))
|
||||
.contains(new StoragePrefixUsage(0, 0, true));
|
||||
keys.clear(); // Absence was verified; do not issue a redundant cleanup load burst.
|
||||
} catch (RuntimeException | Error exception) {
|
||||
failure = exception;
|
||||
throw exception;
|
||||
} finally {
|
||||
try {
|
||||
service.deleteObjects(keys);
|
||||
} catch (RuntimeException exception) {
|
||||
if (failure != null) failure.addSuppressed(exception);
|
||||
else throw exception;
|
||||
} finally {
|
||||
service.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private S3StorageService configuredService() {
|
||||
var properties = Binder.get(new StandardEnvironment())
|
||||
.bind("skillhub.storage.s3", S3StorageProperties.class).orElseThrow(
|
||||
() -> new IllegalStateException("S3 diagnostic configuration is missing"));
|
||||
if (properties.getEndpoint() == null || properties.getEndpoint().isBlank()
|
||||
|| properties.getBucket() == null || properties.getBucket().isBlank()) {
|
||||
throw new IllegalStateException("S3 endpoint and prepared bucket are required");
|
||||
}
|
||||
properties.setAutoCreateBucket(false);
|
||||
// Use the same per-call timeout defaults as the application, including batch operations.
|
||||
return new S3StorageService(properties);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class S3StorageProtocolTest {
|
||||
@Test
|
||||
void usageFollowsContinuationTokensAndSumsEveryPage() {
|
||||
var client = mock(S3Client.class);
|
||||
when(client.listObjectsV2(any(ListObjectsV2Request.class)))
|
||||
.thenReturn(ListObjectsV2Response.builder().isTruncated(true).nextContinuationToken("page-2")
|
||||
.contents(S3Object.builder().key("prefix/a").size(7L).build()).build())
|
||||
.thenReturn(ListObjectsV2Response.builder().isTruncated(false)
|
||||
.contents(S3Object.builder().key("prefix/b").size(11L).build()).build());
|
||||
var service = service(client);
|
||||
|
||||
assertThat(service.listPrefixUsage("prefix/", 10)).contains(new StoragePrefixUsage(18, 2, true));
|
||||
var requests = ArgumentCaptor.forClass(ListObjectsV2Request.class);
|
||||
verify(client, times(2)).listObjectsV2(requests.capture());
|
||||
assertThat(requests.getAllValues().get(0).continuationToken()).isNull();
|
||||
assertThat(requests.getAllValues().get(1).continuationToken()).isEqualTo("page-2");
|
||||
assertThat(requests.getAllValues().get(1).prefix()).isEqualTo("prefix/");
|
||||
}
|
||||
|
||||
@Test
|
||||
void truncatedResponseWithoutContinuationDoesNotClaimCompleteUsage() {
|
||||
var client = mock(S3Client.class);
|
||||
when(client.listObjectsV2(any(ListObjectsV2Request.class)))
|
||||
.thenReturn(ListObjectsV2Response.builder().isTruncated(true)
|
||||
.contents(S3Object.builder().size(7L).build()).build());
|
||||
|
||||
assertThat(service(client).listPrefixUsage("prefix/", 10)).contains(new StoragePrefixUsage(7, 1, false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void permissionFailureIsNotMisreportedAsMissingObject() {
|
||||
var client = mock(S3Client.class);
|
||||
when(client.headObject(any(HeadObjectRequest.class))).thenThrow(S3Exception.builder().statusCode(403).build());
|
||||
|
||||
assertThatThrownBy(() -> service(client).exists("private/key"))
|
||||
.isInstanceOf(StorageAccessException.class).extracting("operation").isEqualTo("exists");
|
||||
}
|
||||
|
||||
private S3StorageService service(S3Client client) {
|
||||
var service = new S3StorageService(new S3StorageProperties());
|
||||
ReflectionTestUtils.setField(service, "s3Client", client);
|
||||
return service;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +1,28 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.http.apache.ApacheHttpClient;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.CreateBucketResponse;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
|
||||
import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
|
||||
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
|
||||
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.S3Error;
|
||||
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
|
||||
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
|
||||
|
||||
|
|
@ -28,6 +33,7 @@ import java.nio.charset.StandardCharsets;
|
|||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
|
@ -304,6 +310,79 @@ class S3StorageServiceTest {
|
|||
assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void putObjectShouldReplayFileStreamsForSigningAndRetry(@org.junit.jupiter.api.io.TempDir java.nio.file.Path directory)
|
||||
throws Exception {
|
||||
S3Client client = mock(S3Client.class);
|
||||
byte[] content = new byte[256 * 1024 + 1];
|
||||
new java.util.Random(937).nextBytes(content);
|
||||
var path = java.nio.file.Files.write(directory.resolve("package.zip"), content);
|
||||
when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenAnswer(invocation -> {
|
||||
RequestBody body = invocation.getArgument(1);
|
||||
for (int read = 0; read < 2; read++) {
|
||||
try (InputStream input = body.contentStreamProvider().newStream()) {
|
||||
assertThat(input.readAllBytes()).isEqualTo(content);
|
||||
}
|
||||
}
|
||||
return PutObjectResponse.builder().build();
|
||||
});
|
||||
var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class));
|
||||
service.init();
|
||||
|
||||
try (InputStream input = java.nio.file.Files.newInputStream(path)) {
|
||||
assertThat(input.markSupported()).isFalse();
|
||||
service.putObject("packages/demo.zip", input, content.length, "application/zip");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void putObjectShouldRejectSizeMismatchBeforeCallingS3() {
|
||||
S3Client client = mock(S3Client.class);
|
||||
var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class));
|
||||
service.init();
|
||||
|
||||
assertThatThrownBy(() -> service.putObject("packages/demo.zip", new ByteArrayInputStream(new byte[]{42}),
|
||||
2, "application/zip"))
|
||||
.isInstanceOf(StorageAccessException.class).extracting("operation").isEqualTo("putObject");
|
||||
verifyNoInteractions(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteObjectsShouldRejectPartialFailureEvenWhenRequestSucceeded() {
|
||||
S3Client client = mock(S3Client.class);
|
||||
when(client.deleteObjects(any(DeleteObjectsRequest.class)))
|
||||
.thenReturn(DeleteObjectsResponse.builder()
|
||||
.errors(S3Error.builder()
|
||||
.key("private/denied.zip").code("AccessDenied").build())
|
||||
.build());
|
||||
var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class));
|
||||
service.init();
|
||||
|
||||
assertThatThrownBy(() -> service.deleteObjects(List.of("private/denied.zip")))
|
||||
.isInstanceOf(StorageAccessException.class)
|
||||
.extracting("operation").isEqualTo("deleteObjects");
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteObjectsShouldPartitionRequestsAtS3Limit() {
|
||||
S3Client client = mock(S3Client.class);
|
||||
when(client.deleteObjects(any(DeleteObjectsRequest.class)))
|
||||
.thenReturn(DeleteObjectsResponse.builder().build());
|
||||
var service = new TestableS3StorageService(properties(false), client, mock(S3Presigner.class));
|
||||
service.init();
|
||||
var keys = IntStream.range(0, 1001).mapToObj(i -> "packages/" + i).toList();
|
||||
|
||||
service.deleteObjects(keys);
|
||||
|
||||
var requests = ArgumentCaptor.forClass(
|
||||
DeleteObjectsRequest.class);
|
||||
verify(client, times(2)).deleteObjects(requests.capture());
|
||||
assertThat(requests.getAllValues().get(0).delete().objects()).hasSize(1000);
|
||||
assertThat(requests.getAllValues().get(1).delete().objects()).hasSize(1);
|
||||
assertThat(requests.getAllValues().stream().flatMap(r -> r.delete().objects().stream())
|
||||
.map(ObjectIdentifier::key).toList()).isEqualTo(keys);
|
||||
}
|
||||
|
||||
private S3StorageProperties properties(boolean autoCreateBucket) {
|
||||
S3StorageProperties properties = createProperties(true);
|
||||
properties.setBucket("skillhub");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.MockedStatic;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.http.apache.ApacheHttpClient;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.junit.jupiter.api.Assumptions.assumeTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.CALLS_REAL_METHODS;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class S3StorageStagingTest {
|
||||
@Test
|
||||
void stagedBodyRemainsOwnerOnlyDuringCopyAndUpload() throws Exception {
|
||||
Path probe = Files.createTempFile("skillhub-s3-permission-probe-", ".tmp");
|
||||
try {
|
||||
assumeTrue(Files.getFileStore(probe).supportsFileAttributeView("posix"),
|
||||
"Owner-only POSIX permissions require a POSIX file system");
|
||||
} finally {
|
||||
Files.deleteIfExists(probe);
|
||||
}
|
||||
var staged = new AtomicReference<Path>();
|
||||
S3Client client = mock(S3Client.class);
|
||||
S3StorageService service = service(client);
|
||||
try (var files = trackStagedBody(staged)) {
|
||||
when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenAnswer(invocation -> {
|
||||
assertOwnerOnly(staged.get());
|
||||
return PutObjectResponse.builder().build();
|
||||
});
|
||||
InputStream input = new InputStream() {
|
||||
private boolean consumed;
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
assertOwnerOnly(staged.get());
|
||||
if (consumed) return -1;
|
||||
consumed = true;
|
||||
return 42;
|
||||
}
|
||||
};
|
||||
service.putObject("private/package.zip", input, 1, "application/zip");
|
||||
assertThat(staged.get()).doesNotExist();
|
||||
} finally {
|
||||
if (staged.get() != null) Files.deleteIfExists(staged.get());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void stagingIOExceptionRemovesPartialFile() throws Exception {
|
||||
assertStagingFailureCleaned(new IOException("read failed"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stagingRuntimeExceptionRemovesPartialFile() throws Exception {
|
||||
assertStagingFailureCleaned(new IllegalStateException("read failed"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stagingErrorRemovesPartialFile() throws Exception {
|
||||
assertStagingFailureCleaned(new AssertionError("read failed"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stagingCleanupFailureDoesNotReplaceReadFailure() throws Exception {
|
||||
assertCleanupFailureSuppressed(new IOException("read failed"), new IOException("delete failed"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stagingCleanupFailureDoesNotReplaceRuntimeFailure() throws Exception {
|
||||
assertCleanupFailureSuppressed(new IllegalStateException("read failed"), new IOException("delete failed"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stagingRuntimeCleanupFailureDoesNotReplaceReadFailure() throws Exception {
|
||||
assertCleanupFailureSuppressed(new IOException("read failed"), new IllegalStateException("delete failed"));
|
||||
}
|
||||
|
||||
private void assertCleanupFailureSuppressed(Exception primary, Exception cleanup) throws Exception {
|
||||
var staged = new AtomicReference<Path>();
|
||||
S3Client client = mock(S3Client.class);
|
||||
S3StorageService service = service(client);
|
||||
try (var files = trackStagedBody(staged)) {
|
||||
InputStream input = new InputStream() {
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
files.when(() -> Files.deleteIfExists(staged.get())).thenThrow(cleanup);
|
||||
if (primary instanceof IOException io) throw io;
|
||||
throw (RuntimeException) primary;
|
||||
}
|
||||
};
|
||||
assertThatThrownBy(() -> service.putObject("private/package.zip", input, 1, "application/zip"))
|
||||
.isInstanceOf(StorageAccessException.class).hasCause(primary);
|
||||
assertThat(primary.getSuppressed()).containsExactly(cleanup);
|
||||
verifyNoInteractions(client);
|
||||
} finally {
|
||||
if (staged.get() != null) Files.deleteIfExists(staged.get());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void s3FailureRemovesStagedFile() throws Exception {
|
||||
var staged = new AtomicReference<Path>();
|
||||
S3Client client = mock(S3Client.class);
|
||||
S3StorageService service = service(client);
|
||||
RuntimeException failure = new IllegalStateException("S3 unavailable");
|
||||
when(client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenThrow(failure);
|
||||
try (var files = trackStagedBody(staged)) {
|
||||
assertThatThrownBy(() -> service.putObject("private/package.zip",
|
||||
new ByteArrayInputStream(new byte[]{42}), 1, "application/zip"))
|
||||
.isInstanceOf(StorageAccessException.class).hasCause(failure);
|
||||
assertThat(staged.get()).doesNotExist();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void sizeMismatchRemovesStagedFileBeforeS3Call() throws Exception {
|
||||
var staged = new AtomicReference<Path>();
|
||||
S3Client client = mock(S3Client.class);
|
||||
S3StorageService service = service(client);
|
||||
try (var files = trackStagedBody(staged)) {
|
||||
assertThatThrownBy(() -> service.putObject("private/package.zip",
|
||||
new ByteArrayInputStream(new byte[]{42}), 2, "application/zip"))
|
||||
.isInstanceOf(StorageAccessException.class).hasCauseInstanceOf(IllegalArgumentException.class);
|
||||
assertThat(staged.get()).doesNotExist();
|
||||
verifyNoInteractions(client);
|
||||
}
|
||||
}
|
||||
|
||||
private void assertStagingFailureCleaned(Throwable failure) throws Exception {
|
||||
var staged = new AtomicReference<Path>();
|
||||
S3Client client = mock(S3Client.class);
|
||||
S3StorageService service = service(client);
|
||||
InputStream input = new InputStream() {
|
||||
private boolean partialWritten;
|
||||
|
||||
@Override
|
||||
public int read(byte[] buffer, int offset, int length) throws IOException {
|
||||
if (!partialWritten) {
|
||||
partialWritten = true;
|
||||
buffer[offset] = 42;
|
||||
return 1;
|
||||
}
|
||||
if (failure instanceof IOException io) throw io;
|
||||
if (failure instanceof RuntimeException runtime) throw runtime;
|
||||
throw (Error) failure;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() {
|
||||
throw new AssertionError("bulk read expected");
|
||||
}
|
||||
};
|
||||
try (var files = trackStagedBody(staged)) {
|
||||
if (failure instanceof Error) {
|
||||
assertThatThrownBy(() -> service.putObject("private/package.zip", input, 2, "application/zip"))
|
||||
.isSameAs(failure);
|
||||
} else {
|
||||
assertThatThrownBy(() -> service.putObject("private/package.zip", input, 2, "application/zip"))
|
||||
.isInstanceOf(StorageAccessException.class).hasCause(failure);
|
||||
}
|
||||
assertThat(staged.get()).doesNotExist();
|
||||
verifyNoInteractions(client);
|
||||
} finally {
|
||||
if (staged.get() != null) Files.deleteIfExists(staged.get());
|
||||
}
|
||||
}
|
||||
|
||||
private MockedStatic<Files> trackStagedBody(AtomicReference<Path> staged) throws IOException {
|
||||
MockedStatic<Files> files = mockStatic(Files.class, CALLS_REAL_METHODS);
|
||||
files.when(() -> Files.createTempFile("skillhub-s3-upload-", ".tmp")).thenAnswer(invocation -> {
|
||||
Path path = (Path) invocation.callRealMethod();
|
||||
staged.set(path);
|
||||
return path;
|
||||
});
|
||||
return files;
|
||||
}
|
||||
|
||||
private void assertOwnerOnly(Path path) throws IOException {
|
||||
assertThat(Files.getPosixFilePermissions(path)).containsExactlyInAnyOrder(
|
||||
PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE);
|
||||
}
|
||||
|
||||
private S3StorageService service(S3Client client) {
|
||||
var properties = new S3StorageProperties();
|
||||
properties.setBucket("private-test-bucket");
|
||||
properties.setAutoCreateBucket(false);
|
||||
var presigner = mock(S3Presigner.class);
|
||||
var service = new S3StorageService(properties) {
|
||||
@Override
|
||||
protected S3Client buildS3Client(ApacheHttpClient.Builder builder) {
|
||||
return client;
|
||||
}
|
||||
|
||||
@Override
|
||||
S3Presigner buildPresigner() {
|
||||
return presigner;
|
||||
}
|
||||
};
|
||||
service.init();
|
||||
return service;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
/** Real object probe shared by every S3-compatible deployment. Never prints signed URLs. */
|
||||
final class StorageContractProbe {
|
||||
private final ObjectStorageService storage;
|
||||
private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();
|
||||
|
||||
StorageContractProbe(ObjectStorageService storage) {
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
void verify() throws Exception {
|
||||
String prefix = "skillhub-diagnostics/" + UUID.randomUUID() + "/";
|
||||
System.out.println("Diagnostic object prefix: " + prefix);
|
||||
List<String> keys = new ArrayList<>();
|
||||
Exception failure = null;
|
||||
try {
|
||||
byte[] content = new byte[256 * 1024 + 1];
|
||||
new Random(937).nextBytes(content);
|
||||
String key = prefix + "中文 空格+%?#/package.zip";
|
||||
keys.add(key);
|
||||
var sourceFile = java.nio.file.Files.createTempFile("skillhub-contract-", ".zip");
|
||||
try {
|
||||
java.nio.file.Files.write(sourceFile, content);
|
||||
try (var source = java.nio.file.Files.newInputStream(sourceFile)) {
|
||||
storage.putObject(key, source, content.length, "application/zip");
|
||||
}
|
||||
} finally {
|
||||
java.nio.file.Files.deleteIfExists(sourceFile);
|
||||
}
|
||||
require(storage.exists(key), "uploaded object missing");
|
||||
try (var input = storage.getObject(key)) {
|
||||
require(Arrays.equals(input.readAllBytes(), content), "object bytes differ");
|
||||
}
|
||||
ObjectMetadata metadata = storage.getMetadata(key);
|
||||
require(metadata.size() == content.length, "object size differs");
|
||||
require("application/zip".equals(metadata.contentType()), "content type differs");
|
||||
String signedGet = storage.generatePresignedUrl(key, Duration.ofMinutes(1), "中文 package.zip");
|
||||
var download = send(HttpRequest.newBuilder(URI.create(signedGet)).GET());
|
||||
require(download.statusCode() == 200 && Arrays.equals(download.body(), content), "signed GET failed");
|
||||
require(download.headers().firstValue("Content-Disposition").orElse("").contains("filename*="),
|
||||
"download filename missing");
|
||||
var tampered = send(HttpRequest.newBuilder(tamperSignature(signedGet)).GET());
|
||||
require(tampered.statusCode() == 403, "tampered signature was not rejected");
|
||||
var anonymous = send(HttpRequest.newBuilder(URI.create(signedGet.split("\\?")[0])).GET());
|
||||
require(anonymous.statusCode() == 403, "anonymous private object read was not rejected");
|
||||
String expiringGet = storage.generatePresignedUrl(key, Duration.ofSeconds(1), null);
|
||||
TimeUnit.SECONDS.sleep(3);
|
||||
require(send(HttpRequest.newBuilder(URI.create(expiringGet)).GET()).statusCode() == 403,
|
||||
"expired signature was not rejected");
|
||||
String putKey = prefix + "staging.zip";
|
||||
keys.add(putKey);
|
||||
PresignedUpload upload = storage.presignPut(putKey, Duration.ofMinutes(1)).orElseThrow();
|
||||
String signedHeaderNames = Arrays.stream(URI.create(upload.url()).getRawQuery().split("&"))
|
||||
.filter(field -> field.startsWith("X-Amz-SignedHeaders="))
|
||||
.map(field -> URLDecoder.decode(field.substring(field.indexOf('=') + 1),
|
||||
StandardCharsets.UTF_8))
|
||||
.findFirst().orElseThrow();
|
||||
System.out.println("PUT signature-bound header names: " + signedHeaderNames);
|
||||
var request = HttpRequest.newBuilder(URI.create(upload.url()));
|
||||
upload.headers().forEach(request::header);
|
||||
// SDK request headers (e.g. Content-Type) need not all be bound into SigV4.
|
||||
// Only require rejection for omitted headers listed in X-Amz-SignedHeaders.
|
||||
if (Arrays.stream(signedHeaderNames.split(";")).anyMatch(name -> !name.equalsIgnoreCase("host"))) {
|
||||
var missingHeaders = send(HttpRequest.newBuilder(URI.create(upload.url()))
|
||||
.PUT(HttpRequest.BodyPublishers.ofByteArray(content)));
|
||||
require(missingHeaders.statusCode() == 400 || missingHeaders.statusCode() == 403,
|
||||
"missing signature-bound PUT headers were not rejected");
|
||||
require(!storage.exists(putKey), "rejected PUT wrote an object");
|
||||
System.out.println("Missing-header PUT rejected with HTTP " + missingHeaders.statusCode());
|
||||
}
|
||||
var badPutRequest = HttpRequest.newBuilder(tamperSignature(upload.url()));
|
||||
upload.headers().forEach(badPutRequest::header);
|
||||
var badPut = send(badPutRequest.PUT(HttpRequest.BodyPublishers.ofByteArray(content)));
|
||||
require(badPut.statusCode() == 403, "tampered PUT signature was not rejected");
|
||||
require(!storage.exists(putKey), "tampered PUT wrote an object");
|
||||
var uploaded = send(request.PUT(HttpRequest.BodyPublishers.ofByteArray(content)));
|
||||
require(uploaded.statusCode() >= 200 && uploaded.statusCode() < 300, "signed PUT failed");
|
||||
try (var input = storage.getObject(putKey)) {
|
||||
require(Arrays.equals(input.readAllBytes(), content), "signed PUT bytes differ");
|
||||
}
|
||||
require("application/zip".equals(storage.getMetadata(putKey).contentType()),
|
||||
"signed PUT content type differs");
|
||||
var usage = storage.listPrefixUsage(prefix, 10).orElseThrow();
|
||||
require(usage.complete() && usage.objectCount() == 2 && usage.bytes() == content.length * 2L,
|
||||
"prefix usage differs");
|
||||
var limited = storage.listPrefixUsage(prefix, 1).orElseThrow();
|
||||
require(!limited.complete() && limited.objectCount() == 1, "truncated usage reported complete");
|
||||
storage.deleteObject(putKey);
|
||||
require(!storage.exists(putKey), "single delete failed");
|
||||
storage.deleteObjects(keys);
|
||||
require(!storage.exists(key), "batch delete failed");
|
||||
} catch (Exception exception) {
|
||||
failure = exception;
|
||||
throw exception;
|
||||
} finally {
|
||||
// Delete only keys created by this invocation, even when a write timed out.
|
||||
Exception cleanupFailure = null;
|
||||
for (String key : keys) {
|
||||
try {
|
||||
storage.deleteObject(key);
|
||||
} catch (Exception exception) {
|
||||
if (cleanupFailure == null) cleanupFailure = new IOException("Diagnostic cleanup failed for " + prefix);
|
||||
cleanupFailure.addSuppressed(exception);
|
||||
}
|
||||
}
|
||||
if (cleanupFailure != null) {
|
||||
if (failure != null) failure.addSuppressed(cleanupFailure);
|
||||
else throw cleanupFailure;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static URI tamperSignature(String signedUrl) {
|
||||
String marker = "X-Amz-Signature=";
|
||||
int start = signedUrl.indexOf(marker);
|
||||
if (start < 0) throw new IllegalArgumentException("Expected a SigV4 query signature");
|
||||
start += marker.length();
|
||||
// Change the digest while retaining valid signature syntax and length.
|
||||
char changed = signedUrl.charAt(start) == '0' ? '1' : '0';
|
||||
return URI.create(signedUrl.substring(0, start) + changed + signedUrl.substring(start + 1));
|
||||
}
|
||||
|
||||
private HttpResponse<byte[]> send(HttpRequest.Builder request) throws Exception {
|
||||
return http.send(request.timeout(Duration.ofSeconds(10)).build(), HttpResponse.BodyHandlers.ofByteArray());
|
||||
}
|
||||
|
||||
private void require(boolean condition, String message) {
|
||||
if (!condition) throw new IllegalStateException(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.io.InputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class StorageContractProbeTest {
|
||||
@Test
|
||||
void signatureTamperingChangesOnlyDigestContentNotItsFormat() {
|
||||
String signature = "a".repeat(64);
|
||||
String url = "https://storage.example.test/bucket/key?X-Amz-Signature=" + signature + "&X-Amz-Expires=60";
|
||||
String tampered = StorageContractProbe.tamperSignature(url).toString();
|
||||
|
||||
assertThat(tampered.length()).isEqualTo(url.length());
|
||||
assertThat(tampered).startsWith("https://storage.example.test/bucket/key?X-Amz-Signature=")
|
||||
.endsWith("&X-Amz-Expires=60");
|
||||
assertThat(IntStream.range(0, url.length())
|
||||
.filter(i -> url.charAt(i) != tampered.charAt(i)).count()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void writeFailureStillCleansOnlyTheAttemptedObject() {
|
||||
var storage = mock(ObjectStorageService.class);
|
||||
var writeFailure = new StorageAccessException("putObject", "object", new IllegalStateException());
|
||||
doThrow(writeFailure).when(storage).putObject(anyString(), any(InputStream.class), anyLong(), anyString());
|
||||
|
||||
assertThatThrownBy(() -> new StorageContractProbe(storage).verify()).isSameAs(writeFailure);
|
||||
|
||||
var key = ArgumentCaptor.forClass(String.class);
|
||||
verify(storage).deleteObject(key.capture());
|
||||
assertThat(key.getValue()).startsWith("skillhub-diagnostics/").endsWith("package.zip");
|
||||
}
|
||||
|
||||
@Test
|
||||
void cleanupFailureIsNotLostWhenPrimaryOperationAlsoFails() {
|
||||
var storage = mock(ObjectStorageService.class);
|
||||
var writeFailure = new IllegalStateException("write failed");
|
||||
doThrow(writeFailure).when(storage).putObject(anyString(), any(InputStream.class), anyLong(), anyString());
|
||||
doThrow(new IllegalStateException("delete failed")).when(storage).deleteObject(anyString());
|
||||
|
||||
assertThatThrownBy(() -> new StorageContractProbe(storage).verify()).isSameAs(writeFailure);
|
||||
assertThat(writeFailure.getSuppressed()).hasSize(1);
|
||||
assertThat(writeFailure.getSuppressed()[0].getMessage()).startsWith("Diagnostic cleanup failed for skillhub-diagnostics/");
|
||||
}
|
||||
|
||||
@Test
|
||||
void probeDetectsCorruptedObjectBytes() {
|
||||
var storage = mock(ObjectStorageService.class);
|
||||
when(storage.exists(anyString())).thenReturn(true);
|
||||
when(storage.getObject(anyString())).thenReturn(new ByteArrayInputStream(new byte[]{0}));
|
||||
|
||||
assertThatThrownBy(() -> new StorageContractProbe(storage).verify())
|
||||
.isInstanceOf(IllegalStateException.class).hasMessage("object bytes differ");
|
||||
verify(storage).deleteObject(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void probeDetectsFalseWriteSuccess() {
|
||||
var storage = mock(ObjectStorageService.class);
|
||||
when(storage.exists(anyString())).thenReturn(false);
|
||||
|
||||
assertThatThrownBy(() -> new StorageContractProbe(storage).verify())
|
||||
.isInstanceOf(IllegalStateException.class).hasMessage("uploaded object missing");
|
||||
verify(storage).deleteObject(anyString());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue