diff --git a/.github/workflows/pr-helm-chart.yml b/.github/workflows/pr-helm-chart.yml index 3e87c9bf..0f26fb97 100644 --- a/.github/workflows/pr-helm-chart.yml +++ b/.github/workflows/pr-helm-chart.yml @@ -176,15 +176,23 @@ jobs: fi - name: Schema validation (kubeconform) - uses: docker://ghcr.io/yannh/kubeconform:latest + uses: docker://ghcr.io/yannh/kubeconform@sha256:faffaf43f95aa6425306e1ab8d6fcad72acb9049158f38e574c085ea1ec0f64e # v0.8.0 with: entrypoint: '/kubeconform' args: "-strict -summary -output text -schema-location default -schema-location https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json charts/skillhub/rendered.yaml" install-upgrade: - name: Install and Upgrade Smoke + name: Install and Upgrade Smoke (${{ matrix.scenario }}) if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + scenario: + - default + - sentinel + - s3 + - ingress-tls steps: - name: Check out repository @@ -204,4 +212,6 @@ jobs: wait: 120s - name: Run install and upgrade smoke + env: + HELM_SMOKE_SCENARIO: ${{ matrix.scenario }} run: bash charts/skillhub/tests/install-upgrade-smoke.sh diff --git a/charts/skillhub/README.md b/charts/skillhub/README.md index f4063034..29c41818 100644 --- a/charts/skillhub/README.md +++ b/charts/skillhub/README.md @@ -39,6 +39,7 @@ redis: ``` ```bash +helm dependency build ./charts/skillhub kubectl create namespace skillhub helm -n skillhub upgrade -i skillhub ./charts/skillhub \ @@ -57,12 +58,16 @@ helm -n skillhub upgrade -i skillhub ./charts/skillhub \ 合并或发布前,可在一个空的测试 Kubernetes 集群中运行可重复的安装/升级 smoke: ```bash -bash charts/skillhub/tests/install-upgrade-smoke.sh +for scenario in default sentinel s3 ingress-tls; do + HELM_SMOKE_SCENARIO="$scenario" \ + bash charts/skillhub/tests/install-upgrade-smoke.sh +done ``` 脚本验证 `install -> Ready -> HTTP health -> upgrade -> Ready`,并确认 Secret 数据、 -PVC UID 与绑定 PV 在升级前后保持不变。默认清理自己创建的 namespace;设置 -`KEEP_HELM_SMOKE=true` 可保留现场用于排查。 +PVC UID 与绑定 PV 在升级前后保持不变。四个场景分别覆盖默认依赖、Redis +Sentinel、实际 MinIO S3 连接,以及由 Kubernetes API 接受的 TLS Ingress 路由。 +默认清理自己创建的 namespace;设置 `KEEP_HELM_SMOKE=true` 可保留现场用于排查。 ### 高可用模式 @@ -144,10 +149,11 @@ redis: 可预测密码,而是在任何随机密码缺失时终止渲染并指出具体配置项。敏感值应放在 受保护的 values、External Secrets、Sealed Secrets 或密钥注入插件中。 -内置 PostgreSQL、Redis、Sentinel 及 metrics exporter 镜像默认使用不可变 digest, -避免 Bitnami 子 Chart 的 `latest` 默认值造成不可复现的安装和回滚。覆盖私有镜像 -仓库或 tag 时,必须同时把对应的 `image.digest` 设为空,或改成私有仓库中该镜像 -的真实 digest;digest 非空时会优先于 tag。 +内置 PostgreSQL、Redis、Sentinel 及 metrics exporter 镜像默认使用不可变的 +多架构 manifest digest,避免 Bitnami 子 Chart 的 `latest` 默认值造成不可复现的 +安装和回滚,同时保留 amd64/arm64 支持。覆盖私有镜像仓库或 tag 时,必须同时把 +对应的 `image.digest` 设为空,或改成私有仓库中该镜像的真实 digest;digest +非空时会优先于 tag。 ## 配置参考 diff --git a/charts/skillhub/tests/configuration-contracts.sh b/charts/skillhub/tests/configuration-contracts.sh index 61d75756..17b77853 100755 --- a/charts/skillhub/tests/configuration-contracts.sh +++ b/charts/skillhub/tests/configuration-contracts.sh @@ -27,10 +27,10 @@ render verify "$CHART_DIR" >"$TMP_DIR/default.yaml" grep -Fq 'name: POSTGRESQL_MAX_CONNECTIONS' "$TMP_DIR/default.yaml" grep -Fq 'value: "verify-postgresql"' "$TMP_DIR/default.yaml" grep -Fq 'value: "verify-redis-master"' "$TMP_DIR/default.yaml" -grep -Fq 'bitnami/postgresql@sha256:52e4eae10959a7ef4810f2d0c9ce656e9337090fd2427d8d15d2e1335ba4f8fb' "$TMP_DIR/default.yaml" -grep -Fq 'bitnami/postgres-exporter@sha256:fbd5fbc3f350c793efef777b0710f17019e8dd7812fc7c7396b13aa5130f3bd9' "$TMP_DIR/default.yaml" -grep -Fq 'bitnami/redis@sha256:5ff436b8a0dbe3808f03c7b682865cd30ddb1f08c1a8d17970ebf475f5689722' "$TMP_DIR/default.yaml" -grep -Fq 'bitnami/redis-exporter@sha256:0468a9a14ebd96953f967f9f3e8673c3eb1ad1dd54d379fef15c3f3ee714c99e' "$TMP_DIR/default.yaml" +grep -Fq 'bitnami/postgresql@sha256:db2312d9b243afa8c3b3f5496e478d17d0dff9791d06f3b93b9567abd86ae92f' "$TMP_DIR/default.yaml" +grep -Fq 'bitnami/postgres-exporter@sha256:53ab72a1b940d7637e91619f1000da9ebef14bc7dad74321a78731d65c79f55b' "$TMP_DIR/default.yaml" +grep -Fq 'bitnami/redis@sha256:08863c2c3f4e051fb6139b38fa223e9c13be5033326a59bead182860d899bf98' "$TMP_DIR/default.yaml" +grep -Fq 'bitnami/redis-exporter@sha256:fb1dae6add1e1104989d086d9407f7d65f58968550aa5fddea20637a758c0773' "$TMP_DIR/default.yaml" if grep -Eq 'image:.*:latest([@"[:space:]]|$)' "$TMP_DIR/default.yaml"; then fail "default workloads must not use mutable latest image tags" fi @@ -114,7 +114,7 @@ grep -A1 -F 'name: SKILLHUB_REDIS_SENTINEL_CHECK_SENTINELS_LIST' "$TMP_DIR/senti render sentinel-full "$CHART_DIR" \ --set redis.architecture=replication \ --set redis.sentinel.enabled=true >"$TMP_DIR/sentinel-full.yaml" -grep -Fq 'bitnami/redis-sentinel@sha256:667bfef3e22ce9910bdcf304b1b7c659cfc15390a2d473187eb3caac2537d0e6' "$TMP_DIR/sentinel-full.yaml" +grep -Fq 'bitnami/redis-sentinel@sha256:ae75dd69c192a632bdeb21baa6721080be5b12347e52add922036398b47631da' "$TMP_DIR/sentinel-full.yaml" if grep -Eq 'image:.*:latest([@"[:space:]]|$)' "$TMP_DIR/sentinel-full.yaml"; then fail "Sentinel workloads must not use mutable latest image tags" fi diff --git a/charts/skillhub/tests/install-upgrade-smoke.sh b/charts/skillhub/tests/install-upgrade-smoke.sh index c7d3145e..fd461d56 100755 --- a/charts/skillhub/tests/install-upgrade-smoke.sh +++ b/charts/skillhub/tests/install-upgrade-smoke.sh @@ -3,13 +3,15 @@ set -euo pipefail CHART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEST_VALUES="$CHART_DIR/tests/test-values.yaml" -NAMESPACE=${HELM_SMOKE_NAMESPACE:-skillhub-helm-smoke} +SCENARIO=${HELM_SMOKE_SCENARIO:-default} +NAMESPACE=${HELM_SMOKE_NAMESPACE:-skillhub-helm-smoke-$SCENARIO} RELEASE=${HELM_SMOKE_RELEASE:-skillhub-smoke} TIMEOUT=${HELM_SMOKE_TIMEOUT:-15m} KEEP_ENVIRONMENT=${KEEP_HELM_SMOKE:-false} TMP_DIR=$(mktemp -d) PORT_FORWARD_PID="" OWNS_NAMESPACE=false +HELM_SCENARIO_ARGS=() fail() { echo "ERROR: $*" >&2 @@ -23,7 +25,6 @@ done if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then fail "namespace $NAMESPACE already exists; choose an unused HELM_SMOKE_NAMESPACE" fi -OWNS_NAMESPACE=true stop_port_forward() { if [[ -n "$PORT_FORWARD_PID" ]]; then @@ -55,12 +56,153 @@ cleanup() { } trap cleanup EXIT +setup_scenario() { + case "$SCENARIO" in + default) + ;; + sentinel) + HELM_SCENARIO_ARGS+=( + --set redis.architecture=replication + --set redis.sentinel.enabled=true + ) + ;; + s3) + kubectl apply --namespace "$NAMESPACE" -f - <<'YAML' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + args: + - server + - /data + env: + - name: MINIO_ROOT_USER + value: smoke-access-key + - name: MINIO_ROOT_PASSWORD + value: smoke-secret-key + ports: + - name: api + containerPort: 9000 + readinessProbe: + httpGet: + path: /minio/health/ready + port: api + periodSeconds: 2 +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api +YAML + kubectl rollout status deployment/minio \ + --namespace "$NAMESPACE" \ + --timeout=5m + HELM_SCENARIO_ARGS+=( + --set s3.enabled=true + --set-string s3.endpoint=http://minio:9000 + --set-string s3.accessKey=smoke-access-key + --set-string s3.secretKey=smoke-secret-key + --set s3.autoCreateBucket=true + ) + ;; + ingress-tls) + command -v openssl >/dev/null 2>&1 || fail "openssl is required for ingress-tls" + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$TMP_DIR/tls.key" \ + -out "$TMP_DIR/tls.crt" \ + -days 1 \ + -subj /CN=skillhub-smoke.local \ + -addext subjectAltName=DNS:skillhub-smoke.local >/dev/null 2>&1 + kubectl create secret tls skillhub-smoke-tls \ + --namespace "$NAMESPACE" \ + --cert "$TMP_DIR/tls.crt" \ + --key "$TMP_DIR/tls.key" + HELM_SCENARIO_ARGS+=( + --set ingress.enabled=true + --set-json 'ingress.hosts=[{"host":"skillhub-smoke.local","paths":[{"path":"/","pathType":"Prefix"}]}]' + --set-json 'ingress.tls=[{"hosts":["skillhub-smoke.local"],"secretName":"skillhub-smoke-tls"}]' + ) + ;; + *) + fail "unknown HELM_SMOKE_SCENARIO: $SCENARIO" + ;; + esac +} + +assert_scenario_contract() { + case "$SCENARIO" in + default) + ;; + sentinel) + kubectl get deployment "$RELEASE-server" --namespace "$NAMESPACE" -o json \ + | jq -e ' + [.spec.template.spec.containers[] + | select(.name == "server") + | .env[] + | select(.name == "SKILLHUB_REDIS_SENTINEL_CHECK_SENTINELS_LIST") + | .value] == ["false"] + ' >/dev/null \ + || fail "Sentinel scenario did not apply the Kubernetes-only address-check override" + ;; + s3) + local storage_provider + storage_provider=$(kubectl get configmap "$RELEASE-config" \ + --namespace "$NAMESPACE" -o json | jq -r '.data["skillhub-storage-provider"]') + [[ "$storage_provider" == "s3" ]] || fail "S3 scenario did not configure S3 storage" + ;; + ingress-tls) + kubectl get ingress "$RELEASE" --namespace "$NAMESPACE" -o json \ + | jq -e --arg server "$RELEASE-server" ' + .spec.tls[0].secretName == "skillhub-smoke-tls" + and ( + [.spec.rules[].http.paths[] + | select( + .path == "/api" + or .path == "/oauth2" + or .path == "/login/oauth2" + or .path == "/.well-known" + ) + | .backend.service.name] + | length == 4 and all(. == $server) + ) + ' >/dev/null \ + || fail "TLS Ingress does not route every reserved path directly to the server" + local cookie_secure + cookie_secure=$(kubectl get configmap "$RELEASE-config" \ + --namespace "$NAMESPACE" -o json | jq -r '.data["session-cookie-secure"]') + [[ "$cookie_secure" == "true" ]] || fail "TLS Ingress did not enable secure session cookies" + ;; + esac +} + probe_service() { local service=$1 local service_port=$2 local local_port=$3 local path=$4 + local expected_status=${5:-200} local log_file="$TMP_DIR/${service}.port-forward.log" + local status stop_port_forward kubectl port-forward \ @@ -70,7 +212,9 @@ probe_service() { PORT_FORWARD_PID=$! for _ in $(seq 1 60); do - if curl --fail --silent --show-error "http://127.0.0.1:$local_port$path" >/dev/null; then + status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + "http://127.0.0.1:$local_port$path" 2>/dev/null || true) + if [[ "$status" == "$expected_status" ]]; then stop_port_forward return 0 fi @@ -81,7 +225,7 @@ probe_service() { done cat "$log_file" >&2 - fail "$service$path did not become healthy" + fail "$service$path did not return HTTP $expected_status" } snapshot_secrets() { @@ -113,7 +257,7 @@ assert_ready_and_healthy() { probe_service "$RELEASE-server" 8080 18081 /actuator/health probe_service "$RELEASE-web" 80 18080 /nginx-health - probe_service "$RELEASE-web" 80 18080 /api/v1/namespaces + probe_service "$RELEASE-web" 80 18080 /api/v1/auth/me 401 probe_service "$RELEASE-scanner" 8000 18082 /health local restarts @@ -123,17 +267,21 @@ assert_ready_and_healthy() { } helm dependency build "$CHART_DIR" +kubectl create namespace "$NAMESPACE" +OWNS_NAMESPACE=true +setup_scenario helm install "$RELEASE" "$CHART_DIR" \ --namespace "$NAMESPACE" \ - --create-namespace \ --values "$TEST_VALUES" \ --set-string fullnameOverride="$RELEASE" \ --set-string publicBaseUrl=http://skillhub-smoke.local \ + "${HELM_SCENARIO_ARGS[@]}" \ --wait \ --timeout "$TIMEOUT" assert_ready_and_healthy +assert_scenario_contract snapshot_secrets "$TMP_DIR/secrets-before" snapshot_pvcs "$TMP_DIR/pvcs-before" revision_before=$(helm history "$RELEASE" --namespace "$NAMESPACE" -o json \ @@ -148,6 +296,7 @@ helm upgrade "$RELEASE" "$CHART_DIR" \ --timeout "$TIMEOUT" assert_ready_and_healthy +assert_scenario_contract snapshot_secrets "$TMP_DIR/secrets-after" snapshot_pvcs "$TMP_DIR/pvcs-after" revision_after=$(helm history "$RELEASE" --namespace "$NAMESPACE" -o json \ @@ -166,4 +315,4 @@ public_base_url=$(kubectl get configmap "$RELEASE-config" \ [[ "$public_base_url" == "https://skillhub-smoke.local" ]] \ || fail "publicBaseUrl was not applied by the upgrade" -echo "Helm install/upgrade smoke passed" +echo "Helm install/upgrade smoke passed for scenario: $SCENARIO" diff --git a/charts/skillhub/values.yaml b/charts/skillhub/values.yaml index c3e70769..e89a7d4c 100644 --- a/charts/skillhub/values.yaml +++ b/charts/skillhub/values.yaml @@ -108,7 +108,7 @@ postgresql: # Bitnami's current chart defaults to a mutable latest tag. Pin the image # digest so installs and rollbacks remain reproducible. image: - digest: "sha256:52e4eae10959a7ef4810f2d0c9ce656e9337090fd2427d8d15d2e1335ba4f8fb" + digest: "sha256:db2312d9b243afa8c3b3f5496e478d17d0dff9791d06f3b93b9567abd86ae92f" auth: postgresPassword: "" @@ -171,7 +171,7 @@ postgresql: metrics: enabled: true image: - digest: "sha256:fbd5fbc3f350c793efef777b0710f17019e8dd7812fc7c7396b13aa5130f3bd9" + digest: "sha256:53ab72a1b940d7637e91619f1000da9ebef14bc7dad74321a78731d65c79f55b" serviceMonitor: enabled: false @@ -193,7 +193,7 @@ redis: # Keep the bundled Redis runtime immutable for repeatable upgrades. image: - digest: "sha256:5ff436b8a0dbe3808f03c7b682865cd30ddb1f08c1a8d17970ebf475f5689722" + digest: "sha256:08863c2c3f4e051fb6139b38fa223e9c13be5033326a59bead182860d899bf98" auth: enabled: true @@ -240,7 +240,7 @@ redis: enabled: false masterSet: mymaster image: - digest: "sha256:667bfef3e22ce9910bdcf304b1b7c659cfc15390a2d473187eb3caac2537d0e6" + digest: "sha256:ae75dd69c192a632bdeb21baa6721080be5b12347e52add922036398b47631da" service: ports: sentinel: 26379 @@ -250,7 +250,7 @@ redis: metrics: enabled: true image: - digest: "sha256:0468a9a14ebd96953f967f9f3e8673c3eb1ad1dd54d379fef15c3f3ee714c99e" + digest: "sha256:fb1dae6add1e1104989d086d9407f7d65f58968550aa5fddea20637a758c0773" serviceMonitor: enabled: false