diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 681a1b40..884b72e9 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -233,6 +233,12 @@ skillhub-storage-s3-secret-key: your-secret-key | postgres-data-0 | 10Gi | PostgreSQL 数据(with-infra only) | | redis-data-0 | 5Gi | Redis 数据(with-infra only) | +#### PostgreSQL 数据目录兼容性 + +`with-infra` 会在启动时检查 PostgreSQL PVC 根目录:如果已存在 `PG_VERSION`,继续使用根目录中的旧集群;否则在 `pgdata/` 子目录初始化新集群,避免新 ext4 卷中的 `lost+found` 阻止 `initdb`。升级现有部署不需要移动数据库文件。 + +回滚到不包含该检测逻辑的旧清单时,如果集群位于 `pgdata/`,必须保留当前启动命令,或显式设置 `PGDATA=/var/lib/postgresql/data/pgdata`。不要把正在运行的数据库目录手动移动到 PVC 根目录。 + ## 镜像说明 | 组件 | 镜像 | diff --git a/deploy/k8s/overlays/with-infra/postgres-statefulset.yaml b/deploy/k8s/overlays/with-infra/postgres-statefulset.yaml index 26f4dc0c..0ee5b8d6 100644 --- a/deploy/k8s/overlays/with-infra/postgres-statefulset.yaml +++ b/deploy/k8s/overlays/with-infra/postgres-statefulset.yaml @@ -18,6 +18,19 @@ spec: containers: - name: postgres image: postgres:16-alpine + # Legacy installations store PostgreSQL directly at the PVC root. + # Fresh ext4 volumes may contain lost+found, so initialize new clusters + # in a child directory without hiding an existing root-level cluster. + command: + - sh + - -ec + - | + if [ -f /var/lib/postgresql/data/PG_VERSION ]; then + export PGDATA=/var/lib/postgresql/data + else + export PGDATA=/var/lib/postgresql/data/pgdata + fi + exec docker-entrypoint.sh postgres ports: - containerPort: 5432 name: postgres @@ -37,7 +50,6 @@ spec: volumeMounts: - name: postgres-data mountPath: /var/lib/postgresql/data - subPath: pgdata readinessProbe: exec: command: diff --git a/docs/skillhub/en/guide/kubernetes.md b/docs/skillhub/en/guide/kubernetes.md index 73eb2cee..5955db4b 100644 --- a/docs/skillhub/en/guide/kubernetes.md +++ b/docs/skillhub/en/guide/kubernetes.md @@ -212,6 +212,12 @@ skillhub-storage-s3-secret-key: your-secret-key value: us-east-1 ``` +### PostgreSQL data-directory compatibility + +The `with-infra` overlay checks the PostgreSQL PVC root at startup. If `PG_VERSION` already exists there, the legacy root-level cluster remains in use. Otherwise, a new cluster is initialized under `pgdata/`, avoiding the `lost+found` directory that can block `initdb` on a fresh ext4 volume. Existing installations do not need to move their database files during upgrade. + +When rolling back to an older manifest without this detection, keep the current startup command or set `PGDATA=/var/lib/postgresql/data/pgdata` explicitly if the cluster lives under `pgdata/`. Do not move an active database directory back to the PVC root by hand. + ### Image Reference | Component | Image | diff --git a/docs/skillhub/guide/kubernetes.md b/docs/skillhub/guide/kubernetes.md index de8f505e..91c974d7 100644 --- a/docs/skillhub/guide/kubernetes.md +++ b/docs/skillhub/guide/kubernetes.md @@ -212,6 +212,12 @@ skillhub-storage-s3-secret-key: your-secret-key value: cn-shanghai ``` +### PostgreSQL 数据目录兼容性 + +`with-infra` 会在启动时检查 PostgreSQL PVC 根目录:如果已存在 `PG_VERSION`,继续使用根目录中的旧集群;否则在 `pgdata/` 子目录初始化新集群,避免新 ext4 卷中的 `lost+found` 阻止 `initdb`。升级现有部署不需要移动数据库文件。 + +回滚到不包含该检测逻辑的旧清单时,如果集群位于 `pgdata/`,必须保留当前启动命令,或显式设置 `PGDATA=/var/lib/postgresql/data/pgdata`。不要把正在运行的数据库目录手动移动到 PVC 根目录。 + ### 镜像说明 | 组件 | 镜像 | diff --git a/scripts/tests/k8s-postgres-storage-test.sh b/scripts/tests/k8s-postgres-storage-test.sh new file mode 100755 index 00000000..f22242bb --- /dev/null +++ b/scripts/tests/k8s-postgres-storage-test.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MANIFEST="$REPO_ROOT/deploy/k8s/overlays/with-infra/postgres-statefulset.yaml" +POSTGRES_IMAGE="${POSTGRES_IMAGE:-postgres:16-alpine}" +RUN_ID="skillhub-k8s-pgdata-$$" +FRESH_VOLUME="${RUN_ID}-fresh" +LEGACY_VOLUME="${RUN_ID}-legacy" +CONTAINERS=() +VOLUMES=("$FRESH_VOLUME" "$LEGACY_VOLUME") + +cleanup() { + for container in "${CONTAINERS[@]}"; do + docker stop "$container" >/dev/null 2>&1 || true + done + for volume in "${VOLUMES[@]}"; do + docker volume rm "$volume" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +for command in docker sed; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "required command not found: $command" >&2 + exit 1 + fi +done + +POSTGRES_START_COMMAND="$( + sed -n '/^ - |$/,/^ ports:$/p' "$MANIFEST" \ + | sed '1d;$d;s/^ //' +)" +if [[ -z "$POSTGRES_START_COMMAND" ]]; then + echo "could not extract the PostgreSQL startup command from $MANIFEST" >&2 + exit 1 +fi + +wait_for_postgres() { + local container="$1" + for _ in $(seq 1 30); do + if docker exec "$container" pg_isready -U skillhub -d skillhub >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + docker logs "$container" >&2 + return 1 +} + +start_postgres() { + local container="$1" + local volume="$2" + local mode="${3:-default}" + CONTAINERS+=("$container") + if [[ "$mode" == "patched" ]]; then + docker run --detach --rm \ + --name "$container" \ + --env POSTGRES_DB=skillhub \ + --env POSTGRES_USER=skillhub \ + --env POSTGRES_PASSWORD=storage-test \ + --volume "$volume:/var/lib/postgresql/data" \ + --entrypoint sh \ + "$POSTGRES_IMAGE" \ + -ec "$POSTGRES_START_COMMAND" >/dev/null + else + docker run --detach --rm \ + --name "$container" \ + --env POSTGRES_DB=skillhub \ + --env POSTGRES_USER=skillhub \ + --env POSTGRES_PASSWORD=storage-test \ + --volume "$volume:/var/lib/postgresql/data" \ + "$POSTGRES_IMAGE" >/dev/null + fi + wait_for_postgres "$container" +} + +stop_postgres() { + docker stop "$1" >/dev/null +} + +assert_marker() { + local container="$1" + local expected="$2" + local actual + actual="$(docker exec "$container" psql -U skillhub -d skillhub -Atqc \ + 'SELECT value FROM storage_upgrade_verification WHERE id = 1')" + [[ "$actual" == "$expected" ]] +} + +docker volume create "$FRESH_VOLUME" >/dev/null +docker run --rm \ + --volume "$FRESH_VOLUME:/data" \ + --entrypoint sh \ + "$POSTGRES_IMAGE" \ + -ec 'mkdir -p /data/lost+found' + +start_postgres "${RUN_ID}-fresh-1" "$FRESH_VOLUME" patched +docker exec "${RUN_ID}-fresh-1" test -f /var/lib/postgresql/data/pgdata/PG_VERSION +docker exec "${RUN_ID}-fresh-1" psql -U skillhub -d skillhub -v ON_ERROR_STOP=1 -qc \ + "CREATE TABLE storage_upgrade_verification (id integer PRIMARY KEY, value text NOT NULL); \ + INSERT INTO storage_upgrade_verification VALUES (1, 'fresh-volume');" +stop_postgres "${RUN_ID}-fresh-1" + +start_postgres "${RUN_ID}-fresh-2" "$FRESH_VOLUME" patched +assert_marker "${RUN_ID}-fresh-2" "fresh-volume" +stop_postgres "${RUN_ID}-fresh-2" +echo "PASS: fresh volume with lost+found initializes in pgdata and survives restart" + +docker volume create "$LEGACY_VOLUME" >/dev/null +start_postgres "${RUN_ID}-legacy-1" "$LEGACY_VOLUME" +docker exec "${RUN_ID}-legacy-1" test -f /var/lib/postgresql/data/PG_VERSION +docker exec "${RUN_ID}-legacy-1" psql -U skillhub -d skillhub -v ON_ERROR_STOP=1 -qc \ + "CREATE TABLE storage_upgrade_verification (id integer PRIMARY KEY, value text NOT NULL); \ + INSERT INTO storage_upgrade_verification VALUES (1, 'legacy-root');" +stop_postgres "${RUN_ID}-legacy-1" + +start_postgres "${RUN_ID}-legacy-2" "$LEGACY_VOLUME" patched +assert_marker "${RUN_ID}-legacy-2" "legacy-root" +stop_postgres "${RUN_ID}-legacy-2" +echo "PASS: legacy root-level cluster remains visible after the manifest upgrade"