mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-15 09:11:18 +00:00
fix(ci): mirror runnable OCI images without unsupported referrers (#1012)
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
XiaoSeS
GitHub
parent
b85629239c
commit
e7e38d7304
4 changed files with 120 additions and 11 deletions
1
.github/workflows/pr-scripts.yml
vendored
1
.github/workflows/pr-scripts.yml
vendored
|
|
@ -28,6 +28,7 @@ on:
|
|||
- '.github/workflows/pr-helm-chart.yml'
|
||||
- '.github/workflows/pr-tests.yml'
|
||||
- '.github/workflows/publish-chart.yml'
|
||||
- '.github/workflows/publish-images.yml'
|
||||
- '.github/workflows/security.yml'
|
||||
- '.github/workflows/pr-scripts.yml'
|
||||
- '.github/workflows/migration-preflight.yml'
|
||||
|
|
|
|||
11
.github/workflows/publish-images.yml
vendored
11
.github/workflows/publish-images.yml
vendored
|
|
@ -150,9 +150,16 @@ jobs:
|
|||
echo "enabled=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
- name: Install pinned registry copy tool
|
||||
if: ${{ steps.mirror.outputs.enabled == 'true' }}
|
||||
uses: docker/setup-buildx-action@v3
|
||||
run: |
|
||||
# Buildx automatically copies OCI referrers rejected by some mirrors.
|
||||
curl -fsSL --retry 3 \
|
||||
https://github.com/regclient/regclient/releases/download/v0.11.7/regctl-linux-amd64 \
|
||||
-o "$RUNNER_TEMP/regctl"
|
||||
echo "3737fbbdd791e04e7a831640b681f3397a376498e80e2ad9b1b2d27296f239bc $RUNNER_TEMP/regctl" | sha256sum -c -
|
||||
chmod +x "$RUNNER_TEMP/regctl"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Log in to mirror registry
|
||||
if: ${{ steps.mirror.outputs.enabled == 'true' }}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
# The publishing workflow installs the checksum-pinned regctl used below.
|
||||
# Python 3 is used only to select descriptors from the public source metadata.
|
||||
IMAGES_FILE="${1:-deploy/runtime-mirror-images.txt}"
|
||||
|
||||
: "${MIRROR_REGISTRY:?MIRROR_REGISTRY is required}"
|
||||
|
|
@ -19,11 +21,61 @@ while read -r source target; do
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if docker buildx imagetools inspect "${target_prefix}/${target}" >/dev/null 2>&1; then
|
||||
if regctl manifest head "${target_prefix}/${target}" >/dev/null 2>&1; then
|
||||
echo "Skipping existing image ${target_prefix}/${target}"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Mirroring ${source} -> ${target_prefix}/${target}"
|
||||
docker buildx imagetools create --tag "${target_prefix}/${target}" "${source}"
|
||||
manifest="$(regctl manifest get "${source}" --format raw-body)"
|
||||
# Copy immutable runnable children, not the index's attestations: some mirror
|
||||
# registries reject their application/vnd.oci.empty.v1+json config media type.
|
||||
# Keep every image platform (including variants), rather than hardcoding amd64.
|
||||
sources="$(printf '%s' "$manifest" | python3 -c '
|
||||
import json, sys
|
||||
source = sys.argv[1]
|
||||
index = json.load(sys.stdin)
|
||||
media_type = index.get("mediaType")
|
||||
print(media_type)
|
||||
if "manifests" not in index:
|
||||
if index.get("artifactType") or media_type not in (
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
):
|
||||
sys.exit("Source is not a runnable image manifest")
|
||||
print(source)
|
||||
else:
|
||||
if media_type not in ("application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json"):
|
||||
sys.exit("Source has an unsupported image index media type")
|
||||
refs = []
|
||||
for item in index["manifests"]:
|
||||
if (item.get("annotations", {}).get("vnd.docker.reference.type") == "attestation-manifest"
|
||||
or item.get("artifactType")):
|
||||
continue
|
||||
platform = item.get("platform", {})
|
||||
if any(platform.get(key) in (None, "", "unknown") for key in ("os", "architecture")):
|
||||
sys.exit("Source contains an unclassified descriptor; refusing to drop a platform")
|
||||
digest = item.get("digest")
|
||||
if not isinstance(digest, str) or ":" not in digest or any(c.isspace() for c in digest):
|
||||
sys.exit("Source contains an invalid image digest")
|
||||
refs.append(source.split("@", 1)[0] + "@" + digest)
|
||||
if not refs:
|
||||
sys.exit("Source contains no runnable image manifests")
|
||||
print("\n".join(refs))
|
||||
' "$source")"
|
||||
mapfile -t source_refs <<<"$sources"
|
||||
source_media_type="${source_refs[0]}"
|
||||
source_refs=("${source_refs[@]:1}")
|
||||
# Buildx also discovers and copies referrers even with explicit child sources.
|
||||
# regctl can copy the original image blobs without adding those artifacts.
|
||||
if [[ ${#source_refs[@]} -eq 1 && "${source_refs[0]}" == "$source" ]]; then
|
||||
regctl image copy --referrers=false "$source" "${target_prefix}/${target}"
|
||||
else
|
||||
mirror_args=(index create "${target_prefix}/${target}" --media-type "$source_media_type" --referrers=false)
|
||||
for source_ref in "${source_refs[@]}"; do
|
||||
mirror_args+=(--ref "$source_ref")
|
||||
done
|
||||
regctl "${mirror_args[@]}"
|
||||
fi
|
||||
done < "${IMAGES_FILE}"
|
||||
|
|
|
|||
|
|
@ -4,23 +4,72 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
mkdir "$work/bin"
|
||||
cat >"$work/bin/docker" <<'STUB'
|
||||
cat >"$work/bin/regctl" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$MIRROR_COMMAND_LOG"
|
||||
case "$*" in
|
||||
'buildx imagetools inspect '*) [[ "${TARGET_EXISTS:-false}" == true ]] ;;
|
||||
'buildx imagetools create '*) exit 0 ;;
|
||||
'manifest get '*)
|
||||
[[ "${INSPECT_EXIT:-0}" == 0 ]] || exit "$INSPECT_EXIT"
|
||||
case "${SOURCE_KIND:-index}" in
|
||||
index)
|
||||
printf '%s\n' '{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[
|
||||
{"digest":"sha256:aaa","platform":{"os":"linux","architecture":"amd64"}},
|
||||
{"digest":"sha256:bbb","platform":{"os":"linux","architecture":"arm64","variant":"v8"}},
|
||||
{"digest":"sha256:ccc","platform":{"os":"linux","architecture":"riscv64"}},
|
||||
{"digest":"sha256:attestation","platform":{"os":"unknown","architecture":"unknown"},"annotations":{"vnd.docker.reference.type":"attestation-manifest"}},
|
||||
{"digest":"sha256:artifact","artifactType":"application/vnd.example.signature"}]}' ;;
|
||||
single) printf '%s\n' '{"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{}}' ;;
|
||||
empty) printf '%s\n' '{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}' ;;
|
||||
unknown) printf '%s\n' '{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"sha256:unknown","platform":{"os":"unknown","architecture":"unknown"}}]}' ;;
|
||||
malformed) printf '%s\n' 'not-json' ;;
|
||||
esac ;;
|
||||
'manifest head '*) [[ "${TARGET_EXISTS:-false}" == true ]] ;;
|
||||
'index create '*|'image copy '*)
|
||||
if [[ "$*" == *'@sha256:abc'* && "${SOURCE_KIND:-index}" == index ]]; then
|
||||
echo 'denied: unknown manifest class for application/vnd.oci.empty.v1+json' >&2
|
||||
exit 42
|
||||
fi
|
||||
exit "${CREATE_EXIT:-0}" ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
STUB
|
||||
chmod +x "$work/bin/docker"
|
||||
chmod +x "$work/bin/regctl"
|
||||
export PATH="$work/bin:$PATH" MIRROR_COMMAND_LOG="$work/commands"
|
||||
export MIRROR_REGISTRY=registry.example.test MIRROR_NAMESPACE=skillhub
|
||||
printf '%s\n' 'docker.io/rustfs/rustfs@sha256:abc rustfs:immutable-test' >"$work/images"
|
||||
TARGET_EXISTS=true bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output"
|
||||
if grep -q 'imagetools create' "$work/commands"; then echo 'FAIL: existing target was refreshed' >&2; exit 1; fi
|
||||
if grep -Eq '^index create|^image copy' "$work/commands"; then echo 'FAIL: existing target was refreshed' >&2; exit 1; fi
|
||||
TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output"
|
||||
grep -q 'imagetools create --tag registry.example.test/skillhub/rustfs:immutable-test docker.io/rustfs/rustfs@sha256:abc' "$work/commands"
|
||||
grep -Fxq 'index create registry.example.test/skillhub/rustfs:immutable-test --media-type application/vnd.oci.image.index.v1+json --referrers=false --ref docker.io/rustfs/rustfs@sha256:aaa --ref docker.io/rustfs/rustfs@sha256:bbb --ref docker.io/rustfs/rustfs@sha256:ccc' "$work/commands"
|
||||
if grep '^index create' "$work/commands" | grep -Eq 'attestation|artifact'; then
|
||||
echo 'FAIL: artifact descriptors were copied' >&2; exit 1
|
||||
fi
|
||||
SOURCE_KIND=single TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output"
|
||||
grep -Fxq 'image copy --referrers=false docker.io/rustfs/rustfs@sha256:abc registry.example.test/skillhub/rustfs:immutable-test' "$work/commands"
|
||||
for kind in empty unknown malformed; do
|
||||
: >"$work/commands"
|
||||
if SOURCE_KIND="$kind" TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output" 2>&1; then
|
||||
echo "FAIL: invalid source $kind was accepted" >&2; exit 1
|
||||
fi
|
||||
if grep -Eq '^index create|^image copy' "$work/commands"; then
|
||||
echo "FAIL: invalid source $kind reached the registry write" >&2; exit 1
|
||||
fi
|
||||
done
|
||||
for stage in inspect create; do
|
||||
expected=23
|
||||
if [[ "$stage" == inspect ]]; then
|
||||
if INSPECT_EXIT="$expected" TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output" 2>&1; then
|
||||
echo 'FAIL: inspection failure was swallowed' >&2; exit 1
|
||||
else [[ $? == "$expected" ]]; fi
|
||||
else
|
||||
if CREATE_EXIT="$expected" TARGET_EXISTS=false bash "$REPO_ROOT/scripts/mirror-runtime-images.sh" "$work/images" >"$work/output" 2>&1; then
|
||||
echo 'FAIL: registry copy failure was swallowed' >&2; exit 1
|
||||
else [[ $? == "$expected" ]]; fi
|
||||
fi
|
||||
done
|
||||
grep -q '^docker.io/pgsty/silo:RELEASE.2026-09-16T00-00-00Z minio:latest$' "$REPO_ROOT/deploy/runtime-mirror-images.txt"
|
||||
grep -Fxq 'docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z@sha256:2636fc4155d9a7e41ff58cdb65ddd13ba9304e67184e13c25bc5cb34016269d7 silo:20261018-2636fc4155d9' "$REPO_ROOT/deploy/runtime-mirror-images.txt"
|
||||
echo 'Storage mirror skip and legacy alias tests passed'
|
||||
grep -Fq 'https://github.com/regclient/regclient/releases/download/v0.11.7/regctl-linux-amd64' "$REPO_ROOT/.github/workflows/publish-images.yml"
|
||||
grep -Fq '3737fbbdd791e04e7a831640b681f3397a376498e80e2ad9b1b2d27296f239bc' "$REPO_ROOT/.github/workflows/publish-images.yml"
|
||||
grep -Fq "'.github/workflows/publish-images.yml'" "$REPO_ROOT/.github/workflows/pr-scripts.yml"
|
||||
echo 'Storage mirror platform filtering, skip, failure propagation and legacy alias tests passed'
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue