diff --git a/.github/scripts/docker_pull_with_retries.sh b/.github/scripts/docker_pull_with_retries.sh new file mode 100644 index 00000000000..cf1cfd823f2 --- /dev/null +++ b/.github/scripts/docker_pull_with_retries.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 || -z "${1:-}" || "${1:-}" == -* ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +readonly image="$1" +log="$(mktemp)" +readonly log +trap 'rm -f "$log"' EXIT + +for attempt in 1 2 3; do + echo "docker pull attempt ${attempt}/3 for ${image}" + if timeout --kill-after=5s 45s docker pull "$image" > "$log" 2>&1; then + cat "$log" + exit 0 + else + status=$? + cat "$log" >&2 + fi + + if [[ "$status" -ne 124 ]] && ! grep -Eiq \ + 'HTTP status: (429|5[0-9]{2})|toomanyrequests|i/o timeout|TLS handshake timeout|connection reset by peer' "$log"; then + exit "$status" + fi + if [[ "$attempt" -eq 3 ]]; then + echo "docker pull failed after ${attempt} attempts" >&2 + exit "$status" + fi + echo "transient image pull failure, retrying in $((attempt * 2))s" >&2 + sleep "$((attempt * 2))" +done diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 0a490bb8f45..eded59f3fbd 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -472,8 +472,8 @@ jobs: python-version: "3.12" workers: 4 reruns: 0 - timeout-minutes: 20 - job-timeout-minutes: 60 + timeout-minutes: 23 + job-timeout-minutes: 63 dist: loadscope - shard: enterprise-repositories-secrets @@ -1048,6 +1048,7 @@ jobs: timeout-minutes: ${{ matrix.timeout-minutes }} env: TEST_PATH: ${{ matrix.test-path }} + UNIT_SHARD: ${{ matrix.shard }} MAX_FAILURES: "10" WORKERS: ${{ matrix.workers }} RERUNS: ${{ matrix.reruns }} @@ -1055,6 +1056,10 @@ jobs: DIST: ${{ matrix.dist }} COVERAGE_CORE: sysmon run: | + if [ "$UNIT_SHARD" = "rust-bridge-harness" ]; then + python -m unittest discover -s tests/code_coverage_tests -p test_docker_pull_with_retries.py -v + bash .github/scripts/docker_pull_with_retries.sh alpine:3.20 + fi echo "has-coverage=false" >> "$GITHUB_OUTPUT" selection="${TEST_PATH}" if [ -z "${selection// /}" ]; then diff --git a/tests/code_coverage_tests/test_docker_pull_with_retries.py b/tests/code_coverage_tests/test_docker_pull_with_retries.py new file mode 100644 index 00000000000..01831052b36 --- /dev/null +++ b/tests/code_coverage_tests/test_docker_pull_with_retries.py @@ -0,0 +1,111 @@ +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Final + +ROOT: Final = Path(__file__).resolve().parents[2] +SCRIPT: Final = ROOT / ".github/scripts/docker_pull_with_retries.sh" +DOCKER: Final = r'''#!/usr/bin/env bash +set -eu +count=$(cat "$STATE/count" 2>/dev/null || printf 0) +count=$((count + 1)) +printf '%s' "$count" > "$STATE/count" +printf '%s\n' "$*" >> "$STATE/calls" +sed -n "${count}p" "$STATE/messages" >&2 +status=$(sed -n "${count}p" "$STATE/statuses") +exit "${status:-99}" +''' +SLEEP: Final = '#!/usr/bin/env bash\nprintf "%s\\n" "$1" >> "$STATE/sleeps"\n' + + +class DockerPullRetryTests(unittest.TestCase): + def run_pull( + self, statuses: tuple[int, ...], messages: tuple[str, ...], args: tuple[str, ...] = ("alpine:3.20",) + ) -> tuple[subprocess.CompletedProcess[str], tuple[str, ...], tuple[str, ...]]: + with tempfile.TemporaryDirectory() as folder: + state: Final = Path(folder) + (state / "statuses").write_text("\n".join(map(str, statuses))) + (state / "messages").write_text("\n".join(messages)) + for name, source in (("docker", DOCKER), ("sleep", SLEEP)): + (state / name).write_text(source) + (state / name).chmod(0o755) + result: Final = subprocess.run( + ("bash", str(SCRIPT), *args), + env={**os.environ, "STATE": folder, "PATH": folder + os.pathsep + os.environ["PATH"]}, + capture_output=True, + text=True, + timeout=10, + ) + calls: Final = tuple((state / "calls").read_text().splitlines()) if (state / "calls").exists() else () + sleeps: Final = tuple((state / "sleeps").read_text().splitlines()) if (state / "sleeps").exists() else () + return result, calls, sleeps + + def test_success_runs_once(self) -> None: + result, calls, sleeps = self.run_pull((0,), ("downloaded",)) + self.assertEqual(result.returncode, 0) + self.assertEqual(calls, ("pull alpine:3.20",)) + self.assertEqual(sleeps, ()) + + def test_transient_failures_recover_without_rerunning_tests(self) -> None: + for message in ( + "docker: received unexpected HTTP status: 502 Bad Gateway", + "HTTP status: 503 Service Unavailable", + "HTTP status: 429 Too Many Requests", + "toomanyrequests: rate limit exceeded", + "net/http: TLS handshake timeout", + "read: connection reset by peer", + "dial tcp: i/o timeout", + ): + with self.subTest(message=message): + result, calls, sleeps = self.run_pull((1, 0), (message, "downloaded")) + self.assertEqual(result.returncode, 0) + self.assertEqual(calls, ("pull alpine:3.20",) * 2) + self.assertEqual(sleeps, ("2",)) + self.assertIn(message, result.stderr) + + def test_timeout_status_is_retried(self) -> None: + result, calls, sleeps = self.run_pull((124, 0), ("", "downloaded")) + self.assertEqual(result.returncode, 0) + self.assertEqual(len(calls), 2) + self.assertEqual(sleeps, ("2",)) + + def test_exhausted_transient_failure_stays_failed(self) -> None: + result, calls, sleeps = self.run_pull((1, 1, 1, 0), ("HTTP status: 502 Bad Gateway",) * 4) + self.assertEqual(result.returncode, 1) + self.assertEqual(len(calls), 3) + self.assertEqual(sleeps, ("2", "4")) + + def test_exhausted_timeout_preserves_exit_status(self) -> None: + result, calls, sleeps = self.run_pull((124, 124, 124), ("", "", "")) + self.assertEqual(result.returncode, 124) + self.assertEqual(len(calls), 3) + self.assertEqual(sleeps, ("2", "4")) + + def test_permanent_failures_do_not_retry(self) -> None: + for status, message in ( + (1, "unauthorized: incorrect credentials"), + (1, "manifest unknown"), + (1, "x509: certificate signed by unknown authority"), + (125, "invalid reference format"), + (127, "docker: command not found"), + (137, "Killed"), + ): + with self.subTest(message=message): + result, calls, sleeps = self.run_pull((status, 0), (message, "downloaded")) + self.assertEqual(result.returncode, status) + self.assertEqual(len(calls), 1) + self.assertEqual(sleeps, ()) + + def test_invalid_arguments_do_not_run_docker(self) -> None: + for args in ((), ("",), ("--help",), ("one", "two")): + with self.subTest(args=args): + result, calls, sleeps = self.run_pull((0,), ("",), args) + self.assertEqual(result.returncode, 2) + self.assertEqual(calls, ()) + self.assertEqual(sleeps, ()) + + +if __name__ == "__main__": + unittest.main()