Merge 38554417ee into 3f4dc93b40

.github/scripts/docker_pull_with_retries.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$#" -ne 1 || -z "${1:-}" || "${1:-}" == -* ]]; then
+  echo "usage: $0 <image>" >&2
+  exit 2
+fi
+
+readonly image="$1"
+log="$(mktemp)"
+readonly log
+trap 'rm -f "$log"' EXIT
+
+for attempt in 1 2 3; do
+  echo "docker pull attempt ${attempt}/3 for ${image}"
+  if timeout --kill-after=5s 45s docker pull "$image" > "$log" 2>&1; then
+    cat "$log"
+    exit 0
+  else
+    status=$?
+    cat "$log" >&2
+  fi
+
+  if [[ "$status" -ne 124 ]] && ! grep -Eiq \
+    'HTTP status: (429|5[0-9]{2})|toomanyrequests|i/o timeout|TLS handshake timeout|connection reset by peer' "$log"; then
+    exit "$status"
+  fi
+  if [[ "$attempt" -eq 3 ]]; then
+    echo "docker pull failed after ${attempt} attempts" >&2
+    exit "$status"
+  fi
+  echo "transient image pull failure, retrying in $((attempt * 2))s" >&2
+  sleep "$((attempt * 2))"
+done

.github/workflows/test-unit.yml

@@ -477,8 +477,8 @@ jobs:
             python-version: "3.12"
             workers: 4
             reruns: 0
-            timeout-minutes: 20
-            job-timeout-minutes: 60
+            timeout-minutes: 23
+            job-timeout-minutes: 63
             dist: loadscope
 
           - shard: enterprise-repositories-secrets
@@ -1063,6 +1063,7 @@ jobs:
         timeout-minutes: ${{ matrix.timeout-minutes }}
         env:
           TEST_PATH: ${{ matrix.test-path }}
+          UNIT_SHARD: ${{ matrix.shard }}
           MAX_FAILURES: "10"
           WORKERS: ${{ matrix.workers }}
           RERUNS: ${{ matrix.reruns }}
@@ -1070,6 +1071,10 @@ jobs:
           DIST: ${{ matrix.dist }}
           COVERAGE_CORE: sysmon
         run: |
+          if [ "$UNIT_SHARD" = "rust-bridge-harness" ]; then
+            python -m unittest discover -s tests/code_coverage_tests -p test_docker_pull_with_retries.py -v
+            bash .github/scripts/docker_pull_with_retries.sh alpine:3.20
+          fi
           echo "has-coverage=false" >> "$GITHUB_OUTPUT"
           selection="${TEST_PATH}"
           if [ -z "${selection// /}" ]; then

tests/code_coverage_tests/test_docker_pull_with_retries.py

@@ -0,0 +1,111 @@
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from typing import Final
+
+ROOT: Final = Path(__file__).resolve().parents[2]
+SCRIPT: Final = ROOT / ".github/scripts/docker_pull_with_retries.sh"
+DOCKER: Final = r'''#!/usr/bin/env bash
+set -eu
+count=$(cat "$STATE/count" 2>/dev/null || printf 0)
+count=$((count + 1))
+printf '%s' "$count" > "$STATE/count"
+printf '%s\n' "$*" >> "$STATE/calls"
+sed -n "${count}p" "$STATE/messages" >&2
+status=$(sed -n "${count}p" "$STATE/statuses")
+exit "${status:-99}"
+'''
+SLEEP: Final = '#!/usr/bin/env bash\nprintf "%s\\n" "$1" >> "$STATE/sleeps"\n'
+
+
+class DockerPullRetryTests(unittest.TestCase):
+    def run_pull(
+        self, statuses: tuple[int, ...], messages: tuple[str, ...], args: tuple[str, ...] = ("alpine:3.20",)
+    ) -> tuple[subprocess.CompletedProcess[str], tuple[str, ...], tuple[str, ...]]:
+        with tempfile.TemporaryDirectory() as folder:
+            state: Final = Path(folder)
+            (state / "statuses").write_text("\n".join(map(str, statuses)))
+            (state / "messages").write_text("\n".join(messages))
+            for name, source in (("docker", DOCKER), ("sleep", SLEEP)):
+                (state / name).write_text(source)
+                (state / name).chmod(0o755)
+            result: Final = subprocess.run(
+                ("bash", str(SCRIPT), *args),
+                env={**os.environ, "STATE": folder, "PATH": folder + os.pathsep + os.environ["PATH"]},
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            calls: Final = tuple((state / "calls").read_text().splitlines()) if (state / "calls").exists() else ()
+            sleeps: Final = tuple((state / "sleeps").read_text().splitlines()) if (state / "sleeps").exists() else ()
+            return result, calls, sleeps
+
+    def test_success_runs_once(self) -> None:
+        result, calls, sleeps = self.run_pull((0,), ("downloaded",))
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(calls, ("pull alpine:3.20",))
+        self.assertEqual(sleeps, ())
+
+    def test_transient_failures_recover_without_rerunning_tests(self) -> None:
+        for message in (
+            "docker: received unexpected HTTP status: 502 Bad Gateway",
+            "HTTP status: 503 Service Unavailable",
+            "HTTP status: 429 Too Many Requests",
+            "toomanyrequests: rate limit exceeded",
+            "net/http: TLS handshake timeout",
+            "read: connection reset by peer",
+            "dial tcp: i/o timeout",
+        ):
+            with self.subTest(message=message):
+                result, calls, sleeps = self.run_pull((1, 0), (message, "downloaded"))
+                self.assertEqual(result.returncode, 0)
+                self.assertEqual(calls, ("pull alpine:3.20",) * 2)
+                self.assertEqual(sleeps, ("2",))
+                self.assertIn(message, result.stderr)
+
+    def test_timeout_status_is_retried(self) -> None:
+        result, calls, sleeps = self.run_pull((124, 0), ("", "downloaded"))
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(len(calls), 2)
+        self.assertEqual(sleeps, ("2",))
+
+    def test_exhausted_transient_failure_stays_failed(self) -> None:
+        result, calls, sleeps = self.run_pull((1, 1, 1, 0), ("HTTP status: 502 Bad Gateway",) * 4)
+        self.assertEqual(result.returncode, 1)
+        self.assertEqual(len(calls), 3)
+        self.assertEqual(sleeps, ("2", "4"))
+
+    def test_exhausted_timeout_preserves_exit_status(self) -> None:
+        result, calls, sleeps = self.run_pull((124, 124, 124), ("", "", ""))
+        self.assertEqual(result.returncode, 124)
+        self.assertEqual(len(calls), 3)
+        self.assertEqual(sleeps, ("2", "4"))
+
+    def test_permanent_failures_do_not_retry(self) -> None:
+        for status, message in (
+            (1, "unauthorized: incorrect credentials"),
+            (1, "manifest unknown"),
+            (1, "x509: certificate signed by unknown authority"),
+            (125, "invalid reference format"),
+            (127, "docker: command not found"),
+            (137, "Killed"),
+        ):
+            with self.subTest(message=message):
+                result, calls, sleeps = self.run_pull((status, 0), (message, "downloaded"))
+                self.assertEqual(result.returncode, status)
+                self.assertEqual(len(calls), 1)
+                self.assertEqual(sleeps, ())
+
+    def test_invalid_arguments_do_not_run_docker(self) -> None:
+        for args in ((), ("",), ("--help",), ("one", "two")):
+            with self.subTest(args=args):
+                result, calls, sleeps = self.run_pull((0,), ("",), args)
+                self.assertEqual(result.returncode, 2)
+                self.assertEqual(calls, ())
+                self.assertEqual(sleeps, ())
+
+
+if __name__ == "__main__":
+    unittest.main()