Restart NVD downloads when resumable responses are invalid

scripts/sync_cve_metadata.py

@@ -881,10 +881,14 @@ def load_feed(name: str, *, cache_dir: Path | None = None) -> list[dict[str, Any
                 if getattr(response, "status", 200) == 206:
                     span = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", response_headers.get("Content-Range", ""))
                     if not span or not data or not etag or response_etag != etag:
-                        raise RuntimeError(f"{name}: invalid resumed NVD response")
+                        data.clear()
+                        etag = None
+                        raise http.client.HTTPException(f"{name}: invalid resumed NVD response")
                     start, end, total = map(int, span.groups())
                     if start != len(data) or end != total - 1:
-                        raise RuntimeError(f"{name}: mismatched NVD byte range")
+                        data.clear()
+                        etag = None
+                        raise http.client.HTTPException(f"{name}: mismatched NVD byte range")
                 else:
                     data.clear()
                     etag = response_etag if response_etag and not response_etag.startswith("W/") else None
@@ -901,12 +905,17 @@ def load_feed(name: str, *, cache_dir: Path | None = None) -> list[dict[str, Any
                 raise
             break
         except urllib.error.HTTPError as problem:
-            if problem.code < 500 or attempt == 2:
+            if attempt == 2:
+                raise
+            if problem.code == 416 and request.has_header("Range"):
+                data.clear()
+                etag = None
+            elif problem.code < 500:
                 raise
         except (OSError, EOFError, http.client.HTTPException, json.JSONDecodeError, zlib.error):
             if attempt == 2:
                 raise
-        print(f"{name}: incomplete NVD download; retrying ({attempt + 2}/3, {len(data)} bytes retained)", flush=True)
+        print(f"{name}: incomplete NVD download; retrying ({attempt + 2}/3, {len(data) if etag else 0} bytes retained)", flush=True)
         time.sleep(2 ** attempt)
     if payload.get("format") != "NVD_CVE" or payload.get("version") != "2.0":
         raise RuntimeError(f"{name}: unexpected NVD feed format")

tests/test_sync_cve_metadata.py

@@ -97,18 +97,52 @@ class FeedDownloadTests(unittest.TestCase):
                 self.assertEqual(metadata.load_feed("modified", cache_dir=cache), rows)
             self.assertEqual((cache / "nvd-modified.json.gz").read_bytes(), newer)
 
-    def test_mismatched_resumed_response_is_rejected(self):
+    def test_mismatched_resumed_response_restarts_without_using_its_bytes(self):
         offset = len(self.body) - 8
         for span, etag in [(f"bytes 0-{len(self.body)-1}/{len(self.body)}", '"v1"'),
                            (f"bytes {offset}-{len(self.body)-1}/{len(self.body)}", '"v2"')]:
             responses = [
                 self.response(self.body[:offset], {"Content-Length": str(len(self.body)), "ETag": '"v1"'}),
                 self.response(self.body[offset:], {"Content-Range": span, "ETag": etag}, 206),
+                self.response(self.body, {}),
             ]
-            with patch.object(metadata.urllib.request, "urlopen", side_effect=responses), \
+            with patch.object(metadata.urllib.request, "urlopen", side_effect=responses) as fetch, \
                     patch.object(metadata.time, "sleep"), redirect_stdout(io.StringIO()):
-                with self.assertRaises(RuntimeError):
-                    metadata.load_feed("modified")
+                self.assertEqual(metadata.load_feed("modified"), self.feed)
+            self.assertIsNone(fetch.call_args_list[2].args[0].get_header("Range"))
+
+    def test_range_not_satisfiable_restarts_but_other_client_errors_do_not(self):
+        for code in (416, 401, 403, 404):
+            responses = [
+                self.response(self.body[:-8], {"Content-Length": str(len(self.body)), "ETag": '"v1"'}),
+                metadata.urllib.error.HTTPError(metadata.feed_url("modified"), code, "rejected", {}, None),
+                self.response(self.body, {}),
+            ]
+            with patch.object(metadata.urllib.request, "urlopen", side_effect=responses) as fetch, \
+                    patch.object(metadata.time, "sleep"), redirect_stdout(io.StringIO()):
+                if code == 416:
+                    self.assertEqual(metadata.load_feed("modified"), self.feed)
+                    self.assertIsNone(fetch.call_args_list[2].args[0].get_header("Range"))
+                else:
+                    with self.assertRaises(metadata.urllib.error.HTTPError):
+                        metadata.load_feed("modified")
+                    self.assertEqual(fetch.call_count, 2)
+
+    def test_corrupt_resumed_gzip_restarts_before_caching(self):
+        offset = len(self.body) - 8
+        trailer = bytes([self.body[offset] ^ 1]) + self.body[offset + 1:]
+        responses = [
+            self.response(self.body[:offset], {"Content-Length": str(len(self.body)), "ETag": '"v1"'}),
+            self.response(trailer, {"Content-Range": f"bytes {offset}-{len(self.body)-1}/{len(self.body)}", "ETag": '"v1"'}, 206),
+            self.response(self.body, {}),
+        ]
+        with tempfile.TemporaryDirectory() as directory:
+            cache = Path(directory)
+            with patch.object(metadata.urllib.request, "urlopen", side_effect=responses) as fetch, \
+                    patch.object(metadata.time, "sleep"), redirect_stdout(io.StringIO()):
+                self.assertEqual(metadata.load_feed("modified", cache_dir=cache), self.feed)
+            self.assertEqual((cache / "nvd-modified.json.gz").read_bytes(), self.body)
+            self.assertIsNone(fetch.call_args_list[2].args[0].get_header("Range"))
 
     def test_partial_feed_without_strong_validator_restarts(self):
         for etag in (None, 'W/"v1"'):