fail on 404 if date > 2026-06-11 #352 (#353)

Author: fqrious

tests/full/other_tests/test_tasks.py

@@ -1,3 +1,4 @@
+from datetime import date
 from unittest.mock import patch, Mock
 from io import BytesIO
 import time
@@ -210,6 +211,19 @@ def test_download_file_retries_on_download_timeout(mock_read_into, mock_path, mo
     # Should have been called 3 times (2 failures + 1 success)
     assert mock_read_into.call_count == 3
 
+@patch('vulmatch.worker.tasks.requests.get')
+@patch('vulmatch.worker.tasks.read_into')
+def test_download_file_handles_empty_bundle_correctly(mock_read_into, mock_get, job, tmp_path, eager_celery):
+    mock_read_into.return_value = 2
+    tempdir = str(tmp_path)
+    mock_response = Mock()
+    mock_response.status_code = 200
+    mock_response.url = 'https://example.com/file.json'
+    mock_get.return_value = mock_response
+    result = tasks.download_file.delay('https://example.com/file.json', tempdir, job_id=job.id)
+    assert result.get() == ""
+
+
 
 @patch('vulmatch.worker.tasks.requests.get')
 def test_download_file_max_retries_exceeded(mock_get, job, tmp_path, eager_celery):
@@ -231,3 +245,35 @@ def test_download_file_max_retries_exceeded(mock_get, job, tmp_path, eager_celer
     assert 'after 3 retries' in job.errors[0]
     assert 'Network error' in job.errors[0]
 
+
+@pytest.mark.parametrize(
+    "d,should_fail",
+    [
+        (date(2027, 1, 1), True),
+        (date(2025, 1, 1), False),
+        (date(2026, 10, 11), True),
+        (date(2026, 6, 10), False),
+    ]
+)
+@patch('vulmatch.worker.tasks.requests.get')
+def test_download_file__fails_on_404(mock_get, job, tmp_path, d, should_fail, eager_celery):
+    """Test download_file fails after max retries"""
+    from requests.models import Response 
+    r = Response()
+    r.status_code = 404
+    mock_get.return_value = r
+    tempdir = str(tmp_path)
+    
+    # with pytest.raises(requests.exceptions.RequestException):
+    r = tasks.download_file.delay('https://example.com/file.json', tempdir, job_id=job.id, file_date=d)
+    
+    # Should have been called 4 times (1 initial + 3 retries as per max_retries=3)
+    assert mock_get.call_count == 1
+    assert r.failed() == should_fail
+    
+    job.refresh_from_db()
+    assert len(job.errors) == 1
+    if should_fail:
+        assert "file does not exist on remote server: https://example.com/file.json" in job.errors[0]
+
+

utilities/s2a_importer/insert_archive_cve.py

@@ -70,6 +70,9 @@ def create_directory(path):
 def download_file(url, destination):
     response = requests.get(url)
     if response.status_code == 200:
+        if response.content == "{}":
+            print(f"No STIX objects created for date")
+            return True
         with open(destination, 'wb') as file:
             file.write(response.content)
         print(f"Downloaded file: {destination}")

vulmatch/worker/tasks.py

@@ -69,7 +69,7 @@ def run_nvd_task(data, job: Job, nvd_type='cve'):
     tasks = []
     for d in dates:
         url = urljoin(settings.CVE2STIX_BUCKET_ROOT_PATH, daily_url(d, nvd_type))
-        task = download_file.si(url, temp_dir, job_id=job.id)
+        task = download_file.si(url, temp_dir, job_id=job.id, file_date=d)
         task |= upload_file.s(f'nvd_{nvd_type}', stix2arango_note=f"vulmatch-{nvd_type}-date={d.strftime('%Y-%m-%d')}", job_id=job.id, params=job.parameters)
         task.set_immutable(True)
         tasks.append(task)
@@ -145,8 +145,11 @@ def read_into(streamed_resp: requests.Response, fp, chunk_size=1024*1024, read_t
                 raise DownloadTimeoutError(downloaded, total)
     return downloaded
 
+class MissingFileError(Exception):
+    pass
+
 @app.task(bind=True, base=CustomTask, max_retries=3, default_retry_delay=5, autoretry_for=(requests.exceptions.RequestException, DownloadTimeoutError))
-def download_file(self, urlpath, tempdir, job_id=None):
+def download_file(self, urlpath, tempdir, job_id=None, file_date: date=None):
     Path(tempdir).mkdir(parents=True, exist_ok=True)
     logging.info('downloading bundle at `%s`', urlpath)
     job = Job.objects.get(pk=job_id)
@@ -159,9 +162,14 @@ def download_file(self, urlpath, tempdir, job_id=None):
             filename = Path(tempdir)/resp.url.split('/')[-1]
             with filename.open('wb') as f:
                 total_bytes = read_into(resp, f, read_timeout=settings.DOWNLOAD_TIMEOUT_SECONDS)
+                if total_bytes == 2: # see https://github.com/muchdogesec/cve2stix/issues/128 [body is {}]
+                    logging.info("found an empty bundle, skipping...")
+                    return ""
                 logging.info(f'downloaded {total_bytes} bytes into {filename}')
             return str(filename)
         elif resp.status_code == 404:
+            if file_date and file_date > date(2026, 6, 11): # see https://github.com/muchdogesec/vulmatch/issues/352
+                raise MissingFileError(f"file does not exist on remote server: {urlpath}")
             job.errors.append(f'{resp.url} not found')
         else:
             job.errors.append(f'{resp.url} failed with status code: {resp.status_code}')