(core) Limited model download threads to 8 and added minor timeout to prevent issues with Zenodo returning HTTP Error 429: TOO MANY REQUESTS (limit seems to be 56 socket connections per some period)

Author: amkrajewski

pysipfenn/core/pysipfenn.py

@@ -3,7 +3,7 @@
 import gc
 import csv
 import json
-from time import perf_counter
+from time import perf_counter, sleep
 from typing import List, Union, Dict
 from importlib.resources import files as resources_files, as_file
 
@@ -264,11 +264,12 @@ def downloadModels(self, network: str = 'all') -> None:
                             downloadObject = SmartDL(
                                 self.models[net]['URL_ONNX'],
                                 f'{modelPath}/{net}.onnx',
-                                threads=16,
+                                threads=8,
                                 request_args=downloadHeaders
                             )
                             downloadObject.start()
                             print('\nONNX Network Successfully Fetched.')
+                            sleep(1)  # Sleep for a second to prevent overwhelming the server with requests
                         else:
                             print(f'{net} not detected on disk and no ONNX URL has been provided.')
                     else:
@@ -284,11 +285,12 @@ def downloadModels(self, network: str = 'all') -> None:
                 downloadObject = SmartDL(
                     self.models[network]['URL_ONNX'],
                     f'{modelPath}/{network}.onnx',
-                    threads=16,
+                    threads=8,
                     request_args=downloadHeaders
                 )
                 downloadObject.start()
                 print('\nONNX Network Successfully Fetched.')
+                sleep(1)  # Sleep for a second to prevent overwhelming the server with requests
             # Not recognized
             else:
                 print('Network name not recognized')