fix&tweaks

Author: VanyaBelyaev

ostap/tools/reweighters.py

@@ -244,16 +244,14 @@ def __init__( self                           , * ,
         self.__norm_factor         = numpy.float32( 1.0 )
         self.__mode                = None
         self.__scale_factors       = {}
-
 
-        n_features = num_features ( targer ) 
+        n_features = num_features ( target ) 
+        n_rows     = num_samples  ( target ) + num_samples  ( original )
+        data_size  = n_rows * n_features 
+        K          = 200000
+         
         n_jobs     = num_jobs     ( params )
-
-        ## 
-        if   100 < n_features : n_jobs = min ( 16 , n_jobs )
-        elif  20 < n_features : n_jobs = min (  8 , n_jobs )
-        elif  10 < n_features : n_jobs = min (  4 , n_jobs )
-        else                  : n_jobs = min (  2 , n_jobs )
+        n_jobs     = min ( n_jobs , max ( 1 , math.floor ( data_size / K ) ) )
         ## 
         params [ 'n_jobs' ] = n_jobs 
 
@@ -1216,7 +1214,7 @@ def _train_single_model ( self,
 
             # Destroy heavy training DMatrix objects and training C++ booster instance
             del dtrain, dval, model,
-            del X_tr, y_tr , w_tr, X_v , Y_v, w_v 
+            del X_tr, y_tr , w_tr, X_v , y_v, w_v 
 
             # Re-instantiate clean C++ booster dedicated strictly to inference
             model = XGBoost.Booster()
@@ -1402,7 +1400,7 @@ def _train_single_model ( self    ,
         verbose_level = params.get ('verbose' , False)
         model.fit ( trn_pool , verbose = verbose_level, **fit_kwargs)
 
-        import gc, pickle  
+        import pickle  
         from ostap.utils.memory import memory
 
         with memory ( 'MEMORY CATBOOST' , logger = logger ) : 
@@ -1411,7 +1409,7 @@ def _train_single_model ( self    ,
             model_bytes = pickle.dumps ( model ) 
 
             # Destroy training pools and heavy training model instance
-            del trn_pool, val_pool, model, X_tr, y_tr, w_tr, X_va, y_va, w_va
+            del trn_pool, val_pool, model, X_tr, y_tr, w_tr, X_v, y_v, w_v
 
             # Re-instantiate clean, standalone inference engine
             model = pickle.loads ( model_bytes )

ostap/tools/tests/test_tools_reweight5.py

@@ -156,6 +156,10 @@ def make_datasets2 ( n_samples = 5000 ,
                 COMPARATOR2 ( parallel = True , nToys = 100 ) ,
                 COMPARATOR3 ( parallel = True , nToys = 100 ) ) 
 
+if hasCatBoost () :  
+    from ostap.stats.adval        import ADVAL_CATB  as COMPARATOR7
+    comparators += ( COMPARATOR7 ( parallel = True , nToys = 25 ) , ) 
+    
 if hasLightGBM () :
     from ostap.stats.adval        import ADVAL_LGBM  as COMPARATOR5
     comparators += ( COMPARATOR5 ( parallel = True , nToys = 25 ) , ) 
@@ -164,10 +168,6 @@ def make_datasets2 ( n_samples = 5000 ,
     from ostap.stats.adval        import ADVAL_XGB  as COMPARATOR6
     comparators += ( COMPARATOR6 ( parallel = False , nToys = 25 , n_jobs = 1) , ) 
 
-if False and hasCatBoost () :  
-    from ostap.stats.adval        import ADVAL_CATB  as COMPARATOR7
-    comparators += ( COMPARATOR7 ( parallel = True , nToys = 25 ) , ) 
-
 if hasSkLearn (): 
     from ostap.stats.adval        import ADVAL_HGBC  as COMPARATOR8
     comparators += ( COMPARATOR8 ( parallel = True , nToys = 25 ) , )
@@ -266,7 +266,7 @@ def run_reweight ( n_dim     = 3     ,
 # =============================================================================
 if '__main__' == __name__ :
 
-    run_reweight ( 3 , 3000 ) 
+    run_reweight ( 3 , 4000 ) 
 
 # =============================================================================
 

ostap/utils/basic.py

@@ -333,20 +333,33 @@ def wm_print ( wm , with_category = True ) :
 
 # ==============================================================================
 ## key-word arguments related to number of cores/threads to be used 
-njobs_kwords = ( 'numthreads'   , 'numthread'   , 
-                 'nthreads'     , 'nthread'     ,
-                 'numjobs'      , 'numjob'      ,
-                 'njobs'        , 'njob'        ,
-                 'threadcounts' , 'threadcount' )
+njobs_kwords = frozenset( [ 'numthreads'   , 'numthread'   , 
+                            'nthreads'     , 'nthread'     ,
+                            'numjobs'      , 'numjob'      ,
+                            'njobs'        , 'njob'        ,
+                            'threadcounts' , 'threadcount' ] )
 # ==============================================================================
 ## get the value of "n_jobs/num_threads/thread_count" parameter 
-def num_jobs ( params , defval = -1 ) :
+def num_jobs ( params = {} , defval = -1 , **kwargs ) :
     """ Get the value of "n_jobs/num_threads/thread_count" parameter
     """
-    nj = defval 
-    for kw in njobs_kwords :
-        key = kw.lower().strip().replace('_','')
-        nj  = params.pop ( kkey , nj )
+    nj    = defval
+    found = set()  
+    for k, v in params.items () : 
+        kw  = k.lower().strip().replace('_','').replace(' ','').replace('-','' )
+        if kw in njobs_kwords :
+            found.add ( k )
+            if  isinstance ( v , int ) :  nj = v 
+    
+    ##  remove the keys 
+    for k in found : params.pop ( k , None )
+    
+    ## loop over keyword arguments 
+    for k , v in kwargs :
+        kw = k.lower().strip().replace('_','').replace(' ','').replace('-','' )
+        if kw in njobs_kwords :
+            if isinstance ( v , int ) : nj = v
+            
     ## 
     if not nj : return 1
     ##