Skip to content

Commit a9a54fc

Browse files
VanyaBelyaevVanyaBelyaev
committed
?
1 parent aaf61b3 commit a9a54fc

3 files changed

Lines changed: 85 additions & 76 deletions

File tree

‎ostap/tools/reweighters.py‎

Lines changed: 78 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
from ostap.utils.basic import numcpu, num_jobs, NoContext
2727
from ostap.logger.utils import map2table_ex
2828
from ostap.logger.pretty import nice_print
29-
from ostap.logger.symbols import arrow_right
3029
from ostap.utils.progress_bar import progress_bar, ProgressBar
3130
from ostap.tools.reweighter import Reweighter
3231
from ostap.stats.counters import SE, table_counters
@@ -39,6 +38,7 @@
3938
check_all ,
4039
nEff )
4140
from ostap.stats.tools import hasSkLearn
41+
from ostap.utils.memory import memory
4242
import ostap.logger.symbols as S
4343
import numpy, abc, math, gc
4444
# =============================================================================
@@ -248,14 +248,12 @@ def __init__( self , * ,
248248
n_features = num_features ( target )
249249
n_rows = num_samples ( target ) + num_samples ( original )
250250
data_size = n_rows * n_features
251-
K = 200000
251+
K = 100_000
252252

253253
n_jobs = num_jobs ( params )
254254
n_jobs = min ( n_jobs , max ( 1 , math.floor ( data_size / K ) ) )
255-
##
256255
params [ 'n_jobs' ] = n_jobs
257-
258-
256+
259257
reg_case = self.needs_regularization ( original = original ,
260258
target = target ,
261259
original_weight = original_weight ,
@@ -330,7 +328,7 @@ def __init__( self , * ,
330328
if neff_after < 0.10 * neff_before :
331329
n1 = nice_print ( neff_before )
332330
n2 = nice_print ( neff_after )
333-
logger.warning ( "%s: Large degradation of nEff %s %s %s" % ( self.method , n1 , arrow_right , n2 ) )
331+
logger.warning ( "%s: Large degradation of nEff %s %s %s" % ( self.method , n1 , S.arrow_right , n2 ) )
334332

335333
clipped_count = numpy.sum ( original_ratios >= ( self.__clip_threshold * 0.99 ) )
336334
if 0 < clipped_count :
@@ -652,35 +650,25 @@ def __fit_eval_stream ( self ,
652650
n_folds = 1
653651

654652
stream_models = []
655-
print ( 'PROGRESS-BAR', typename ( self ) )
656653
for train_idx, val_idx in progress_bar ( splits ,
657654
max_value = n_folds ,
658655
description = 'Folds:' ,
659656
silent = not self.progress or self.silent or 1 >= self.n_splits ) :
660657

661-
print ( 'START-LOOP', typename ( self ) )
662-
663658
X_tr, y_tr = X_comb [ train_idx ] , y_comb [ train_idx ]
664659
X_va, y_va = X_comb [ val_idx ] , y_comb [ val_idx ]
665660

666661
w_tr = w_comb [ train_idx ] if w_comb is not None else None
667662
w_va = w_comb [ val_idx ] if w_comb is not None else None
668-
669-
670-
print ( 'BEFORE-TRAIN', typename ( self ) )
671663

672664
model, _ = self._train_single_model ( X_tr, y_tr, w_tr, X_va, y_va, w_va )
673665
stream_models.append( model )
674666

675-
print ( 'BEFORE-PREDICT', typename ( self ) )
676-
677667
raw_val_p = self._predict_single_model ( model, X_va )
678668
if 1 < raw_val_p.ndim : raw_val_p = raw_val_p[:, 1]
679669

680670
oof_raw [ val_idx ] = raw_val_p.astype ( numpy.float32, copy = False )
681671

682-
print ( 'END-LOOP', typename ( self ) )
683-
684672
self.__fitted_models [ stream_key ] = stream_models
685673
self.__scale_factors [ stream_key ] = scale_factor
686674
self.__priors [ stream_key ] = numpy.float32 ( 1.0 )
@@ -982,23 +970,17 @@ def _train_single_model ( self ,
982970
if early_stopping_rounds is not None :
983971
callbacks.append ( LightGBM.early_stopping ( stopping_rounds = early_stopping_rounds , verbose = False ) )
984972

985-
print ( 'BEFORE-LIGHTGBM-TRAIN' , typename ( self ) )
986973
model = LightGBM.train ( params = params ,
987974
train_set = trn_data ,
988975
num_boost_round = num_boost_round ,
989976
valid_sets = [ val_data ] ,
990977
callbacks = callbacks )
991978

992-
print ( 'AFTER-LIGHTGBM-PREDICT' , typename ( self ) )
993-
994-
data_size = num_features ( X_train ) * num_samples ( X_train ) * 4 / ( 1024 * 1024 )
995-
996-
from ostap.utils.memory import memory
997-
998-
print ( 'data_size [MB]' , data_size )
999-
1000-
with memory ( "MEMORY LIGHTGBM" , logger = logger ) :
1001-
979+
# =====================================================================
980+
## PHOENIX
981+
# =====================================================================
982+
if True : # ===========================================================
983+
# =================================================================
1002984
model_bytes = model.model_to_string()
1003985

1004986
model.free_dataset()
@@ -1011,25 +993,9 @@ def _train_single_model ( self ,
1011993
del model_bytes
1012994

1013995
gc.collect ()
1014-
1015996

1016-
print ( 'BEFORE-LIGHTGBM-PREDICT' , typename ( self ) )
1017-
1018-
"""
1019-
print ('--- ТЕСТ ИЗОЛЯЦИИ: СТАРТ ТРЕНИРОВКИ БЕЗ VALIDATION ---', flush=True)
1020-
1021-
model = LightGBM.train (
1022-
params = params ,
1023-
train_set = trn_data ,
1024-
num_boost_round = num_boost_round,
1025-
valid_sets = None , # Никакой валидации
1026-
callbacks = None # Никаких колбэков ранней остановки
1027-
)
1028-
print ('--- ТЕСТ ИЗОЛЯЦИИ: УСПЕХ! ---', flush=True)
1029-
"""
1030-
997+
1031998
val_preds = self._predict_single_model ( model , X_val )
1032-
print ( 'AFTER-LIGHTGBM-PREDICT' , typename ( self ) )
1033999

10341000
return model , val_preds
10351001

@@ -1205,9 +1171,11 @@ def _train_single_model ( self,
12051171
verbose_eval = False
12061172
)
12071173

1208-
from ostap.utils.memory import memory
1209-
1210-
with memory ( "MEMORY XGBOOST" , logger = logger ) :
1174+
# =====================================================================
1175+
## PHOENIX
1176+
# =====================================================================
1177+
if True : # ===========================================================
1178+
# =================================================================
12111179

12121180
# Export tree structure into a lightweight binary/JSON byte array
12131181
model_bytes = model.save_raw ( raw_format = "json" )
@@ -1222,7 +1190,7 @@ def _train_single_model ( self,
12221190
del model_bytes
12231191

12241192
gc.collect()
1225-
1193+
12261194
# Compute validation predictions safely using pre-allocated contiguous array
12271195
val_preds = self._predict_single_model ( model , X_val )
12281196

@@ -1400,10 +1368,12 @@ def _train_single_model ( self ,
14001368
verbose_level = params.get ('verbose' , False)
14011369
model.fit ( trn_pool , verbose = verbose_level, **fit_kwargs)
14021370

1403-
import pickle
1404-
from ostap.utils.memory import memory
1405-
1406-
with memory ( 'MEMORY CATBOOST' , logger = logger ) :
1371+
# =====================================================================
1372+
## PHOENIX
1373+
# =====================================================================
1374+
if True : # ===========================================================
1375+
# =================================================================
1376+
import pickle
14071377

14081378
# Extract tree geometry into a lightweight binary blob
14091379
model_bytes = pickle.dumps ( model )
@@ -1586,7 +1556,7 @@ def forward ( self , x ) :
15861556

15871557
device = self.params.get ( 'device' , 'cuda' if torch.cuda.is_available() else 'cpu' )
15881558
if not torch.cuda.is_available() : device = 'cpu'
1589-
device = torch.device ( device_str )
1559+
device = torch.device ( device)
15901560

15911561
patience = self.params.get ( 'patience' , 250 )
15921562
nepochs = self.params.get ( 'epochs' , 2000 )
@@ -1699,11 +1669,12 @@ def forward ( self , x ) :
16991669
if best_state is None :
17001670
best_state = { k : v.detach().cpu().clone() for k , v in model.state_dict().items() }
17011671

1702-
import gc
1703-
from ostap.utils.memory import memory
1672+
# =====================================================================
1673+
## PHOENIX
1674+
# =====================================================================
1675+
if True : # ===========================================================
1676+
# =================================================================
17041677

1705-
with memory ( 'MEMORY PYTORCH' , logger = logger ) :
1706-
17071678
# Destroy heavy training objects, optimizer states, and CUDA/CPU tensors
17081679
del model, optimizer, scheduler, criterion_base
17091680
del X_tr_t , y_tr_t , w_tr_t , X_va_t, y_va_t, w_va_t
@@ -1713,9 +1684,6 @@ def forward ( self , x ) :
17131684
##
17141685
if torch.cuda.is_available() : torch.cuda.empty_cache()
17151686

1716-
# -------------------------------------------------------------------------
1717-
# RESTORATION OF CLEAN INFERENCE MODEL
1718-
# -------------------------------------------------------------------------
17191687
# Instantiate a clean, stateless model dedicated strictly to inference
17201688
model = TabularMLP ( in_features = X_train.shape[1] ,
17211689
hidden_dims = hidden_dims ,
@@ -1771,7 +1739,8 @@ def _predict_single_model ( self , model , X ) :
17711739
'multi_class' ,
17721740
'verbose' ,
17731741
'warm_start' ,
1774-
'l1_ratio'
1742+
'l1_ratio' ,
1743+
'n_jobs'
17751744
)
17761745
# =============================================================================
17771746
## @class LogRegressionDensityReweighter
@@ -1798,9 +1767,16 @@ def __init__( self , * ,
17981767
config = { 'C' : 1.0 ,
17991768
'solver' : 'lbfgs' ,
18001769
'max_iter' : 2000 ,
1801-
'polynomials' : 2 }
1770+
'polynomials' : 2 }
1771+
18021772
config.update ( params )
18031773

1774+
poly = config.get ( 'polynomials', 2 )
1775+
n_features = num_features ( original )
1776+
1777+
if 20 <= n_features and 1 < poly :
1778+
config [ 'polynomials' ] = 1
1779+
18041780
super().__init__ ( original = original ,
18051781
target = target ,
18061782
original_weight = original_weight ,
@@ -1817,8 +1793,10 @@ def method ( self ) :
18171793
def regularization ( self , params , n_features , n_samples ) :
18181794
""" Dynamic regularization rules for Logistic Regression.
18191795
"""
1820-
current_c = params.get ( 'C', 1.0 )
1821-
params [ 'C' ] = max ( 0.1, min ( current_c, 0.5 ) )
1796+
penalty = params.get ( 'penalty' , 'l2' )
1797+
if penalty not in ( None, 'none' ) :
1798+
current_c = params.get ( 'C', 1.0 )
1799+
params [ 'C' ] = max ( 0.1, min ( current_c, 0.5 ) )
18221800
return params
18231801

18241802
# =========================================================================
@@ -1837,13 +1815,15 @@ def _train_single_model ( self ,
18371815
params.pop ( 'n_jobs', None )
18381816

18391817
poly = params.pop ( 'polynomials', 2 )
1840-
1818+
n_features = num_features ( X_train )
1819+
if 20 <= n_features and 1 < poly : poly = 1
1820+
18411821
lr_kwargs = { k: v for k, v in params.items() if k in valid_LR_params }
18421822

18431823
# Build the pipeline
18441824
steps = [ ( 'scaler1' , StandardScaler () ) ]
18451825

1846-
if poly and isinstance ( poly , int ) and 0 < poly <= 5 :
1826+
if poly and isinstance ( poly , int ) and 1 < poly <= 5 :
18471827
steps += [ ( 'poly' , PolynomialFeatures ( degree = poly , include_bias = False ) ) ]
18481828
steps += [ ( 'scaler2' , StandardScaler () ) ]
18491829

@@ -1857,6 +1837,22 @@ def _train_single_model ( self ,
18571837
if w_tr is not None : model.fit ( X_train , y_train , logistic__sample_weight = w_tr )
18581838
else : model.fit ( X_train , y_train )
18591839

1840+
# =====================================================================
1841+
## PHOENIX
1842+
# =====================================================================
1843+
if True : # ===========================================================
1844+
# =================================================================
1845+
import pickle
1846+
1847+
model_bytes = pickle.dumps ( model )
1848+
del model
1849+
del w_tr
1850+
1851+
model = pickle.loads ( model_bytes )
1852+
del model_bytes
1853+
1854+
gc.collect()
1855+
18601856
val_preds = self._predict_single_model ( model , X_val )
18611857
return model , val_preds.astype ( numpy.float32 , copy = False )
18621858

@@ -1865,8 +1861,8 @@ def _train_single_model ( self ,
18651861
def _predict_single_model ( self , model , X ) :
18661862
""" Predict probabilities using a Logistic Regression model.
18671863
"""
1868-
X_clean = numpy.ascontiguousarray ( X , dtype = numpy.float64 )
1869-
p = model.predict_proba ( X_clean ) [ : , 1 ]
1864+
X_clean = numpy.ascontiguousarray ( X , dtype = numpy.float32 )
1865+
p = model.predict_proba ( X_clean ) [ : , 1 ]
18701866
return p.astype ( numpy.float32 , copy = False )
18711867

18721868
# ==============================================================================
@@ -1979,6 +1975,20 @@ def __init__ ( self , * ,
19791975
original_weight = original_weight ,
19801976
target_weight = target_weight )
19811977

1978+
# =====================================================================
1979+
## PHOENIX
1980+
# =====================================================================
1981+
if True : # ===========================================================
1982+
# =================================================================
1983+
import pickle
1984+
1985+
model_bytes = pickle.dumps ( self.__reweighter )
1986+
del self.__reweighter
1987+
self.__reweighter = pickle.loads ( model_bytes )
1988+
del model_bytes
1989+
1990+
gc.collect()
1991+
19821992
self.__original_ratios = None
19831993
self.__original_reweighted_weights = None
19841994
if store_original_weights :

‎ostap/tools/tests/test_tools_reweight5.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
# @author Vanya BELYAEV Ivan.Belyaev@itep.ru
77
# @date 2023-01-20
88
# =============================================================================
9-
"""Test for nD-reweighting machinery
9+
""" Test for nD-reweighting machinery
1010
"""
1111
# =============================================================================
1212
__version__ = "$Revision:"
@@ -16,8 +16,8 @@
1616
# =============================================================================
1717
## ATTENTION!
1818
import os
19-
os.environ [ "OMP_NUM_THREADS" ] = "1"
2019
os.environ [ "MKL_NUM_THREADS" ] = "1"
20+
os.environ [ "OMP_NUM_THREADS" ] = "1"
2121
os.environ [ "OPENBLAS_NUM_THREADS" ] = "1"
2222
# =============================================================================
2323
os.environ [ 'KMP_DUPLICATE_LIB_OK' ] = 'TRUE'
@@ -156,19 +156,19 @@ def make_datasets2 ( n_samples = 5000 ,
156156
COMPARATOR2 ( parallel = True , nToys = 100 ) ,
157157
COMPARATOR3 ( parallel = True , nToys = 100 ) )
158158

159-
if hasCatBoost () :
159+
if False and hasCatBoost () :
160160
from ostap.stats.adval import ADVAL_CATB as COMPARATOR7
161161
comparators += ( COMPARATOR7 ( parallel = True , nToys = 25 ) , )
162162

163-
if hasLightGBM () :
163+
if False and hasLightGBM () :
164164
from ostap.stats.adval import ADVAL_LGBM as COMPARATOR5
165165
comparators += ( COMPARATOR5 ( parallel = True , nToys = 25 ) , )
166166

167-
if hasXGBoost () :
167+
if False and hasXGBoost () :
168168
from ostap.stats.adval import ADVAL_XGB as COMPARATOR6
169169
comparators += ( COMPARATOR6 ( parallel = False , nToys = 25 , n_jobs = 1) , )
170170

171-
if hasSkLearn ():
171+
if False and hasSkLearn ():
172172
from ostap.stats.adval import ADVAL_HGBC as COMPARATOR8
173173
comparators += ( COMPARATOR8 ( parallel = True , nToys = 25 ) , )
174174

@@ -266,7 +266,7 @@ def run_reweight ( n_dim = 3 ,
266266
# =============================================================================
267267
if '__main__' == __name__ :
268268

269-
run_reweight ( 3 , 4000 )
269+
run_reweight ( 3 , 3000 )
270270

271271
# =============================================================================
272272

‎scripts/.thisostap.sh.in‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,6 @@ export KMP_BLOCKTIME=0
193193
export OMP_PROC_BIND=false
194194
export OMP_WAIT_POLICY=PASSIVE
195195

196-
197196
# =====================================================================
198197
env_after=$(printenv | sort)
199198
flag=$(echo "${show_environment:-true}" | tr '[:upper:]' '[:lower:]')

0 commit comments

Comments
 (0)