Skip to content

Commit e590470

Browse files
victor-pcllvictor-pcll
committed
Hsitogram task tied - init
1 parent 49f9931 commit e590470

14 files changed

Lines changed: 1446 additions & 541 deletions

‎experiments/BO/BO_exp_cluster.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,9 @@ def run_experiment(D=100, L=2, beta=1.0,
171171
"MSE_mean": float(np.mean(MSE_runs)),
172172
"MSE_std": float(np.std(MSE_runs, ddof=1)) if len(MSE_runs) > 1 else 0.0,
173173
"label_err_mean": float(np.mean(label_err_runs)/D**2),
174-
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**4) if len(label_err_runs) > 1 else 0.0,
174+
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**2) if len(label_err_runs) > 1 else 0.0,
175175
"label_err_mean_noise": float(np.mean(label_err_runs_noise)/D**2),
176-
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**4) if len(label_err_runs_noise) > 1 else 0.0,
176+
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**2) if len(label_err_runs_noise) > 1 else 0.0,
177177
"train_data_mean": float(np.mean(train_data_runs)/D),
178178
"train_reg_mean": float(np.mean(train_reg_runs)/D),
179179
"train_total_mean": float(np.mean(total_loss_runs)/D),

‎experiments/BO/BO_exp_cluster.sbatch‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#SBATCH --mem-per-cpu=7000
1313

1414
# Create a timestamp for the run directory
15-
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
15+
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M%")
1616

1717
# Load all modules
1818
module load gcc

‎experiments/ERM_S/ERM_S_exp_cluster.py‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,22 @@ def __init__(self, input_dim, number_tokens, norm=1.0, beta=1.0):
4040
self.D = input_dim
4141
self.L = number_tokens
4242
self.S = nn.Linear(input_dim, input_dim, bias=False)
43-
# Initialisation random
44-
self.S.weight.data.uniform_(0.0, norm)
43+
self.S.weight.data.uniform_(0.0, norm) # Initialisation random
44+
45+
def init_teacher(self, R):
46+
"""
47+
Initialize S as a PSD matrix: S = W W^T / sqrt(r * D)
48+
where W is a random Gaussian matrix.
49+
"""
50+
# Create a random weight matrix W ~ N(0, 1)
51+
self.R = R
52+
W = torch.randn(self.D, self.R, device=self.S.weight.device)
53+
54+
# Compute PSD matrix
55+
S_psd = (W @ W.T) / np.sqrt(self.R * self.D)
56+
57+
# Copy into model weights
58+
self.S.weight.data = S_psd.clone()
4559

4660

4761
def forward(self, x, delta_in):
@@ -96,6 +110,8 @@ def run_experiment(alpha_idx=0, D=100, L=2, rho=1.00, rho_star=0.5, beta=1.0,
96110

97111
all_results = []
98112

113+
R_star = int(rho_star * D)
114+
99115
if run_index is not None and alpha_list is not None:
100116
alpha_list = [alpha_list[run_index]]
101117

@@ -109,13 +125,14 @@ def run_experiment(alpha_idx=0, D=100, L=2, rho=1.00, rho_star=0.5, beta=1.0,
109125
N = int(alpha * D**2)
110126
with torch.no_grad():
111127
teacher = Net(D, L, norm=1.0, beta=beta_star)
128+
teacher.init_teacher(R_star)
112129
S_teacher = teacher.S.weight.detach().cpu().numpy()
113130

114131
MSE_runs, label_err_runs, label_err_runs_noise = [], [], []
115132
train_data_runs, train_reg_runs, total_loss_runs = [], [], []
116133
S_runs = []
117134

118-
for i in range(samples):
135+
for _ in range(samples):
119136
x_train = torch.normal(0, 1, (N, L, D))
120137
with torch.no_grad():
121138
y_train = teacher(x_train, delta_in=Delta_in)
@@ -157,9 +174,9 @@ def run_experiment(alpha_idx=0, D=100, L=2, rho=1.00, rho_star=0.5, beta=1.0,
157174
"MSE_mean": float(np.mean(MSE_runs)),
158175
"MSE_std": float(np.std(MSE_runs, ddof=1)) if len(MSE_runs) > 1 else 0.0,
159176
"label_err_mean": float(np.mean(label_err_runs)/D**2),
160-
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**4) if len(label_err_runs) > 1 else 0.0,
177+
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**2) if len(label_err_runs) > 1 else 0.0,
161178
"label_err_mean_noise": float(np.mean(label_err_runs_noise)/D**2),
162-
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**4) if len(label_err_runs_noise) > 1 else 0.0,
179+
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**2) if len(label_err_runs_noise) > 1 else 0.0,
163180
"train_data_mean": float(np.mean(train_data_runs)/D**2),
164181
"train_reg_mean": float(np.mean(train_reg_runs)/D**2),
165182
"train_total_mean": float(np.mean(total_loss_runs)/D**2),
@@ -231,9 +248,9 @@ def get_run_dir(base_path="/home/peucelle/tpiv-simulations/results"):
231248
# Charger la configuration
232249
config = {
233250
"verbose": False,
234-
"alpha_start": 0.005,
235-
"alpha_end": 1.0,
236-
"alpha_steps": 10,
251+
"alpha_start": 0.0005,
252+
"alpha_end": 0.05,
253+
"alpha_steps": 15,
237254
"d": 100,
238255
"L": 2,
239256
"beta": 1.0,
@@ -247,7 +264,7 @@ def get_run_dir(base_path="/home/peucelle/tpiv-simulations/results"):
247264
"tol": 1e-6,
248265
"n_test": 2000,
249266
"rho": 1.0,
250-
"rho_star": 0.5
267+
"rho_star": 0.5
251268
}
252269

253270
# Initialiser les graines

‎experiments/ERM_S/ERM_S_exp_cluster.sbatch‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
#SBATCH --job-name=ERM_untied_exp
2+
#SBATCH --job-name=ERM_S_exp
33
#SBATCH --nodes=1
44
#SBATCH --ntasks=1
55
#SBATCH --cpus-per-task=1
@@ -8,11 +8,11 @@
88
#SBATCH --output=/home/peucelle/tpiv-simulations/results/run_%x/log.txt
99
#SBATCH --error=/home/peucelle/tpiv-simulations/results/run_%x/err.txt
1010
#SBATCH --chdir /home/peucelle/tpiv-simulations/experiments
11-
#SBATCH --array=0-9
11+
#SBATCH --array=0-14
1212
#SBATCH --mem-per-cpu=7000
1313

1414
# Create a timestamp for the run directory
15-
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
15+
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M")
1616

1717
# Load all modules
1818
module load gcc

‎experiments/ERM_tied/ERM_tied_exp_cluster.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,9 +191,9 @@ def run_experiment(alpha_list, base_dir, run_index, D, L, rho, rho_star, beta, l
191191
"MSE_mean": float(np.mean(MSE_runs)),
192192
"MSE_std": float(np.std(MSE_runs, ddof=1)) if len(MSE_runs) > 1 else 0.0,
193193
"label_err_mean": float(np.mean(label_err_runs)/D**2),
194-
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**4) if len(label_err_runs) > 1 else 0.0,
194+
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**2) if len(label_err_runs) > 1 else 0.0,
195195
"label_err_mean_noise": float(np.mean(label_err_runs_noise)/D**2),
196-
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**4) if len(label_err_runs_noise) > 1 else 0.0,
196+
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**2) if len(label_err_runs_noise) > 1 else 0.0,
197197
"train_data_mean": float(np.mean(train_data_runs)/D**2),
198198
"train_reg_mean": float(np.mean(train_reg_runs)/D**2),
199199
"train_total_mean": float(np.mean(total_loss_runs)/D**2),
@@ -286,7 +286,10 @@ def run_experiment(alpha_list, base_dir, run_index, D, L, rho, rho_star, beta, l
286286
config_csv_path = os.path.join(run_dir, "config.csv")
287287
try:
288288
df_config = pd.DataFrame([config])
289-
df_config.to_csv(config_csv_path, index=False)
289+
if os.path.isfile(config_csv_path):
290+
df_config.to_csv(config_csv_path, mode='a', header=False, index=False)
291+
else:
292+
df_config.to_csv(config_csv_path, mode='a', header=True, index=False)
290293
logger.info(f"💾 Configuration saved as CSV: {config_csv_path}")
291294
except Exception as e:
292295
logger.warning(f"[ERR] Failed to save config.csv: {e}")

‎experiments/ERM_tied/ERM_tied_exp_cluster.sbatch‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#SBATCH --mem-per-cpu=7000
1313

1414
# Create a timestamp for the run directory
15-
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
15+
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M")
1616

1717
# Load all modules
1818
module load gcc

‎experiments/ERM_untied/ERM_untied_exp_cluster.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ def run_experiment(alpha_idx=0, D=100, L=2, rho=1.00, rho_star=0.5, beta=1.0,
176176
"MSE_mean": float(np.mean(MSE_runs)),
177177
"MSE_std": float(np.std(MSE_runs, ddof=1)) if len(MSE_runs) > 1 else 0.0,
178178
"label_err_mean": float(np.mean(label_err_runs)/D**2),
179-
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**4) if len(label_err_runs) > 1 else 0.0,
179+
"label_err_std": float(np.std(label_err_runs, ddof=1)/D**2) if len(label_err_runs) > 1 else 0.0,
180180
"label_err_mean_noise": float(np.mean(label_err_runs_noise)/D**2),
181181
"label_err_std_noise": float(np.std(label_err_runs_noise, ddof=1)/D**4) if len(label_err_runs_noise) > 1 else 0.0,
182182
"train_data_mean": float(np.mean(train_data_runs)/D**2),
@@ -262,14 +262,14 @@ def get_run_dir(base_path="/home/peucelle/tpiv-simulations/results"):
262262
"verbose": False,
263263
"alpha_start": 0.005,
264264
"alpha_end": 1.0,
265-
"alpha_steps": 10,
265+
"alpha_steps": 15,
266266
"d": 100,
267267
"L": 2,
268268
"beta": 1.0,
269-
"lmbda": [0.1, 0.01, 0.001, 0.0005, 0.0001, 0.00001],
269+
"lmbda": [0.1, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00001],
270270
"Delta_in": 0.5,
271271
"Delta_list": [0.0],
272-
"samples": 8,
272+
"samples": 16,
273273
"T": 10000,
274274
"lr": 0.1,
275275
"norm_init": 1.0,

‎experiments/ERM_untied/ERM_untied_exp_cluster.sbatch‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@
88
#SBATCH --output=/home/peucelle/tpiv-simulations/results/run_%x/log.txt
99
#SBATCH --error=/home/peucelle/tpiv-simulations/results/run_%x/err.txt
1010
#SBATCH --chdir /home/peucelle/tpiv-simulations/experiments
11-
#SBATCH --array=0-9
11+
#SBATCH --array=0-14
1212
#SBATCH --mem-per-cpu=7000
1313

1414
# Create a timestamp for the run directory
15-
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
15+
RUN_TIMESTAMP=$(date +"%Y%m%d_%H%M")
1616

1717
# Load all modules
1818
module load gcc

0 commit comments

Comments
 (0)