Skip to content

Commit 77920d7

Browse files
Merge pull request #120 from drunkenbot-ai/develop
fix and optimize instruction and conversation fine tunning
2 parents 2369cba + e032d45 commit 77920d7

7 files changed

Lines changed: 468 additions & 17 deletions

File tree

interface/screens/fine_tuning_screen.py

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,36 @@ def apply_recommended_fine_tune_settings(self) -> None:
150150
synced = self._sync_architecture_from_fine_tune_base()
151151
self._set_combo_text(self.peft_method, "LoRA adapters")
152152
self.lora_dropout.setValue(0.05)
153-
self._set_combo_text(self.lora_targets, "Attention projections")
153+
self._set_combo_text(self.lora_targets, "Attention + MLP")
154154
self.max_grad_norm.setValue(0.5)
155155
self.weight_decay.setValue(0.05)
156+
# Dynamically determine recommended batch size and gradient accumulation from detected hardware
157+
rec_batch = 16
158+
if torch.cuda.is_available():
159+
try:
160+
_, total_bytes = torch.cuda.mem_get_info()
161+
total_gb = total_bytes / (1024 ** 3)
162+
if total_gb >= 22.0:
163+
rec_batch = 32
164+
elif total_gb >= 10.0:
165+
rec_batch = 16
166+
elif total_gb >= 6.0:
167+
rec_batch = 8
168+
else:
169+
rec_batch = 4
170+
except Exception:
171+
rec_batch = 16
172+
else:
173+
rec_batch = 4
174+
if hasattr(self, "batch_size"):
175+
self.batch_size.setValue(min(self.batch_size.maximum(), rec_batch))
176+
if hasattr(self, "gradient_accumulation"):
177+
target_eff = 32
178+
self.gradient_accumulation.setValue(max(1, (target_eff + rec_batch - 1) // rec_batch))
179+
if hasattr(self, "precision"):
180+
bf16_supported = torch.cuda.is_available() and getattr(torch.cuda, "is_bf16_supported", lambda: False)()
181+
rec_precision = "BF16" if bf16_supported else ("FP16" if torch.cuda.is_available() else "FP32")
182+
self._set_combo_text(self.precision, rec_precision)
156183
self._set_combo_by_data(self.scheduler_name, "cosine", {
157184
"warmup_linear": "Warmup linear",
158185
"cosine": "Cosine decay",
@@ -161,30 +188,35 @@ def apply_recommended_fine_tune_settings(self) -> None:
161188
"constant": "Constant",
162189
})
163190
if stage == "conversation":
191+
self._set_combo_text(self.lora_targets, "Attention + MLP")
164192
self.lora_rank.setValue(16)
165193
self.lora_alpha.setValue(32.0)
166194
self.learning_rate.setValue(0.00003)
167195
self.epochs.setValue(max(1, min(self.epochs.value(), 2)))
168196
elif stage == "tool_call":
197+
self._set_combo_text(self.lora_targets, "Attention + MLP")
169198
self.lora_rank.setValue(16)
170199
self.lora_alpha.setValue(32.0)
171200
self.learning_rate.setValue(0.00003)
172201
self.epochs.setValue(max(1, min(self.epochs.value(), 3)))
173202
elif stage == "code":
203+
self._set_combo_text(self.lora_targets, "Attention projections")
174204
self.lora_rank.setValue(8)
175205
self.lora_alpha.setValue(16.0)
176206
self.lora_dropout.setValue(0.05)
177207
self.learning_rate.setValue(0.00005)
178208
self.max_grad_norm.setValue(0.5)
179209
self.epochs.setValue(max(1, min(self.epochs.value(), 3)))
180210
elif stage == "instruction":
181-
self.lora_rank.setValue(8)
182-
self.lora_alpha.setValue(16.0)
211+
self._set_combo_text(self.lora_targets, "Attention + MLP")
212+
self.lora_rank.setValue(16)
213+
self.lora_alpha.setValue(32.0)
183214
self.learning_rate.setValue(0.00005)
184215
self.epochs.setValue(max(1, min(self.epochs.value(), 3)))
185216
else:
186-
self.lora_rank.setValue(8)
187-
self.lora_alpha.setValue(16.0)
217+
self._set_combo_text(self.lora_targets, "Attention + MLP")
218+
self.lora_rank.setValue(16)
219+
self.lora_alpha.setValue(32.0)
188220
self.learning_rate.setValue(0.00005)
189221
self._update_training_mode_controls()
190222
message = "Recommended LoRA settings applied."

interface/screens/training_estimation.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,24 @@ def _update_architecture_advisor(
122122
notes.append("Grouped/multi-query attention reduces KV memory and is useful for longer contexts.")
123123
if model_config.mlp_type == "swiglu" and model_config.norm_type == "rmsnorm":
124124
notes.append("Llama-like blocks improve modern compatibility but must match checkpoints when resuming.")
125-
if training_config is not None and training_config.device == "cuda" and vram_bytes > 3.5 * 1024**3:
126-
notes.append("Estimated VRAM is high for 4 GB GPUs. Try lower batch, context, embedding, or layers.")
127-
if label == "Advisor: balanced":
128-
label = "Advisor: memory check"
125+
if training_config is not None and training_config.device.startswith("cuda") and vram_bytes > 0:
126+
if torch.cuda.is_available():
127+
try:
128+
free_bytes, total_bytes = torch.cuda.mem_get_info()
129+
if vram_bytes > free_bytes * 0.85:
130+
notes.append(
131+
f"Estimated VRAM ({vram_bytes / 1024**3:.1f} GB) is close to free GPU memory "
132+
f"({free_bytes / 1024**3:.1f} GB of {total_bytes / 1024**3:.1f} GB). "
133+
"Consider reducing micro-batch size or enabling activation checkpointing."
134+
)
135+
if label == "Advisor: balanced":
136+
label = "Advisor: memory check"
137+
except Exception:
138+
pass
139+
elif vram_bytes > 3.5 * 1024**3:
140+
notes.append("Estimated VRAM is high. Try lower batch, context, embedding, or layers.")
141+
if label == "Advisor: balanced":
142+
label = "Advisor: memory check"
129143
self.architecture_advisor_metric.setText(label)
130144
self._tip(self.architecture_advisor_metric, "\n".join(notes))
131145

interface/screens/training_screen.py

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,19 @@ def _configure_device_options(self) -> None:
4646

4747
self.device.clear()
4848
if torch.cuda.is_available():
49-
device_name = torch.cuda.get_device_name(0)
49+
device_count = torch.cuda.device_count()
5050
self.device.addItem("cuda")
51+
if device_count > 1:
52+
for idx in range(device_count):
53+
self.device.addItem(f"cuda:{idx}")
5154
self.device.addItem("cpu")
52-
self.device_info.setText(f"CUDA ready: {device_name}")
55+
device_name = torch.cuda.get_device_name(0)
56+
try:
57+
free_bytes, total_bytes = torch.cuda.mem_get_info(0)
58+
vram_gb = total_bytes / (1024 ** 3)
59+
self.device_info.setText(f"CUDA ready: {device_name} ({vram_gb:.1f} GB VRAM)")
60+
except Exception:
61+
self.device_info.setText(f"CUDA ready: {device_name}")
5362
self.use_amp_default = True
5463
else:
5564
self.device.addItem("cpu")
@@ -220,7 +229,24 @@ def _apply_profile_runtime_defaults(self, profile: str) -> None:
220229
batch_size = 32
221230
else:
222231
batch_size = 16
223-
batch_size = min(self.batch_size.maximum(), batch_size)
232+
233+
# Scale micro-batch dynamically based on detected hardware capacity
234+
if torch.cuda.is_available():
235+
try:
236+
_, total_bytes = torch.cuda.mem_get_info()
237+
vram_total_gb = total_bytes / (1024 ** 3)
238+
if vram_total_gb >= 22.0:
239+
batch_size = min(batch_size * 2, 64)
240+
elif vram_total_gb < 6.0:
241+
batch_size = min(batch_size, 4)
242+
elif vram_total_gb < 8.0:
243+
batch_size = min(batch_size, 8)
244+
except Exception:
245+
pass
246+
else:
247+
batch_size = min(batch_size, 4)
248+
249+
batch_size = min(self.batch_size.maximum(), max(1, batch_size))
224250
self.batch_size.setValue(batch_size)
225251
accumulation = max(1, (target_effective_batch + batch_size - 1) // batch_size)
226252
self.gradient_accumulation.setValue(min(self.gradient_accumulation.maximum(), accumulation))
@@ -262,6 +288,9 @@ def apply_training_profile(self) -> None:
262288
"""
263289

264290
profile = self.training_profile.currentText()
291+
bf16_supported = torch.cuda.is_available() and getattr(torch.cuda, "is_bf16_supported", lambda: False)()
292+
rec_precision = "BF16" if bf16_supported else ("FP16" if torch.cuda.is_available() else "FP32")
293+
265294
if profile == "Low-memory":
266295
self._set_combo_text(self.optimizer_name, "Adafactor")
267296
self._set_combo_text(self.scheduler_name, "Cosine decay")
@@ -270,7 +299,7 @@ def apply_training_profile(self) -> None:
270299
self.min_lr_ratio.setValue(0.05)
271300
self.polynomial_power.setValue(1.0)
272301
self.max_grad_norm.setValue(1.0)
273-
self._set_combo_text(self.precision, "BF16" if torch.cuda.is_available() else "FP32")
302+
self._set_combo_text(self.precision, rec_precision)
274303
self.use_amp.setChecked(True)
275304
self._set_combo_text(self.attention_type, "Grouped-query")
276305
self.kv_head_count.setValue(max(1, self.n_head.value() // 2))
@@ -293,7 +322,7 @@ def apply_training_profile(self) -> None:
293322
self.min_lr_ratio.setValue(0.1)
294323
self.polynomial_power.setValue(1.0)
295324
self.max_grad_norm.setValue(0.5)
296-
self._set_combo_text(self.precision, "FP16")
325+
self._set_combo_text(self.precision, rec_precision)
297326
self.use_amp.setChecked(True)
298327
self.activation_checkpointing.setChecked(False)
299328
self.batch_size.setValue(16)
@@ -320,7 +349,7 @@ def apply_training_profile(self) -> None:
320349
self.max_grad_norm.setValue(1.0)
321350
# Lion is reported to be more sensitive to fp16 under/overflow
322351
# than AdamW; prefer bf16 where available, fp32 otherwise.
323-
self._set_combo_text(self.precision, "BF16" if torch.cuda.is_available() else "FP32")
352+
self._set_combo_text(self.precision, rec_precision)
324353
self.use_amp.setChecked(True)
325354
self.activation_checkpointing.setChecked(False)
326355
self.batch_size.setValue(16)
@@ -337,7 +366,7 @@ def apply_training_profile(self) -> None:
337366
self.min_lr_ratio.setValue(0.1)
338367
self.polynomial_power.setValue(1.0)
339368
self.max_grad_norm.setValue(1.0)
340-
self._set_combo_text(self.precision, "FP16")
369+
self._set_combo_text(self.precision, rec_precision)
341370
self.use_amp.setChecked(True)
342371
self.activation_checkpointing.setChecked(False)
343372
self.batch_size.setValue(16)
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
"""Unit tests for conversation fine-tuning and multi-turn chat handling."""
2+
3+
import tempfile
4+
import unittest
5+
from pathlib import Path
6+
7+
import numpy as np
8+
import torch
9+
from tokenizers import Tokenizer
10+
from tokenizers.models import BPE
11+
from tokenizers.trainers import BpeTrainer
12+
from tokenizers.pre_tokenizers import ByteLevel
13+
14+
from engine.config import ModelConfig
15+
from engine.data_core import _extract_structured_text
16+
from engine.microgpt_chat import STOP_SEQUENCES, MicroGPTChatSession
17+
from engine.target_masking import (
18+
IGNORE_INDEX,
19+
InstructionDataset,
20+
find_completion_spans,
21+
mask_prompt_targets,
22+
)
23+
24+
25+
class TestConversationFineTune(unittest.TestCase):
26+
def setUp(self) -> None:
27+
self.tokenizer = Tokenizer(BPE(unk_token="<unk>"))
28+
self.tokenizer.pre_tokenizer = ByteLevel()
29+
trainer = BpeTrainer(special_tokens=["<unk>", "<|endoftext|>", "<pad>", "<bos>", "<eos>"], vocab_size=500)
30+
sample_corpus = [
31+
"System: You are a conversational assistant.\n"
32+
"User: Hi\nAssistant: Hello! How can I help?\n"
33+
"User: Tell me more.\nAssistant: Certainly, here is more detail.\n<eos>\n",
34+
]
35+
self.tokenizer.train_from_iterator(sample_corpus, trainer)
36+
37+
def test_multi_turn_spans_detection(self) -> None:
38+
record = {
39+
"messages": [
40+
{"role": "system", "content": "Be concise."},
41+
{"role": "user", "content": "Question 1"},
42+
{"role": "assistant", "content": "Answer 1"},
43+
{"role": "user", "content": "Question 2"},
44+
{"role": "assistant", "content": "Answer 2"},
45+
]
46+
}
47+
text = _extract_structured_text(record, "conversation")
48+
self.assertIn("System: Be concise.", text)
49+
self.assertIn("User: Question 1", text)
50+
self.assertIn("Assistant: Answer 1", text)
51+
52+
spans = find_completion_spans(text)
53+
self.assertEqual(len(spans), 2)
54+
span1 = text[spans[0][0]:spans[0][1]]
55+
span2 = text[spans[1][0]:spans[1][1]]
56+
self.assertEqual(span1.strip(), "Answer 1")
57+
self.assertEqual(span2.strip(), "Answer 2")
58+
59+
def test_multi_turn_target_masking(self) -> None:
60+
text = (
61+
"System: Sys\n"
62+
"User: Q1\n"
63+
"Assistant: A1\n"
64+
"User: Q2\n"
65+
"Assistant: A2\n"
66+
"<eos>"
67+
)
68+
enc = self.tokenizer.encode(text)
69+
targets = mask_prompt_targets(text, self.tokenizer, enc.ids, enc.offsets)
70+
self.assertEqual(len(targets), len(enc.ids))
71+
72+
# First tokens (System: Sys) must be -100
73+
self.assertEqual(targets[0], IGNORE_INDEX)
74+
75+
# Unmasked tokens must exist for both A1 and A2
76+
unmasked = [tid for tid in targets if tid != IGNORE_INDEX]
77+
self.assertGreater(len(unmasked), 0)
78+
79+
# Trailing <eos> must be unmasked
80+
self.assertEqual(targets[-1], enc.ids[-1])
81+
82+
def test_instruction_dataset_keeps_multi_turn_intact(self) -> None:
83+
# Single 5-turn conversation ending in EOS=4
84+
tokens = np.array([10, 11, 12, 13, 14, 15, 16, 17, 4], dtype=np.int64)
85+
targets = np.array([-100, -100, 12, -100, -100, 15, 16, 17, 4], dtype=np.int64)
86+
ds = InstructionDataset(tokens, targets=targets, context_length=32, eos_token_id=4)
87+
# Should be exactly 1 discrete sample containing all turns
88+
self.assertEqual(len(ds), 1)
89+
x, y = ds[0]
90+
self.assertEqual(len(x), len(tokens) - 1)
91+
self.assertEqual(len(y), len(targets) - 1)
92+
93+
def test_microgpt_chat_stop_sequences_contain_user(self) -> None:
94+
self.assertIn("\nUser:", STOP_SEQUENCES)
95+
self.assertIn("\nSystem:", STOP_SEQUENCES)
96+
self.assertIn("<eos>", STOP_SEQUENCES)
97+
98+
def test_microgpt_chat_render_prompt_pruning(self) -> None:
99+
session = MicroGPTChatSession.__new__(MicroGPTChatSession)
100+
session.config = ModelConfig(vocab_size=500, context_length=64)
101+
session.tokenizer = self.tokenizer
102+
session._messages = [
103+
{"role": "user", "content": "Very old question that should be dropped"},
104+
{"role": "assistant", "content": "Very old answer that should be dropped"},
105+
{"role": "user", "content": "Recent question"},
106+
{"role": "assistant", "content": "Recent answer"},
107+
]
108+
prompt = "Current question"
109+
system_prompt = "Permanent System Instruction"
110+
111+
rendered = session._render_prompt(
112+
prompt, system_prompt, reasoning_effort="None", thinking_enabled=False, max_tokens=16
113+
)
114+
115+
# Permanent system instruction and latest turn MUST be preserved
116+
self.assertIn("System: Permanent System Instruction", rendered)
117+
self.assertIn("User: Current question", rendered)
118+
self.assertIn("Assistant:", rendered)
119+
120+
# Token length must fit inside context_length - 16
121+
enc = self.tokenizer.encode(rendered)
122+
self.assertLessEqual(len(enc.ids), session.config.context_length - 16)
123+
124+
125+
if __name__ == "__main__":
126+
unittest.main()

0 commit comments

Comments
 (0)