added SGD and Adam

Author: hexqnt

Cargo.lock

@@ -24,6 +24,7 @@ egui_extras = { version = "0.34.0", default-features = false, features = [
     "svg_text",
 ] }
 egui_plot = "0.35.0"
+stochastic_optimizers = "0.3.0"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 eframe = { version = "0.34.0", default-features = false, features = [

Cargo.toml

@@ -40,10 +40,11 @@ use self::i18n::{
     stop_icon_image, tool_icon_image, tool_label, tr, undo_icon_image, view_icon_image,
 };
 use self::optimizer::{
-    LbfgsInputState, NelderMeadInputState, OptimizerPreset, OptimizerUiMode,
-    SteepestDescentInputState, infer_lbfgs_preset, infer_nelder_mead_preset,
-    infer_steepest_descent_preset, lbfgs_config_from_preset, nelder_mead_config_from_preset,
-    optimizer_method_label, optimizer_preset_label, steepest_descent_config_from_preset,
+    AdamInputState, LbfgsInputState, NelderMeadInputState, OptimizerPreset, OptimizerUiMode,
+    SgdInputState, SteepestDescentInputState, adam_config_from_preset, infer_adam_preset,
+    infer_lbfgs_preset, infer_nelder_mead_preset, infer_sgd_preset, infer_steepest_descent_preset,
+    lbfgs_config_from_preset, nelder_mead_config_from_preset, optimizer_method_label,
+    optimizer_preset_label, sgd_config_from_preset, steepest_descent_config_from_preset,
 };
 use self::param_init::{
     data_based_params_for_family, is_advanced_param_init_supported, polynomial_family,
@@ -55,8 +56,8 @@ use self::replay::ReplayState;
 #[cfg(test)]
 use self::replay::{ReplayFrame, ReplayFramePayload};
 use crate::domain::{
-    CurveFamily, CurveParams, FitResult, LbfgsConfig, NelderMeadConfig, OptimizerConfig,
-    OptimizerMethod, Point, Points, SteepestDescentConfig,
+    AdamConfig, CurveFamily, CurveParams, FitResult, LbfgsConfig, NelderMeadConfig,
+    OptimizerConfig, OptimizerMethod, Point, Points, SgdConfig, SteepestDescentConfig,
 };
 use crate::fit::IterationMetricSnapshot;
 use crate::fit::OptimizationLossMetric;
@@ -525,6 +526,10 @@ pub struct CurveFitApp {
     nelder_mead_preset: OptimizerPreset,
     steepest_descent_inputs: SteepestDescentInputState,
     steepest_descent_preset: OptimizerPreset,
+    sgd_inputs: SgdInputState,
+    sgd_preset: OptimizerPreset,
+    adam_inputs: AdamInputState,
+    adam_preset: OptimizerPreset,
     ui_language: UiLanguage,
     plot_tool: PlotTool,
     spray_points_per_second: usize,
@@ -590,6 +595,8 @@ impl CurveFitApp {
             OptimizerMethod::Lbfgs => self.lbfgs_preset,
             OptimizerMethod::NelderMead => self.nelder_mead_preset,
             OptimizerMethod::SteepestDescent => self.steepest_descent_preset,
+            OptimizerMethod::Sgd => self.sgd_preset,
+            OptimizerMethod::Adam => self.adam_preset,
         }
     }
 
@@ -598,6 +605,8 @@ impl CurveFitApp {
             OptimizerMethod::Lbfgs => self.lbfgs_preset = preset,
             OptimizerMethod::NelderMead => self.nelder_mead_preset = preset,
             OptimizerMethod::SteepestDescent => self.steepest_descent_preset = preset,
+            OptimizerMethod::Sgd => self.sgd_preset = preset,
+            OptimizerMethod::Adam => self.adam_preset = preset,
         }
     }
 
@@ -618,6 +627,14 @@ impl CurveFitApp {
                 );
                 self.steepest_descent_preset = preset;
             }
+            OptimizerMethod::Sgd => {
+                self.sgd_inputs = SgdInputState::from_config(&sgd_config_from_preset(preset));
+                self.sgd_preset = preset;
+            }
+            OptimizerMethod::Adam => {
+                self.adam_inputs = AdamInputState::from_config(&adam_config_from_preset(preset));
+                self.adam_preset = preset;
+            }
         }
     }
 
@@ -632,6 +649,8 @@ impl CurveFitApp {
                 .steepest_descent_inputs
                 .to_config()
                 .map(OptimizerConfig::SteepestDescent),
+            OptimizerMethod::Sgd => self.sgd_inputs.to_config().map(OptimizerConfig::Sgd),
+            OptimizerMethod::Adam => self.adam_inputs.to_config().map(OptimizerConfig::Adam),
         }
     }
 
@@ -1055,6 +1074,8 @@ impl Default for CurveFitApp {
         let default_lbfgs = LbfgsConfig::default();
         let default_nelder_mead = NelderMeadConfig::default();
         let default_steepest_descent = SteepestDescentConfig::default();
+        let default_sgd = SgdConfig::default();
+        let default_adam = AdamConfig::default();
 
         Self {
             points: PointsEditorState::default(),
@@ -1072,6 +1093,10 @@ impl Default for CurveFitApp {
                 &default_steepest_descent,
             ),
             steepest_descent_preset: infer_steepest_descent_preset(&default_steepest_descent),
+            sgd_inputs: SgdInputState::from_config(&default_sgd),
+            sgd_preset: infer_sgd_preset(&default_sgd),
+            adam_inputs: AdamInputState::from_config(&default_adam),
+            adam_preset: infer_adam_preset(&default_adam),
             ui_language: UiLanguage::English,
             plot_tool: PlotTool::SinglePoint,
             spray_points_per_second: 140,

src/app.rs

@@ -1,4 +1,6 @@
-use crate::domain::{LbfgsConfig, NelderMeadConfig, OptimizerMethod, SteepestDescentConfig};
+use crate::domain::{
+    AdamConfig, LbfgsConfig, NelderMeadConfig, OptimizerMethod, SgdConfig, SteepestDescentConfig,
+};
 
 use super::{C1_MIN, C2_MAX, STEP_MAX_MAX, STEP_MIN_MIN, UiLanguage};
 
@@ -10,9 +12,13 @@ pub(super) fn optimizer_method_label(
         (UiLanguage::English, OptimizerMethod::Lbfgs) => "LBFGS",
         (UiLanguage::English, OptimizerMethod::NelderMead) => "Nelder-Mead",
         (UiLanguage::English, OptimizerMethod::SteepestDescent) => "Steepest Descent",
+        (UiLanguage::English, OptimizerMethod::Sgd) => "SGD",
+        (UiLanguage::English, OptimizerMethod::Adam) => "Adam",
         (UiLanguage::Russian, OptimizerMethod::Lbfgs) => "LBFGS",
         (UiLanguage::Russian, OptimizerMethod::NelderMead) => "Нелдер-Мид",
         (UiLanguage::Russian, OptimizerMethod::SteepestDescent) => "Наискорейший спуск",
+        (UiLanguage::Russian, OptimizerMethod::Sgd) => "SGD",
+        (UiLanguage::Russian, OptimizerMethod::Adam) => "Adam",
     }
 }
 
@@ -122,6 +128,50 @@ pub(super) fn infer_steepest_descent_preset(config: &SteepestDescentConfig) -> O
     OptimizerPreset::Custom
 }
 
+pub(super) fn sgd_config_from_preset(preset: OptimizerPreset) -> SgdConfig {
+    match preset {
+        OptimizerPreset::Fast => {
+            SgdConfig::try_new(250, 3e-2).expect("fast SGD preset must be valid")
+        }
+        OptimizerPreset::Balanced => SgdConfig::default(),
+        OptimizerPreset::Precise => {
+            SgdConfig::try_new(4_000, 3e-3).expect("precise SGD preset must be valid")
+        }
+        OptimizerPreset::Custom => SgdConfig::default(),
+    }
+}
+
+pub(super) fn infer_sgd_preset(config: &SgdConfig) -> OptimizerPreset {
+    for preset in OptimizerPreset::ALL {
+        if &sgd_config_from_preset(preset) == config {
+            return preset;
+        }
+    }
+    OptimizerPreset::Custom
+}
+
+pub(super) fn adam_config_from_preset(preset: OptimizerPreset) -> AdamConfig {
+    match preset {
+        OptimizerPreset::Fast => {
+            AdamConfig::try_new(200, 2e-2).expect("fast Adam preset must be valid")
+        }
+        OptimizerPreset::Balanced => AdamConfig::default(),
+        OptimizerPreset::Precise => {
+            AdamConfig::try_new(3_000, 1e-3).expect("precise Adam preset must be valid")
+        }
+        OptimizerPreset::Custom => AdamConfig::default(),
+    }
+}
+
+pub(super) fn infer_adam_preset(config: &AdamConfig) -> OptimizerPreset {
+    for preset in OptimizerPreset::ALL {
+        if &adam_config_from_preset(preset) == config {
+            return preset;
+        }
+    }
+    OptimizerPreset::Custom
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub(super) struct LbfgsInputState {
     pub(super) history_size: usize,
@@ -262,3 +312,49 @@ impl SteepestDescentInputState {
         .map_err(|error| error.to_string())
     }
 }
+
+#[derive(Debug, Clone, PartialEq)]
+pub(super) struct SgdInputState {
+    pub(super) max_iters: u64,
+    pub(super) learning_rate: f64,
+}
+
+impl SgdInputState {
+    pub(super) fn from_config(config: &SgdConfig) -> Self {
+        Self {
+            max_iters: config.max_iters,
+            learning_rate: config.learning_rate,
+        }
+    }
+
+    pub(super) fn normalize_after_ui(&mut self) {
+        self.learning_rate = self.learning_rate.clamp(1e-6, 1.0);
+    }
+
+    pub(super) fn to_config(&self) -> Result<SgdConfig, String> {
+        SgdConfig::try_new(self.max_iters, self.learning_rate).map_err(|error| error.to_string())
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub(super) struct AdamInputState {
+    pub(super) max_iters: u64,
+    pub(super) learning_rate: f64,
+}
+
+impl AdamInputState {
+    pub(super) fn from_config(config: &AdamConfig) -> Self {
+        Self {
+            max_iters: config.max_iters,
+            learning_rate: config.learning_rate,
+        }
+    }
+
+    pub(super) fn normalize_after_ui(&mut self) {
+        self.learning_rate = self.learning_rate.clamp(1e-6, 1.0);
+    }
+
+    pub(super) fn to_config(&self) -> Result<AdamConfig, String> {
+        AdamConfig::try_new(self.max_iters, self.learning_rate).map_err(|error| error.to_string())
+    }
+}

src/app/optimizer.rs

@@ -104,6 +104,18 @@ fn optimizer_config_matches_selected_method() {
         app.optimizer_config(),
         Ok(OptimizerConfig::SteepestDescent(_))
     ));
+
+    app.optimizer_method = OptimizerMethod::Sgd;
+    assert!(matches!(
+        app.optimizer_config(),
+        Ok(OptimizerConfig::Sgd(_))
+    ));
+
+    app.optimizer_method = OptimizerMethod::Adam;
+    assert!(matches!(
+        app.optimizer_config(),
+        Ok(OptimizerConfig::Adam(_))
+    ));
 }
 
 #[test]
@@ -174,6 +186,12 @@ fn optimizer_presets_are_stored_per_method() {
     app.optimizer_method = OptimizerMethod::SteepestDescent;
     app.apply_selected_optimizer_preset(OptimizerPreset::Fast);
 
+    app.optimizer_method = OptimizerMethod::Sgd;
+    app.apply_selected_optimizer_preset(OptimizerPreset::Balanced);
+
+    app.optimizer_method = OptimizerMethod::Adam;
+    app.apply_selected_optimizer_preset(OptimizerPreset::Precise);
+
     app.optimizer_method = OptimizerMethod::Lbfgs;
     assert_eq!(app.selected_optimizer_preset(), OptimizerPreset::Fast);
 
@@ -182,6 +200,12 @@ fn optimizer_presets_are_stored_per_method() {
 
     app.optimizer_method = OptimizerMethod::SteepestDescent;
     assert_eq!(app.selected_optimizer_preset(), OptimizerPreset::Fast);
+
+    app.optimizer_method = OptimizerMethod::Sgd;
+    assert_eq!(app.selected_optimizer_preset(), OptimizerPreset::Balanced);
+
+    app.optimizer_method = OptimizerMethod::Adam;
+    assert_eq!(app.selected_optimizer_preset(), OptimizerPreset::Precise);
 }
 
 #[test]
@@ -209,6 +233,66 @@ fn optimizer_preset_changes_active_config_values() {
     assert!(precise_max_iters > fast_max_iters);
 }
 
+#[test]
+fn sgd_preset_changes_active_config_values() {
+    let mut app = CurveFitApp {
+        optimizer_method: OptimizerMethod::Sgd,
+        ..Default::default()
+    };
+    app.apply_selected_optimizer_preset(OptimizerPreset::Fast);
+    let fast_config = app
+        .optimizer_config()
+        .expect("optimizer config must be valid");
+
+    app.apply_selected_optimizer_preset(OptimizerPreset::Precise);
+    let precise_config = app
+        .optimizer_config()
+        .expect("optimizer config must be valid");
+
+    let (fast_max_iters, precise_max_iters, fast_lr, precise_lr) =
+        match (fast_config, precise_config) {
+            (OptimizerConfig::Sgd(fast), OptimizerConfig::Sgd(precise)) => (
+                fast.max_iters,
+                precise.max_iters,
+                fast.learning_rate,
+                precise.learning_rate,
+            ),
+            _ => panic!("SGD must remain active"),
+        };
+    assert!(precise_max_iters > fast_max_iters);
+    assert!(precise_lr < fast_lr);
+}
+
+#[test]
+fn adam_preset_changes_active_config_values() {
+    let mut app = CurveFitApp {
+        optimizer_method: OptimizerMethod::Adam,
+        ..Default::default()
+    };
+    app.apply_selected_optimizer_preset(OptimizerPreset::Fast);
+    let fast_config = app
+        .optimizer_config()
+        .expect("optimizer config must be valid");
+
+    app.apply_selected_optimizer_preset(OptimizerPreset::Precise);
+    let precise_config = app
+        .optimizer_config()
+        .expect("optimizer config must be valid");
+
+    let (fast_max_iters, precise_max_iters, fast_lr, precise_lr) =
+        match (fast_config, precise_config) {
+            (OptimizerConfig::Adam(fast), OptimizerConfig::Adam(precise)) => (
+                fast.max_iters,
+                precise.max_iters,
+                fast.learning_rate,
+                precise.learning_rate,
+            ),
+            _ => panic!("Adam must remain active"),
+        };
+    assert!(precise_max_iters > fast_max_iters);
+    assert!(precise_lr < fast_lr);
+}
+
 #[test]
 fn diagnostics_initialize_stores_iteration_zero_state() {
     let points = line_points();