Add subset option for faster iteration (#154)

* Upload first draft

* Revert upload toggle

* Toggle subset for testing

* Toggle subset

* Guard against low sale volume

* Lint

* Make names simpler

* Adjust lock file

* Update dvc

* enable subset for bartch

* Fix breaks

* Update lock and params

* Add bold and italics

* Revert lock file

* Update params.yaml

set subset enable to true to test

* Update params.yaml

Changed the default for subset enable back to false

---------

Co-authored-by: TimCookCountyDS <Timothy.Sparer@cookcountyil.gov>
Co-authored-by: William Ridgeway <10358980+wrridgeway@users.noreply.github.com>

Author: 3 people

params.yaml

@@ -74,6 +74,16 @@ input:
     weight_min: 0.1
     weight_max: 1.0
 
+  # Parameters for running the pipeline with a stratified subset of training
+  # data. Useful for faster development iterations. The subset is stratified
+  # by year, township, and class to maintain representation across categories
+  subset:
+    # Whether to use a subset of the training data instead of the full dataset
+    enable: false
+
+    # Fraction of the training data to keep (between 0 and 1)
+    fraction: 0.25
+
 
 # Cross-validation -------------------------------------------------------------
 

pipeline/00-ingest.R

@@ -615,6 +615,25 @@ training_data_clean <- training_data_clean %>%
   as_tibble() %>%
   write_parquet(paths$input$training$local)
 
+# Reproducible via the model seed and the subset fraction params tracked by DVC
+if (params$input$subset$enable) {
+  set.seed(params$model$seed)
+
+  message(
+    "Creating stratified training data subset ",
+    "(fraction: ", params$input$subset$fraction, ")"
+  )
+
+  training_data_clean %>%
+    group_by(time_sale_year, meta_township_code, meta_class) %>%
+    group_modify(~ {
+      n_sample <- max(1L, ceiling(nrow(.x) * params$input$subset$fraction))
+      slice_sample(.x, n = n_sample)
+    }) %>%
+    ungroup() %>%
+    write_parquet(paths$input$training$local)
+}
+
 assessment_data_clean <- assessment_data_clean %>%
   left_join(
     bldg_rolling_means_dt %>%

pipeline/05-finalize.R

@@ -69,6 +69,8 @@ metadata <- tibble::tibble(
   input_min_sale_year = params$input$min_sale_year,
   input_max_sale_year = params$input$max_sale_year,
   input_n_years_prior = params$input$n_years_prior,
+  input_subset_enable = params$input$subset$enable,
+  input_subset_fraction = params$input$subset$fraction,
   input_building_weight_min = params$input$building$weight_min,
   input_building_weight_max = params$input$building$weight_max,
   ratio_study_far_year = params$ratio_study$far_year,

reports/performance/_model.qmd

@@ -2,6 +2,22 @@
 source("../_setup.R")
 ```
 
+```{r _subset_warning, results='asis'}
+if (isTRUE(metadata$input_subset_enable)) {
+  cat(paste0(
+    "\n\n::: {.callout-warning}\n",
+    "## Subset Mode Active\n\n",
+    "This model was trained on a **",
+    scales::percent(metadata$input_subset_fraction, accuracy = 1),
+    " stratified subset** of the training data. ",
+    "Test set performance metrics are based on the subset and may not ",
+    "reflect full model performance. Assessment set performance uses the ",
+    "full assessment data.\n",
+    ":::\n\n"
+  ))
+}
+```
+
 # Model
 
 The following document describes performance for ```r metadata$run_id```, a ```r model_parameter_final$engine``` model built to re-assess the ```r metadata$assessment_triad``` triad in ```r metadata$assessment_year```. The document uses four sets of data to measure performance:
@@ -188,7 +204,7 @@ model_township_stats_assessment_df %>%
 
 :::
 
-Prior year values (latest stage available for each year) compared to target year sales and estimates.
+Prior year values (latest stage available for each year) compared to ***target year*** sales and estimates.
 The median percent change is comparing the nearest assessed value to the estimated value.
 PIN and sale counts include residential, regression-class properties only.
 
@@ -281,9 +297,11 @@ model_ratio_distribution_df <- model_performance_test_quantile_merged %>%
     )
   )
 
-model_ratio_distribution_labels <- model_ratio_distribution_df %>%
+model_ratio_distribution <- model_ratio_distribution_df %>%
   filter(stage == "Main Model") %>%
-  pull(custom_label)
+  arrange(quantile)
+model_ratio_distribution_breaks <- model_ratio_distribution$quantile
+model_ratio_distribution_labels <- model_ratio_distribution$custom_label
 
 # Manually set axes to highest and lowest contained in the data
 model_ratio_distribution_lims <- model_performance_test_quantile_merged %>%
@@ -318,7 +336,10 @@ model_ratio_distribution_triad_plot <- ggplot(
   ) +
   geom_hline(yintercept = 1, linetype = "dashed", color = "darkgray") +
   labs(x = "Decile", y = "Median Ratio", color = "Model Type") +
-  scale_x_continuous(breaks = 1:10, labels = model_ratio_distribution_labels) +
+  scale_x_continuous(
+    breaks = model_ratio_distribution_breaks,
+    labels = model_ratio_distribution_labels
+  ) +
   scale_color_manual(
     values = c(
       "Main Model" = plot_colors$main,
@@ -366,9 +387,11 @@ generate_model_ratio_township_graph <- function(data, township, lims) {
       )
     )
 
-  data_to_plot_labels <- data_to_plot %>%
+  data_to_plot_main <- data_to_plot %>%
     filter(stage == "Main Model") %>%
-    pull(custom_label)
+    arrange(quantile)
+  data_to_plot_breaks <- data_to_plot_main$quantile
+  data_to_plot_labels <- data_to_plot_main$custom_label
 
   ggplot(
     data = data_to_plot,
@@ -400,7 +423,10 @@ generate_model_ratio_township_graph <- function(data, township, lims) {
       lims$min_value,
       lims$max_value + 0.08
     )) +
-    scale_x_continuous(breaks = 1:10, labels = data_to_plot_labels) +
+    scale_x_continuous(
+      breaks = data_to_plot_breaks,
+      labels = data_to_plot_labels
+    ) +
     scale_color_manual(
       values = c(
         "Main Model" = plot_colors$main,
@@ -937,11 +963,10 @@ model_big_misses_test <- test_card %>%
   ) %>%
   mutate(
     Difference = abs(`Sale Price` - `Est. FMV`),
-    `Qnt.` = cut(
-      `Sale Price`,
-      breaks = quantile(`Sale Price`, probs = c(0, 0.25, 0.5, 0.75, 1)),
-      labels = c("Q1", "Q2", "Q3", "Q4"),
-      include.lowest = TRUE
+    `Qnt.` = factor(
+      ntile(`Sale Price`, n = 4),
+      levels = 1:4,
+      labels = c("Q1", "Q2", "Q3", "Q4")
     ),
     .by = Town
   ) %>%
@@ -995,11 +1020,10 @@ model_big_misses_assessment <- assessment_pin %>%
   ) %>%
   mutate(
     Difference = abs(`Sale 1 Price` - `Est. FMV`),
-    `Qnt.` = cut(
-      `Sale 1 Price`,
-      breaks = quantile(`Sale 1 Price`, probs = c(0, 0.25, 0.5, 0.75, 1)),
-      labels = c("Q1", "Q2", "Q3", "Q4"),
-      include.lowest = TRUE
+    `Qnt.` = factor(
+      ntile(`Sale 1 Price`, n = 4),
+      levels = 1:4,
+      labels = c("Q1", "Q2", "Q3", "Q4")
     ),
     .by = Town
   ) %>%

reports/performance/_outliers.qmd

@@ -300,9 +300,10 @@ outliers_ratio_comparison <- training_data_outlier_analysis %>%
   ungroup() %>%
   distinct(meta_township_name, percent, above_below, triad)
 
-axis_limit_outlier_ratio_comparison <- outliers_ratio_comparison %>%
-  slice_max(percent, n = 1) %>%
-  pull(percent)
+axis_limit_outlier_ratio_comparison <- max(
+  outliers_ratio_comparison$percent,
+  na.rm = TRUE
+)
 
 outliers_ratio_comparison %>%
   ggplot(aes(x = reorder(meta_township_name, percent), y = percent)) +