Consume cchs_missing_data.yaml: schema-driven fallback pattern and priority rules

The CCHS missing-data pattern schema was shipped but dormant. It now has
two thin consumers reading only its normative blocks:

- clean_variables()'s fallback pattern (used when a variable has no
  worksheet metadata) is built from the schema's pattern families instead
  of hardcoded values. Net behavior gain: the early-cycle decimal missing
  codes (999.6-999.9), declared in the schema but absent from the old
  hardcoded fallback, are now handled.
- load_priority_rules() resolves the not-applicable-over-missing priority
  from the schema's declared hierarchy, replacing the built-in fallback
  and its once-per-session 'YAML files not found' warning. The priority
  decision is now read from a versioned, reviewable file.

The schema gains an applies_to: source_codes declaration (codes are in
the raw StatCan domain, not the harmonized domain) and a consumer note
marking variable_assignments as reference-only - variable_details.csv
remains the per-variable source of truth. Also adds .DS_Store to
.Rbuildignore so Finder droppings stay out of the built package.

Suite: 842 passing, 0 failures. Relates to CEP-017 Track 3.

Author: DougManuel

.Rbuildignore

@@ -13,3 +13,4 @@
 ^papers$
 ^\.claude$
 ^ceps$
+^.*\.DS_Store$

NAMESPACE

@@ -110,7 +110,7 @@ export(has_cached_pattern)
 export(if_else2)
 export(is_equal)
 export(list_subjects)
-export(load_database_registry)
+export(load_cchs_missing_data)
 export(load_schema)
 export(load_worksheet_metadata)
 export(load_worksheet_schemas)

R/clean-variables.R

@@ -127,22 +127,17 @@ clean_variables <- function(vars, output_format = "tagged_na", check_length = TR
     pattern <- tryCatch({
       get_complete_pattern(var_name)
     }, error = function(e) {
-      # Fallback to default CCHS pattern when metadata lookup fails
-      # This allows functions to work before worksheets are fully migrated
+      # Fallback to the declared CCHS pattern schema when metadata lookup
+      # fails. This allows functions to work before worksheets are fully
+      # migrated.
       warning_key <- paste0("pattern_fallback_", var_name)
       cache <- .get_pattern_warnings_cache()
       if (!exists(warning_key, envir = cache)) {
         assign(warning_key, TRUE, envir = cache)
         warning("Using default CCHS pattern for '", var_name,
                 "' (metadata lookup failed). ", call. = FALSE)
       }
-      # Return default CCHS single-digit pattern (most common for categorical vars)
-      list(
-        na_a_codes = c(6, 96, 996),   # Not applicable
-        na_b_codes = c(7, 8, 9, 97, 98, 99, 997, 998, 999),  # Not stated/don't know/refusal
-        copy_mappings = list(list(min = 1, max = 95)),  # Valid range for most vars
-        else_mappings = list()
-      )
+      .default_cchs_pattern()
     })
 
     # Process data using pattern and output format
@@ -534,6 +529,46 @@ apply_else_rule <- function(value, else_mappings) {
 # UTILITY FUNCTIONS
 # ==============================================================================
 
+#' Default CCHS Missing Pattern from the Declared Schema
+#'
+#' Builds the fallback missing pattern from the CCHS missing-data schema
+#' (inst/metadata/schemas/cchs/cchs_missing_data.yaml): the union of all
+#' pattern families' codes by tag, including the early-cycle decimal
+#' variants. Used when a variable has no worksheet metadata. If the schema
+#' cannot be read, falls back to the equivalent hardcoded values so the
+#' cleaning path never fails on a schema problem.
+#'
+#' @return Pattern list with na_a_codes, na_b_codes, copy_mappings,
+#'   else_mappings
+#' @noRd
+.default_cchs_pattern <- function() {
+  na_a <- c(6, 96, 996)
+  na_b <- c(7, 8, 9, 97, 98, 99, 997, 998, 999)
+
+  tryCatch({
+    schema <- load_cchs_missing_data()
+    a <- numeric(0)
+    b <- numeric(0)
+    for (fam in schema$pattern_definitions$patterns) {
+      hier <- fam$priority_hierarchy
+      a <- c(a, unlist(hier$not_applicable$original_codes),
+             unlist(hier$not_applicable$decimal_codes))
+      b <- c(b, unlist(hier$missing_data$original_codes),
+             unlist(hier$missing_data$decimal_codes))
+    }
+    if (length(a) > 0) na_a <- sort(unique(as.numeric(a)))
+    if (length(b) > 0) na_b <- sort(unique(as.numeric(b)))
+  }, error = function(e) NULL)
+
+  list(
+    na_a_codes = na_a,
+    na_b_codes = na_b,
+    # Valid range for most variables; not declared in the schema
+    copy_mappings = list(list(min = 1, max = 95)),
+    else_mappings = list()
+  )
+}
+
 #' Coerce CCHS Label Strings to Numeric with Tagged NAs
 #'
 #' Converts character input to numeric while mapping recognized CCHS

R/load-schema.R

@@ -60,6 +60,43 @@ load_schema <- function(file_type) {
 #' \dontrun{
 #' load_database_registry()
 #' }
+#' Load the CCHS missing-data pattern schema
+#'
+#' @description Loads the machine-actionable CCHS missing-data pattern
+#'   definitions (inst/metadata/schemas/cchs/cchs_missing_data.yaml): the
+#'   single/double/triple-digit code families with their decimal-era
+#'   variants and the not-applicable-over-missing priority hierarchy.
+#'
+#'   Consumers read the normative blocks only (`pattern_definitions` and
+#'   `transformation_rules$na_category_definitions`); the file's
+#'   variable-level assignments are reference documentation -- the
+#'   per-variable source of truth remains variable_details.csv.
+#'
+#' @return Named list parsed from the YAML schema.
+#'
+#' @export
+#'
+#' @examples
+#' \dontrun{
+#' schema <- load_cchs_missing_data()
+#' names(schema$pattern_definitions$patterns)
+#' }
+load_cchs_missing_data <- function() {
+  schema_path <- system.file(
+    "metadata", "schemas", "cchs", "cchs_missing_data.yaml",
+    package = "cchsflow",
+    mustWork = TRUE
+  )
+
+  tryCatch(
+    yaml::read_yaml(schema_path),
+    error = function(e) {
+      stop("Failed to load the CCHS missing-data schema at ", schema_path,
+           ": ", e$message)
+    }
+  )
+}
+
 load_database_registry <- function(registry_file = "database_registry.yaml") {
   registry_path <- system.file(
     "metadata", "schemas", "core", registry_file,

R/missing-data-functions.R

@@ -190,7 +190,21 @@ load_priority_rules <- function() {
       warning("Could not load core priority rules: ", e$message)
     })
   }
-  
+
+  # Read the declared priority hierarchy from the CCHS missing-data schema:
+  # every pattern family declares not_applicable wins over missing_data
+  # (inst/metadata/schemas/cchs/cchs_missing_data.yaml)
+  if (is.null(rules)) {
+    tryCatch({
+      schema <- load_cchs_missing_data()
+      fams <- schema$pattern_definitions$patterns
+      if (length(fams) > 0 &&
+          !is.null(fams[[1]]$priority_hierarchy$not_applicable)) {
+        rules <- list(na_a = 1, na_b = 2)
+      }
+    }, error = function(e) NULL)
+  }
+
   # Built-in fallback (general data handling: "Not Applicable" > "Not Stated")
   if (is.null(rules)) {
     rules <- list(na_a = 1, na_b = 2)  # na_a higher priority as general fallback

inst/metadata/schemas/cchs/cchs_missing_data.yaml

@@ -2,6 +2,14 @@ schema_version: "1.0.0"
 schema_date: "2025-01-20"
 description: "CCHS missing data patterns and transformation rules - machine-actionable metadata reference"
 
+# All codes and ranges in this schema are in the SOURCE coding domain (raw
+# StatCan codes), not the harmonized/derived domain. Consumers: the
+# clean_variables() fallback pattern and load_priority_rules() read the
+# normative blocks (pattern_definitions, na_category_definitions);
+# variable_assignments is reference documentation only - the per-variable
+# source of truth is variable_details.csv.
+applies_to: "source_codes"
+
 # Cross-reference with human-readable documentation
 related_documentation:
   user_guide: "vignettes/missing_value_conventions.qmd"
@@ -201,7 +209,7 @@ implementation:
 # ============================================================================
 metadata:
   schema_authority: "cchsflow development team"
-  last_reviewed: "2025-01-20"
+  last_reviewed: "2026-07-15"
   next_review_due: "2025-07-20"
   version_history:
     "1.0.0": "Initial comprehensive specification with standardized pattern names (single/double/triple_digit_missing)"

man/load_cchs_missing_data.Rd

@@ -461,3 +461,26 @@ test_that("check_cross_file_keys flags orphaned variable_details entries", {
   expect_equal(errs[[1]]$variable, "ZZZ_typo")
   expect_equal(errs[[1]]$error_type, "orphaned_variable_details")
 })
+
+# ==============================================================================
+# CCHS missing-data schema consumption
+# ==============================================================================
+
+test_that("the fallback pattern comes from the CCHS missing-data schema", {
+  pat <- cchsflow:::.default_cchs_pattern()
+  expect_true(all(c(6, 96, 996, 999.6) %in% pat$na_a_codes))
+  expect_true(all(c(7, 8, 9, 97, 98, 99, 997, 998, 999, 999.7, 999.8, 999.9)
+                  %in% pat$na_b_codes))
+})
+
+test_that("priority rules resolve from the schema without a fallback warning", {
+  # Clear the session cache so load_priority_rules() re-resolves
+  rm(list = ls(envir = cchsflow:::.priority_rules_cache),
+     envir = cchsflow:::.priority_rules_cache)
+  expect_no_warning(rules <- cchsflow:::load_priority_rules())
+  expect_equal(rules$na_a, 1)
+  expect_equal(rules$na_b, 2)
+  # Behavioural check: not applicable wins over missing
+  pri <- get_priority_missing(haven::tagged_na("a"), haven::tagged_na("b"))
+  expect_true(haven::is_tagged_na(pri, "a"))
+})